* drivers/cpufreq/cpufreq_interactive.c
*
* Copyright (C) 2010 Google, Inc.
+ * Copyright (c) 2012, NVIDIA CORPORATION. All rights reserved.
*
* This software is licensed under the terms of the GNU General Public
* License version 2, as published by the Free Software Foundation, and
#include <asm/cputime.h>
-static atomic_t active_count = ATOMIC_INIT(0);
-
struct cpufreq_interactive_cpuinfo {
struct timer_list cpu_timer;
int timer_idlecancel;
u64 time_in_idle;
+ u64 time_in_iowait;
u64 idle_exit_time;
u64 timer_run_time;
int idling;
u64 freq_change_time;
u64 freq_change_time_in_idle;
+ u64 freq_change_time_in_iowait;
+ u64 last_high_freq_time;
struct cpufreq_policy *policy;
struct cpufreq_frequency_table *freq_table;
unsigned int target_freq;
static cpumask_t down_cpumask;
static spinlock_t down_cpumask_lock;
static struct mutex set_speed_lock;
+static struct mutex gov_state_lock;
+static struct kobject *interactive_kobj;
+static unsigned int active_count;
/* Go to max speed when CPU load at or above this value. */
-#define DEFAULT_GO_MAXSPEED_LOAD 95
+#define DEFAULT_GO_MAXSPEED_LOAD 85
static unsigned long go_maxspeed_load;
/* Base of exponential raise to max speed; if 0 - jump to maximum */
static unsigned long boost_factor;
+/* Max frequency boost in Hz; if 0 - no max is enforced */
+static unsigned long max_boost;
+
+/* Consider IO as busy */
+static unsigned long io_is_busy;
+
/*
* Targeted sustainable load relatively to current frequency.
* If 0, target is set realtively to the max speed
/*
* The minimum amount of time to spend at a frequency before we can ramp down.
*/
-#define DEFAULT_MIN_SAMPLE_TIME 20000;
+#define DEFAULT_MIN_SAMPLE_TIME 30000;
static unsigned long min_sample_time;
/*
* The sample rate of the timer used to increase frequency
*/
-#define DEFAULT_TIMER_RATE 10000;
+#define DEFAULT_TIMER_RATE 20000;
static unsigned long timer_rate;
+/*
+ * The minimum delay before frequency is allowed to raise over normal rate.
+ * Since it must remain at high frequency for a minimum of MIN_SAMPLE_TIME
+ * once it rises, setting this delay to a multiple of MIN_SAMPLE_TIME
+ * becomes the best way to enforce a square wave.
+ * e.g. 5*MIN_SAMPLE_TIME = 20% high freq duty cycle
+ */
+#define DEFAULT_HIGH_FREQ_MIN_DELAY 5*DEFAULT_MIN_SAMPLE_TIME
+static unsigned long high_freq_min_delay;
+
+/*
+ * The maximum frequency CPUs are allowed to run normally
+ * 0 if disabled
+ */
+#define DEFAULT_MAX_NORMAL_FREQ 0
+static unsigned long max_normal_freq;
+
+
+/* Defines to control mid-range frequencies */
+#define DEFAULT_MID_RANGE_GO_MAXSPEED_LOAD 95
+
+static unsigned long midrange_freq;
+static unsigned long midrange_go_maxspeed_load;
+static unsigned long midrange_max_boost;
+
static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
unsigned int event);
int cpu_load, int load_since_change, struct cpufreq_policy *policy)
{
unsigned int target_freq;
+ unsigned int maxspeed_load = go_maxspeed_load;
+ unsigned int mboost = max_boost;
/*
* Choose greater of short-term load (since last idle timer
if (load_since_change > cpu_load)
cpu_load = load_since_change;
- if (cpu_load >= go_maxspeed_load) {
+ if (midrange_freq && policy->cur > midrange_freq) {
+ maxspeed_load = midrange_go_maxspeed_load;
+ mboost = midrange_max_boost;
+ }
+
+ if (cpu_load >= maxspeed_load) {
if (!boost_factor)
return policy->max;
target_freq = policy->cur * boost_factor;
+
+ if (mboost && target_freq > policy->cur + mboost)
+ target_freq = policy->cur + mboost;
}
else {
if (!sustain_load)
return target_freq;
}
+static inline cputime64_t get_cpu_iowait_time(
+ unsigned int cpu, cputime64_t *wall)
+{
+ u64 iowait_time = get_cpu_iowait_time_us(cpu, wall);
+
+ if (iowait_time == -1ULL)
+ return 0;
+
+ return iowait_time;
+}
+
static void cpufreq_interactive_timer(unsigned long data)
{
unsigned int delta_idle;
+ unsigned int delta_iowait;
unsigned int delta_time;
int cpu_load;
int load_since_change;
u64 time_in_idle;
+ u64 time_in_iowait;
u64 idle_exit_time;
struct cpufreq_interactive_cpuinfo *pcpu =
&per_cpu(cpuinfo, data);
u64 now_idle;
+ u64 now_iowait;
unsigned int new_freq;
unsigned int index;
unsigned long flags;
* until more time passes).
*/
time_in_idle = pcpu->time_in_idle;
+ time_in_iowait = pcpu->time_in_iowait;
idle_exit_time = pcpu->idle_exit_time;
now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
+ now_iowait = get_cpu_iowait_time(data, NULL);
smp_wmb();
/* If we raced with cancelling a timer, skip. */
goto exit;
delta_idle = (unsigned int) cputime64_sub(now_idle, time_in_idle);
+ delta_iowait = (unsigned int) cputime64_sub(now_iowait, time_in_iowait);
delta_time = (unsigned int) cputime64_sub(pcpu->timer_run_time,
idle_exit_time);
if (delta_idle > delta_time)
cpu_load = 0;
- else
+ else {
+ if (io_is_busy && delta_idle >= delta_iowait)
+ delta_idle -= delta_iowait;
+
cpu_load = 100 * (delta_time - delta_idle) / delta_time;
+ }
delta_idle = (unsigned int) cputime64_sub(now_idle,
pcpu->freq_change_time_in_idle);
+ delta_iowait = (unsigned int) cputime64_sub(now_iowait,
+ pcpu->freq_change_time_in_iowait);
delta_time = (unsigned int) cputime64_sub(pcpu->timer_run_time,
pcpu->freq_change_time);
if ((delta_time == 0) || (delta_idle > delta_time))
load_since_change = 0;
- else
+ else {
+ if (io_is_busy && delta_idle >= delta_iowait)
+ delta_idle -= delta_iowait;
+
load_since_change =
100 * (delta_time - delta_idle) / delta_time;
+ }
/*
* Combine short-term load (since last idle timer started or timer
goto rearm;
}
+ /*
+ * Can only overclock if the delay is satisfy. Otherwise, cap it to
+ * maximum allowed normal frequency
+ */
+ if (max_normal_freq && (new_freq > max_normal_freq)) {
+ if (cputime64_sub(pcpu->timer_run_time, pcpu->last_high_freq_time)
+ < high_freq_min_delay) {
+ new_freq = max_normal_freq;
+ }
+ else {
+ pcpu->last_high_freq_time = pcpu->timer_run_time;
+ }
+ }
+
if (new_freq < pcpu->target_freq) {
pcpu->target_freq = new_freq;
spin_lock_irqsave(&down_cpumask_lock, flags);
pcpu->time_in_idle = get_cpu_idle_time_us(
data, &pcpu->idle_exit_time);
+ pcpu->time_in_iowait = get_cpu_iowait_time(
+ data, NULL);
+
mod_timer(&pcpu->cpu_timer,
jiffies + usecs_to_jiffies(timer_rate));
}
if (!pending) {
pcpu->time_in_idle = get_cpu_idle_time_us(
smp_processor_id(), &pcpu->idle_exit_time);
+ pcpu->time_in_iowait = get_cpu_iowait_time(
+ smp_processor_id(), NULL);
pcpu->timer_idlecancel = 0;
mod_timer(&pcpu->cpu_timer,
jiffies + usecs_to_jiffies(timer_rate));
pcpu->time_in_idle =
get_cpu_idle_time_us(smp_processor_id(),
&pcpu->idle_exit_time);
+ pcpu->time_in_iowait =
+ get_cpu_iowait_time(smp_processor_id(),
+ NULL);
pcpu->timer_idlecancel = 0;
mod_timer(&pcpu->cpu_timer,
jiffies + usecs_to_jiffies(timer_rate));
max_freq = pjcpu->target_freq;
}
- if (max_freq != pcpu->policy->cur)
- __cpufreq_driver_target(pcpu->policy,
- max_freq,
- CPUFREQ_RELATION_H);
+ __cpufreq_driver_target(pcpu->policy,
+ max_freq,
+ CPUFREQ_RELATION_H);
mutex_unlock(&set_speed_lock);
pcpu->freq_change_time_in_idle =
get_cpu_idle_time_us(cpu,
&pcpu->freq_change_time);
+ pcpu->freq_change_time_in_iowait =
+ get_cpu_iowait_time(cpu, NULL);
}
}
max_freq = pjcpu->target_freq;
}
- if (max_freq != pcpu->policy->cur)
- __cpufreq_driver_target(pcpu->policy, max_freq,
- CPUFREQ_RELATION_H);
+ __cpufreq_driver_target(pcpu->policy, max_freq,
+ CPUFREQ_RELATION_H);
mutex_unlock(&set_speed_lock);
pcpu->freq_change_time_in_idle =
get_cpu_idle_time_us(cpu,
&pcpu->freq_change_time);
+ pcpu->freq_change_time_in_iowait =
+ get_cpu_iowait_time(cpu, NULL);
}
}
-static ssize_t show_go_maxspeed_load(struct kobject *kobj,
- struct attribute *attr, char *buf)
-{
- return sprintf(buf, "%lu\n", go_maxspeed_load);
-}
-
-static ssize_t store_go_maxspeed_load(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
-{
- int ret;
- unsigned long val;
-
- ret = strict_strtoul(buf, 0, &val);
- if (ret < 0)
- return ret;
- go_maxspeed_load = val;
- return count;
-}
-
-static struct global_attr go_maxspeed_load_attr = __ATTR(go_maxspeed_load, 0644,
- show_go_maxspeed_load, store_go_maxspeed_load);
-
-static ssize_t show_boost_factor(struct kobject *kobj,
- struct attribute *attr, char *buf)
-{
- return sprintf(buf, "%lu\n", boost_factor);
-}
-
-static ssize_t store_boost_factor(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
-{
- return strict_strtoul(buf, 0, &boost_factor);
-}
-
-static struct global_attr boost_factor_attr = __ATTR(boost_factor, 0644,
- show_boost_factor, store_boost_factor);
-
-static ssize_t show_sustain_load(struct kobject *kobj,
- struct attribute *attr, char *buf)
-{
- return sprintf(buf, "%lu\n", sustain_load);
-}
-
-static ssize_t store_sustain_load(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
-{
- return strict_strtoul(buf, 0, &sustain_load);
-}
-
-static struct global_attr sustain_load_attr = __ATTR(sustain_load, 0644,
- show_sustain_load, store_sustain_load);
-
-static ssize_t show_min_sample_time(struct kobject *kobj,
- struct attribute *attr, char *buf)
-{
- return sprintf(buf, "%lu\n", min_sample_time);
-}
-
-static ssize_t store_min_sample_time(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
-{
- int ret;
- unsigned long val;
-
- ret = strict_strtoul(buf, 0, &val);
- if (ret < 0)
- return ret;
- min_sample_time = val;
- return count;
-}
-
-static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
- show_min_sample_time, store_min_sample_time);
-
-static ssize_t show_timer_rate(struct kobject *kobj,
- struct attribute *attr, char *buf)
-{
- return sprintf(buf, "%lu\n", timer_rate);
-}
-
-static ssize_t store_timer_rate(struct kobject *kobj,
- struct attribute *attr, const char *buf, size_t count)
-{
- int ret;
- unsigned long val;
-
- ret = strict_strtoul(buf, 0, &val);
- if (ret < 0)
- return ret;
- timer_rate = val;
- return count;
-}
-
-static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
- show_timer_rate, store_timer_rate);
+#define DECL_CPUFREQ_INTERACTIVE_ATTR(name) \
+static ssize_t show_##name(struct kobject *kobj, \
+ struct attribute *attr, char *buf) \
+{ \
+ return sprintf(buf, "%lu\n", name); \
+} \
+\
+static ssize_t store_##name(struct kobject *kobj,\
+ struct attribute *attr, const char *buf, size_t count) \
+{ \
+ int ret; \
+ unsigned long val; \
+\
+ ret = strict_strtoul(buf, 0, &val); \
+ if (ret < 0) \
+ return ret; \
+ name = val; \
+ return count; \
+} \
+\
+static struct global_attr name##_attr = __ATTR(name, 0644, \
+ show_##name, store_##name);
+
+DECL_CPUFREQ_INTERACTIVE_ATTR(go_maxspeed_load)
+DECL_CPUFREQ_INTERACTIVE_ATTR(midrange_freq)
+DECL_CPUFREQ_INTERACTIVE_ATTR(midrange_go_maxspeed_load)
+DECL_CPUFREQ_INTERACTIVE_ATTR(boost_factor)
+DECL_CPUFREQ_INTERACTIVE_ATTR(io_is_busy)
+DECL_CPUFREQ_INTERACTIVE_ATTR(max_boost)
+DECL_CPUFREQ_INTERACTIVE_ATTR(midrange_max_boost)
+DECL_CPUFREQ_INTERACTIVE_ATTR(sustain_load)
+DECL_CPUFREQ_INTERACTIVE_ATTR(min_sample_time)
+DECL_CPUFREQ_INTERACTIVE_ATTR(timer_rate)
+DECL_CPUFREQ_INTERACTIVE_ATTR(high_freq_min_delay)
+DECL_CPUFREQ_INTERACTIVE_ATTR(max_normal_freq)
+
+#undef DECL_CPUFREQ_INTERACTIVE_ATTR
static struct attribute *interactive_attributes[] = {
&go_maxspeed_load_attr.attr,
+ &midrange_freq_attr.attr,
+ &midrange_go_maxspeed_load_attr.attr,
&boost_factor_attr.attr,
+ &max_boost_attr.attr,
+ &midrange_max_boost_attr.attr,
+ &io_is_busy_attr.attr,
&sustain_load_attr.attr,
&min_sample_time_attr.attr,
&timer_rate_attr.attr,
+ &high_freq_min_delay_attr.attr,
+ &max_normal_freq_attr.attr,
NULL,
};
&pcpu->freq_change_time);
pcpu->time_in_idle = pcpu->freq_change_time_in_idle;
pcpu->idle_exit_time = pcpu->freq_change_time;
+ pcpu->freq_change_time_in_iowait =
+ get_cpu_iowait_time(j, NULL);
+ pcpu->time_in_iowait = pcpu->freq_change_time_in_iowait;
+ if (!pcpu->last_high_freq_time)
+ pcpu->last_high_freq_time = pcpu->freq_change_time;
pcpu->timer_idlecancel = 1;
pcpu->governor_enabled = 1;
smp_wmb();
mod_timer(&pcpu->cpu_timer, jiffies + 2);
}
+ mutex_lock(&gov_state_lock);
+ active_count++;
/*
* Do not register the idle hook and create sysfs
* entries if we have already done so.
*/
- if (atomic_inc_return(&active_count) > 1)
- return 0;
-
- rc = sysfs_create_group(cpufreq_global_kobject,
- &interactive_attr_group);
- if (rc)
- return rc;
+ if (active_count == 1) {
+ rc = sysfs_create_group(cpufreq_global_kobject,
+ &interactive_attr_group);
+ interactive_kobj = kobject_create_and_add(
+ "gov_interactive",
+ cpufreq_global_kobject);
+ kobject_uevent(interactive_kobj, KOBJ_ADD);
+ if (rc) {
+ mutex_unlock(&gov_state_lock);
+ return rc;
+ }
+ }
+ mutex_unlock(&gov_state_lock);
break;
}
flush_work(&freq_scale_down_work);
- if (atomic_dec_return(&active_count) > 0)
- return 0;
+ mutex_lock(&gov_state_lock);
+
+ active_count--;
+
+ if (active_count == 0) {
+ sysfs_remove_group(cpufreq_global_kobject,
+ &interactive_attr_group);
+ kobject_uevent(interactive_kobj, KOBJ_REMOVE);
+ kobject_put(interactive_kobj);
+ }
- sysfs_remove_group(cpufreq_global_kobject,
- &interactive_attr_group);
+ mutex_unlock(&gov_state_lock);
break;
struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
go_maxspeed_load = DEFAULT_GO_MAXSPEED_LOAD;
+ midrange_go_maxspeed_load = DEFAULT_MID_RANGE_GO_MAXSPEED_LOAD;
min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
timer_rate = DEFAULT_TIMER_RATE;
+ high_freq_min_delay = DEFAULT_HIGH_FREQ_MIN_DELAY;
+ max_normal_freq = DEFAULT_MAX_NORMAL_FREQ;
/* Initalize per-cpu timers */
for_each_possible_cpu(i) {
spin_lock_init(&up_cpumask_lock);
spin_lock_init(&down_cpumask_lock);
mutex_init(&set_speed_lock);
+ mutex_init(&gov_state_lock);
idle_notifier_register(&cpufreq_interactive_idle_nb);