cpufreq: mid-range frequency bias
[linux-2.6.git] / drivers / cpufreq / cpufreq_interactive.c
1 /*
2  * drivers/cpufreq/cpufreq_interactive.c
3  *
4  * Copyright (C) 2010 Google, Inc.
5  * Copyright (c) 2012, NVIDIA CORPORATION.  All rights reserved.
6  *
7  * This software is licensed under the terms of the GNU General Public
8  * License version 2, as published by the Free Software Foundation, and
9  * may be copied, distributed, and modified under those terms.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * Author: Mike Chan (mike@android.com)
17  *
18  */
19
20 #include <linux/cpu.h>
21 #include <linux/cpumask.h>
22 #include <linux/cpufreq.h>
23 #include <linux/module.h>
24 #include <linux/mutex.h>
25 #include <linux/sched.h>
26 #include <linux/tick.h>
27 #include <linux/timer.h>
28 #include <linux/workqueue.h>
29 #include <linux/kthread.h>
30 #include <linux/mutex.h>
31
32 #include <asm/cputime.h>
33
34 static atomic_t active_count = ATOMIC_INIT(0);
35
36 struct cpufreq_interactive_cpuinfo {
37         struct timer_list cpu_timer;
38         int timer_idlecancel;
39         u64 time_in_idle;
40         u64 time_in_iowait;
41         u64 idle_exit_time;
42         u64 timer_run_time;
43         int idling;
44         u64 freq_change_time;
45         u64 freq_change_time_in_idle;
46         u64 freq_change_time_in_iowait;
47         struct cpufreq_policy *policy;
48         struct cpufreq_frequency_table *freq_table;
49         unsigned int target_freq;
50         int governor_enabled;
51 };
52
53 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
54
55 /* Workqueues handle frequency scaling */
56 static struct task_struct *up_task;
57 static struct workqueue_struct *down_wq;
58 static struct work_struct freq_scale_down_work;
59 static cpumask_t up_cpumask;
60 static spinlock_t up_cpumask_lock;
61 static cpumask_t down_cpumask;
62 static spinlock_t down_cpumask_lock;
63 static struct mutex set_speed_lock;
64
65 /* Go to max speed when CPU load at or above this value. */
66 #define DEFAULT_GO_MAXSPEED_LOAD 85
67 static unsigned long go_maxspeed_load;
68
69 /* Base of exponential raise to max speed; if 0 - jump to maximum */
70 static unsigned long boost_factor;
71
72 /* Max frequency boost in Hz; if 0 - no max is enforced */
73 static unsigned long max_boost;
74
75 /* Consider IO as busy */
76 static unsigned long io_is_busy;
77
78 /*
79  * Targeted sustainable load relatively to current frequency.
80  * If 0, target is set realtively to the max speed
81  */
82 static unsigned long sustain_load;
83
84 /*
85  * The minimum amount of time to spend at a frequency before we can ramp down.
86  */
87 #define DEFAULT_MIN_SAMPLE_TIME 30000;
88 static unsigned long min_sample_time;
89
90 /*
91  * The sample rate of the timer used to increase frequency
92  */
93 #define DEFAULT_TIMER_RATE 20000;
94 static unsigned long timer_rate;
95
96 /* Defines to control mid-range frequencies */
97 #define DEFAULT_MID_RANGE_GO_MAXSPEED_LOAD 95
98
99 static unsigned long midrange_freq;
100 static unsigned long midrange_go_maxspeed_load;
101 static unsigned long midrange_max_boost;
102
103 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
104                 unsigned int event);
105
106 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
107 static
108 #endif
109 struct cpufreq_governor cpufreq_gov_interactive = {
110         .name = "interactive",
111         .governor = cpufreq_governor_interactive,
112         .max_transition_latency = 10000000,
113         .owner = THIS_MODULE,
114 };
115
116 static unsigned int cpufreq_interactive_get_target(
117         int cpu_load, int load_since_change, struct cpufreq_policy *policy)
118 {
119         unsigned int target_freq;
120         unsigned int maxspeed_load = go_maxspeed_load;
121         unsigned int mboost = max_boost;
122
123         /*
124          * Choose greater of short-term load (since last idle timer
125          * started or timer function re-armed itself) or long-term load
126          * (since last frequency change).
127          */
128         if (load_since_change > cpu_load)
129                 cpu_load = load_since_change;
130
131         if (midrange_freq && policy->cur > midrange_freq) {
132                 maxspeed_load = midrange_go_maxspeed_load;
133                 mboost = midrange_max_boost;
134         }
135
136         if (cpu_load >= maxspeed_load) {
137                 if (!boost_factor)
138                         return policy->max;
139
140                 target_freq = policy->cur * boost_factor;
141
142                 if (mboost && target_freq > policy->cur + mboost)
143                         target_freq = policy->cur + mboost;
144         }
145         else {
146                 if (!sustain_load)
147                         return policy->max * cpu_load / 100;
148
149                 target_freq = policy->cur * cpu_load / sustain_load;
150         }
151
152         target_freq = min(target_freq, policy->max);
153         return target_freq;
154 }
155
156 static inline cputime64_t get_cpu_iowait_time(
157         unsigned int cpu, cputime64_t *wall)
158 {
159         u64 iowait_time = get_cpu_iowait_time_us(cpu, wall);
160
161         if (iowait_time == -1ULL)
162                 return 0;
163
164         return iowait_time;
165 }
166
167 static void cpufreq_interactive_timer(unsigned long data)
168 {
169         unsigned int delta_idle;
170         unsigned int delta_iowait;
171         unsigned int delta_time;
172         int cpu_load;
173         int load_since_change;
174         u64 time_in_idle;
175         u64 time_in_iowait;
176         u64 idle_exit_time;
177         struct cpufreq_interactive_cpuinfo *pcpu =
178                 &per_cpu(cpuinfo, data);
179         u64 now_idle;
180         u64 now_iowait;
181         unsigned int new_freq;
182         unsigned int index;
183         unsigned long flags;
184
185         smp_rmb();
186
187         if (!pcpu->governor_enabled)
188                 goto exit;
189
190         /*
191          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
192          * this lets idle exit know the current idle time sample has
193          * been processed, and idle exit can generate a new sample and
194          * re-arm the timer.  This prevents a concurrent idle
195          * exit on that CPU from writing a new set of info at the same time
196          * the timer function runs (the timer function can't use that info
197          * until more time passes).
198          */
199         time_in_idle = pcpu->time_in_idle;
200         time_in_iowait = pcpu->time_in_iowait;
201         idle_exit_time = pcpu->idle_exit_time;
202         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
203         now_iowait = get_cpu_iowait_time(data, NULL);
204         smp_wmb();
205
206         /* If we raced with cancelling a timer, skip. */
207         if (!idle_exit_time)
208                 goto exit;
209
210         delta_idle = (unsigned int)(now_idle - time_in_idle);
211         delta_iowait = (unsigned int)(now_iowait - time_in_iowait);
212         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
213
214         /*
215          * If timer ran less than 1ms after short-term sample started, retry.
216          */
217         if (delta_time < 1000)
218                 goto rearm;
219
220         if (delta_idle > delta_time)
221                 cpu_load = 0;
222         else {
223                 if (io_is_busy && delta_idle >= delta_iowait)
224                         delta_idle -= delta_iowait;
225
226                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
227         }
228
229         delta_idle = (unsigned int)(now_idle - pcpu->freq_change_time_in_idle);
230         delta_iowait = (unsigned int)(now_iowait - pcpu->freq_change_time_in_iowait);
231         delta_time = (unsigned int)(pcpu->timer_run_time - pcpu->freq_change_time);
232
233         if ((delta_time == 0) || (delta_idle > delta_time))
234                 load_since_change = 0;
235         else {
236                 if (io_is_busy && delta_idle >= delta_iowait)
237                         delta_idle -= delta_iowait;
238
239                 load_since_change =
240                         100 * (delta_time - delta_idle) / delta_time;
241         }
242
243         /*
244          * Combine short-term load (since last idle timer started or timer
245          * function re-armed itself) and long-term load (since last frequency
246          * change) to determine new target frequency
247          */
248         new_freq = cpufreq_interactive_get_target(cpu_load, load_since_change,
249                                                   pcpu->policy);
250
251         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
252                                            new_freq, CPUFREQ_RELATION_H,
253                                            &index)) {
254                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
255                              (int) data);
256                 goto rearm;
257         }
258
259         new_freq = pcpu->freq_table[index].frequency;
260
261         if (pcpu->target_freq == new_freq)
262                 goto rearm_if_notmax;
263
264         /*
265          * Do not scale down unless we have been at this frequency for the
266          * minimum sample time.
267          */
268         if (new_freq < pcpu->target_freq) {
269                 if (pcpu->timer_run_time - pcpu->freq_change_time
270                     < min_sample_time)
271                         goto rearm;
272         }
273
274         if (new_freq < pcpu->target_freq) {
275                 pcpu->target_freq = new_freq;
276                 spin_lock_irqsave(&down_cpumask_lock, flags);
277                 cpumask_set_cpu(data, &down_cpumask);
278                 spin_unlock_irqrestore(&down_cpumask_lock, flags);
279                 queue_work(down_wq, &freq_scale_down_work);
280         } else {
281                 pcpu->target_freq = new_freq;
282                 spin_lock_irqsave(&up_cpumask_lock, flags);
283                 cpumask_set_cpu(data, &up_cpumask);
284                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
285                 wake_up_process(up_task);
286         }
287
288 rearm_if_notmax:
289         /*
290          * Already set max speed and don't see a need to change that,
291          * wait until next idle to re-evaluate, don't need timer.
292          */
293         if (pcpu->target_freq == pcpu->policy->max)
294                 goto exit;
295
296 rearm:
297         if (!timer_pending(&pcpu->cpu_timer)) {
298                 /*
299                  * If already at min: if that CPU is idle, don't set timer.
300                  * Else cancel the timer if that CPU goes idle.  We don't
301                  * need to re-evaluate speed until the next idle exit.
302                  */
303                 if (pcpu->target_freq == pcpu->policy->min) {
304                         smp_rmb();
305
306                         if (pcpu->idling)
307                                 goto exit;
308
309                         pcpu->timer_idlecancel = 1;
310                 }
311
312                 pcpu->time_in_idle = get_cpu_idle_time_us(
313                         data, &pcpu->idle_exit_time);
314                 pcpu->time_in_iowait = get_cpu_iowait_time(
315                         data, NULL);
316
317                 mod_timer(&pcpu->cpu_timer,
318                           jiffies + usecs_to_jiffies(timer_rate));
319         }
320
321 exit:
322         return;
323 }
324
325 static void cpufreq_interactive_idle_start(void)
326 {
327         struct cpufreq_interactive_cpuinfo *pcpu =
328                 &per_cpu(cpuinfo, smp_processor_id());
329         int pending;
330
331         if (!pcpu->governor_enabled)
332                 return;
333
334         pcpu->idling = 1;
335         smp_wmb();
336         pending = timer_pending(&pcpu->cpu_timer);
337
338         if (pcpu->target_freq != pcpu->policy->min) {
339 #ifdef CONFIG_SMP
340                 /*
341                  * Entering idle while not at lowest speed.  On some
342                  * platforms this can hold the other CPU(s) at that speed
343                  * even though the CPU is idle. Set a timer to re-evaluate
344                  * speed so this idle CPU doesn't hold the other CPUs above
345                  * min indefinitely.  This should probably be a quirk of
346                  * the CPUFreq driver.
347                  */
348                 if (!pending) {
349                         pcpu->time_in_idle = get_cpu_idle_time_us(
350                                 smp_processor_id(), &pcpu->idle_exit_time);
351                         pcpu->time_in_iowait = get_cpu_iowait_time(
352                                 smp_processor_id(), NULL);
353                         pcpu->timer_idlecancel = 0;
354                         mod_timer(&pcpu->cpu_timer,
355                                   jiffies + usecs_to_jiffies(timer_rate));
356                 }
357 #endif
358         } else {
359                 /*
360                  * If at min speed and entering idle after load has
361                  * already been evaluated, and a timer has been set just in
362                  * case the CPU suddenly goes busy, cancel that timer.  The
363                  * CPU didn't go busy; we'll recheck things upon idle exit.
364                  */
365                 if (pending && pcpu->timer_idlecancel) {
366                         del_timer(&pcpu->cpu_timer);
367                         /*
368                          * Ensure last timer run time is after current idle
369                          * sample start time, so next idle exit will always
370                          * start a new idle sampling period.
371                          */
372                         pcpu->idle_exit_time = 0;
373                         pcpu->timer_idlecancel = 0;
374                 }
375         }
376
377 }
378
379 static void cpufreq_interactive_idle_end(void)
380 {
381         struct cpufreq_interactive_cpuinfo *pcpu =
382                 &per_cpu(cpuinfo, smp_processor_id());
383
384         pcpu->idling = 0;
385         smp_wmb();
386
387         /*
388          * Arm the timer for 1-2 ticks later if not already, and if the timer
389          * function has already processed the previous load sampling
390          * interval.  (If the timer is not pending but has not processed
391          * the previous interval, it is probably racing with us on another
392          * CPU.  Let it compute load based on the previous sample and then
393          * re-arm the timer for another interval when it's done, rather
394          * than updating the interval start time to be "now", which doesn't
395          * give the timer function enough time to make a decision on this
396          * run.)
397          */
398         if (timer_pending(&pcpu->cpu_timer) == 0 &&
399             pcpu->timer_run_time >= pcpu->idle_exit_time &&
400             pcpu->governor_enabled) {
401                 pcpu->time_in_idle =
402                         get_cpu_idle_time_us(smp_processor_id(),
403                                              &pcpu->idle_exit_time);
404                 pcpu->time_in_iowait =
405                         get_cpu_iowait_time(smp_processor_id(),
406                                                 NULL);
407                 pcpu->timer_idlecancel = 0;
408                 mod_timer(&pcpu->cpu_timer,
409                           jiffies + usecs_to_jiffies(timer_rate));
410         }
411
412 }
413
414 static int cpufreq_interactive_up_task(void *data)
415 {
416         unsigned int cpu;
417         cpumask_t tmp_mask;
418         unsigned long flags;
419         struct cpufreq_interactive_cpuinfo *pcpu;
420
421         while (1) {
422                 set_current_state(TASK_INTERRUPTIBLE);
423                 spin_lock_irqsave(&up_cpumask_lock, flags);
424
425                 if (cpumask_empty(&up_cpumask)) {
426                         spin_unlock_irqrestore(&up_cpumask_lock, flags);
427                         schedule();
428
429                         if (kthread_should_stop())
430                                 break;
431
432                         spin_lock_irqsave(&up_cpumask_lock, flags);
433                 }
434
435                 set_current_state(TASK_RUNNING);
436                 tmp_mask = up_cpumask;
437                 cpumask_clear(&up_cpumask);
438                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
439
440                 for_each_cpu(cpu, &tmp_mask) {
441                         unsigned int j;
442                         unsigned int max_freq = 0;
443
444                         pcpu = &per_cpu(cpuinfo, cpu);
445                         smp_rmb();
446
447                         if (!pcpu->governor_enabled)
448                                 continue;
449
450                         mutex_lock(&set_speed_lock);
451
452                         for_each_cpu(j, pcpu->policy->cpus) {
453                                 struct cpufreq_interactive_cpuinfo *pjcpu =
454                                         &per_cpu(cpuinfo, j);
455
456                                 if (pjcpu->target_freq > max_freq)
457                                         max_freq = pjcpu->target_freq;
458                         }
459
460                         __cpufreq_driver_target(pcpu->policy,
461                                                 max_freq,
462                                                 CPUFREQ_RELATION_H);
463                         mutex_unlock(&set_speed_lock);
464
465                         pcpu->freq_change_time_in_idle =
466                                 get_cpu_idle_time_us(cpu,
467                                                      &pcpu->freq_change_time);
468                         pcpu->freq_change_time_in_iowait =
469                                 get_cpu_iowait_time(cpu, NULL);
470                 }
471         }
472
473         return 0;
474 }
475
476 static void cpufreq_interactive_freq_down(struct work_struct *work)
477 {
478         unsigned int cpu;
479         cpumask_t tmp_mask;
480         unsigned long flags;
481         struct cpufreq_interactive_cpuinfo *pcpu;
482
483         spin_lock_irqsave(&down_cpumask_lock, flags);
484         tmp_mask = down_cpumask;
485         cpumask_clear(&down_cpumask);
486         spin_unlock_irqrestore(&down_cpumask_lock, flags);
487
488         for_each_cpu(cpu, &tmp_mask) {
489                 unsigned int j;
490                 unsigned int max_freq = 0;
491
492                 pcpu = &per_cpu(cpuinfo, cpu);
493                 smp_rmb();
494
495                 if (!pcpu->governor_enabled)
496                         continue;
497
498                 mutex_lock(&set_speed_lock);
499
500                 for_each_cpu(j, pcpu->policy->cpus) {
501                         struct cpufreq_interactive_cpuinfo *pjcpu =
502                                 &per_cpu(cpuinfo, j);
503
504                         if (pjcpu->target_freq > max_freq)
505                                 max_freq = pjcpu->target_freq;
506                 }
507
508                 __cpufreq_driver_target(pcpu->policy, max_freq,
509                                         CPUFREQ_RELATION_H);
510
511                 mutex_unlock(&set_speed_lock);
512                 pcpu->freq_change_time_in_idle =
513                         get_cpu_idle_time_us(cpu,
514                                              &pcpu->freq_change_time);
515                 pcpu->freq_change_time_in_iowait =
516                         get_cpu_iowait_time(cpu, NULL);
517         }
518 }
519
520 static ssize_t show_go_maxspeed_load(struct kobject *kobj,
521                                      struct attribute *attr, char *buf)
522 {
523         return sprintf(buf, "%lu\n", go_maxspeed_load);
524 }
525
526 static ssize_t store_go_maxspeed_load(struct kobject *kobj,
527                         struct attribute *attr, const char *buf, size_t count)
528 {
529         int ret;
530         unsigned long val;
531
532         ret = strict_strtoul(buf, 0, &val);
533         if (ret < 0)
534                 return ret;
535         go_maxspeed_load = val;
536         return count;
537 }
538
539 static struct global_attr go_maxspeed_load_attr = __ATTR(go_maxspeed_load, 0644,
540                 show_go_maxspeed_load, store_go_maxspeed_load);
541
542 static ssize_t show_midrange_freq(struct kobject *kobj,
543                                      struct attribute *attr, char *buf)
544 {
545         return sprintf(buf, "%lu\n", midrange_freq);
546 }
547
548 static ssize_t store_midrange_freq(struct kobject *kobj,
549                         struct attribute *attr, const char *buf, size_t count)
550 {
551         int ret;
552         unsigned long val;
553
554         ret = strict_strtoul(buf, 0, &val);
555         if (ret < 0)
556                 return ret;
557         midrange_freq = val;
558         return count;
559 }
560
561 static struct global_attr midrange_freq_attr = __ATTR(midrange_freq, 0644,
562                 show_midrange_freq, store_midrange_freq);
563
564 static ssize_t show_midrange_go_maxspeed_load(struct kobject *kobj,
565                                      struct attribute *attr, char *buf)
566 {
567         return sprintf(buf, "%lu\n", midrange_go_maxspeed_load);
568 }
569
570 static ssize_t store_midrange_go_maxspeed_load(struct kobject *kobj,
571                         struct attribute *attr, const char *buf, size_t count)
572 {
573         int ret;
574         unsigned long val;
575
576         ret = strict_strtoul(buf, 0, &val);
577         if (ret < 0)
578                 return ret;
579         midrange_go_maxspeed_load = val;
580         return count;
581 }
582
583 static struct global_attr midrange_go_maxspeed_load_attr = __ATTR(midrange_go_maxspeed_load, 0644,
584                 show_midrange_go_maxspeed_load, store_midrange_go_maxspeed_load);
585
586 static ssize_t show_boost_factor(struct kobject *kobj,
587                                      struct attribute *attr, char *buf)
588 {
589         return sprintf(buf, "%lu\n", boost_factor);
590 }
591
592 static ssize_t store_boost_factor(struct kobject *kobj,
593                         struct attribute *attr, const char *buf, size_t count)
594 {
595         int ret;
596         unsigned long val;
597
598         ret = strict_strtoul(buf, 0, &val);
599         if (ret < 0)
600                 return ret;
601         boost_factor = val;
602         return count;
603 }
604
605 static struct global_attr boost_factor_attr = __ATTR(boost_factor, 0644,
606                 show_boost_factor, store_boost_factor);
607
608 static ssize_t show_io_is_busy(struct kobject *kobj,
609                                      struct attribute *attr, char *buf)
610 {
611         return sprintf(buf, "%lu\n", io_is_busy);
612 }
613
614 static ssize_t store_io_is_busy(struct kobject *kobj,
615                         struct attribute *attr, const char *buf, size_t count)
616 {
617         if (!strict_strtoul(buf, 0, &io_is_busy))
618                 return count;
619         return -EINVAL;
620 }
621
622 static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644,
623                 show_io_is_busy, store_io_is_busy);
624
625 static ssize_t show_max_boost(struct kobject *kobj,
626                                      struct attribute *attr, char *buf)
627 {
628         return sprintf(buf, "%lu\n", max_boost);
629 }
630
631 static ssize_t store_max_boost(struct kobject *kobj,
632                         struct attribute *attr, const char *buf, size_t count)
633 {
634         int ret;
635         unsigned long val;
636
637         ret = strict_strtoul(buf, 0, &val);
638         if (ret < 0)
639                 return ret;
640         max_boost = val;
641         return count;
642 }
643
644 static struct global_attr max_boost_attr = __ATTR(max_boost, 0644,
645                 show_max_boost, store_max_boost);
646
647 static ssize_t show_midrange_max_boost(struct kobject *kobj,
648                                      struct attribute *attr, char *buf)
649 {
650         return sprintf(buf, "%lu\n", midrange_max_boost);
651 }
652
653 static ssize_t store_midrange_max_boost(struct kobject *kobj,
654                         struct attribute *attr, const char *buf, size_t count)
655 {
656         int ret;
657         unsigned long val;
658
659         ret = strict_strtoul(buf, 0, &val);
660         if (ret < 0)
661                 return ret;
662         midrange_max_boost = val;
663         return count;
664 }
665
666 static struct global_attr midrange_max_boost_attr = __ATTR(midrange_max_boost, 0644,
667                 show_midrange_max_boost, store_midrange_max_boost);
668
669 static ssize_t show_sustain_load(struct kobject *kobj,
670                                      struct attribute *attr, char *buf)
671 {
672         return sprintf(buf, "%lu\n", sustain_load);
673 }
674
675 static ssize_t store_sustain_load(struct kobject *kobj,
676                         struct attribute *attr, const char *buf, size_t count)
677 {
678         int ret;
679         unsigned long val;
680
681         ret = strict_strtoul(buf, 0, &val);
682         if (ret < 0)
683                 return ret;
684         sustain_load = val;
685         return count;
686 }
687
688 static struct global_attr sustain_load_attr = __ATTR(sustain_load, 0644,
689                 show_sustain_load, store_sustain_load);
690
691 static ssize_t show_min_sample_time(struct kobject *kobj,
692                                 struct attribute *attr, char *buf)
693 {
694         return sprintf(buf, "%lu\n", min_sample_time);
695 }
696
697 static ssize_t store_min_sample_time(struct kobject *kobj,
698                         struct attribute *attr, const char *buf, size_t count)
699 {
700         int ret;
701         unsigned long val;
702
703         ret = strict_strtoul(buf, 0, &val);
704         if (ret < 0)
705                 return ret;
706         min_sample_time = val;
707         return count;
708 }
709
710 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
711                 show_min_sample_time, store_min_sample_time);
712
713 static ssize_t show_timer_rate(struct kobject *kobj,
714                         struct attribute *attr, char *buf)
715 {
716         return sprintf(buf, "%lu\n", timer_rate);
717 }
718
719 static ssize_t store_timer_rate(struct kobject *kobj,
720                         struct attribute *attr, const char *buf, size_t count)
721 {
722         int ret;
723         unsigned long val;
724
725         ret = strict_strtoul(buf, 0, &val);
726         if (ret < 0)
727                 return ret;
728         timer_rate = val;
729         return count;
730 }
731
732 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
733                 show_timer_rate, store_timer_rate);
734
735 static struct attribute *interactive_attributes[] = {
736         &go_maxspeed_load_attr.attr,
737         &midrange_freq_attr.attr,
738         &midrange_go_maxspeed_load_attr.attr,
739         &boost_factor_attr.attr,
740         &max_boost_attr.attr,
741         &midrange_max_boost_attr.attr,
742         &io_is_busy_attr.attr,
743         &sustain_load_attr.attr,
744         &min_sample_time_attr.attr,
745         &timer_rate_attr.attr,
746         NULL,
747 };
748
749 static struct attribute_group interactive_attr_group = {
750         .attrs = interactive_attributes,
751         .name = "interactive",
752 };
753
754 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
755                 unsigned int event)
756 {
757         int rc;
758         unsigned int j;
759         struct cpufreq_interactive_cpuinfo *pcpu;
760         struct cpufreq_frequency_table *freq_table;
761
762         switch (event) {
763         case CPUFREQ_GOV_START:
764                 if (!cpu_online(policy->cpu))
765                         return -EINVAL;
766
767                 freq_table =
768                         cpufreq_frequency_get_table(policy->cpu);
769
770                 for_each_cpu(j, policy->cpus) {
771                         pcpu = &per_cpu(cpuinfo, j);
772                         pcpu->policy = policy;
773                         pcpu->target_freq = policy->cur;
774                         pcpu->freq_table = freq_table;
775                         pcpu->freq_change_time_in_idle =
776                                 get_cpu_idle_time_us(j,
777                                              &pcpu->freq_change_time);
778                         pcpu->time_in_idle = pcpu->freq_change_time_in_idle;
779                         pcpu->idle_exit_time = pcpu->freq_change_time;
780                         pcpu->freq_change_time_in_iowait =
781                                 get_cpu_iowait_time(j, NULL);
782                         pcpu->time_in_iowait = pcpu->freq_change_time_in_iowait;
783
784                         pcpu->timer_idlecancel = 1;
785                         pcpu->governor_enabled = 1;
786                         smp_wmb();
787
788                         if (!timer_pending(&pcpu->cpu_timer))
789                                 mod_timer(&pcpu->cpu_timer, jiffies + 2);
790                 }
791
792                 /*
793                  * Do not register the idle hook and create sysfs
794                  * entries if we have already done so.
795                  */
796                 if (atomic_inc_return(&active_count) > 1)
797                         return 0;
798
799                 rc = sysfs_create_group(cpufreq_global_kobject,
800                                 &interactive_attr_group);
801                 if (rc)
802                         return rc;
803
804                 break;
805
806         case CPUFREQ_GOV_STOP:
807                 for_each_cpu(j, policy->cpus) {
808                         pcpu = &per_cpu(cpuinfo, j);
809                         pcpu->governor_enabled = 0;
810                         smp_wmb();
811                         del_timer_sync(&pcpu->cpu_timer);
812
813                         /*
814                          * Reset idle exit time since we may cancel the timer
815                          * before it can run after the last idle exit time,
816                          * to avoid tripping the check in idle exit for a timer
817                          * that is trying to run.
818                          */
819                         pcpu->idle_exit_time = 0;
820                 }
821
822                 flush_work(&freq_scale_down_work);
823                 if (atomic_dec_return(&active_count) > 0)
824                         return 0;
825
826                 sysfs_remove_group(cpufreq_global_kobject,
827                                 &interactive_attr_group);
828
829                 break;
830
831         case CPUFREQ_GOV_LIMITS:
832                 if (policy->max < policy->cur)
833                         __cpufreq_driver_target(policy,
834                                         policy->max, CPUFREQ_RELATION_H);
835                 else if (policy->min > policy->cur)
836                         __cpufreq_driver_target(policy,
837                                         policy->min, CPUFREQ_RELATION_L);
838                 break;
839         }
840         return 0;
841 }
842
843 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
844                                              unsigned long val,
845                                              void *data)
846 {
847         switch (val) {
848         case IDLE_START:
849                 cpufreq_interactive_idle_start();
850                 break;
851         case IDLE_END:
852                 cpufreq_interactive_idle_end();
853                 break;
854         }
855
856         return 0;
857 }
858
859 static struct notifier_block cpufreq_interactive_idle_nb = {
860         .notifier_call = cpufreq_interactive_idle_notifier,
861 };
862
863 static int __init cpufreq_interactive_init(void)
864 {
865         unsigned int i;
866         struct cpufreq_interactive_cpuinfo *pcpu;
867         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
868
869         go_maxspeed_load = DEFAULT_GO_MAXSPEED_LOAD;
870         midrange_go_maxspeed_load = DEFAULT_MID_RANGE_GO_MAXSPEED_LOAD;
871         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
872         timer_rate = DEFAULT_TIMER_RATE;
873
874         /* Initalize per-cpu timers */
875         for_each_possible_cpu(i) {
876                 pcpu = &per_cpu(cpuinfo, i);
877                 init_timer(&pcpu->cpu_timer);
878                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
879                 pcpu->cpu_timer.data = i;
880         }
881
882         up_task = kthread_create(cpufreq_interactive_up_task, NULL,
883                                  "kinteractiveup");
884         if (IS_ERR(up_task))
885                 return PTR_ERR(up_task);
886
887         sched_setscheduler_nocheck(up_task, SCHED_FIFO, &param);
888         get_task_struct(up_task);
889
890         /* No rescuer thread, bind to CPU queuing the work for possibly
891            warm cache (probably doesn't matter much). */
892         down_wq = alloc_workqueue("knteractive_down", 0, 1);
893
894         if (!down_wq)
895                 goto err_freeuptask;
896
897         INIT_WORK(&freq_scale_down_work,
898                   cpufreq_interactive_freq_down);
899
900         spin_lock_init(&up_cpumask_lock);
901         spin_lock_init(&down_cpumask_lock);
902         mutex_init(&set_speed_lock);
903
904         idle_notifier_register(&cpufreq_interactive_idle_nb);
905
906         return cpufreq_register_governor(&cpufreq_gov_interactive);
907
908 err_freeuptask:
909         put_task_struct(up_task);
910         return -ENOMEM;
911 }
912
913 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
914 fs_initcall(cpufreq_interactive_init);
915 #else
916 module_init(cpufreq_interactive_init);
917 #endif
918
919 static void __exit cpufreq_interactive_exit(void)
920 {
921         cpufreq_unregister_governor(&cpufreq_gov_interactive);
922         kthread_stop(up_task);
923         put_task_struct(up_task);
924         destroy_workqueue(down_wq);
925 }
926
927 module_exit(cpufreq_interactive_exit);
928
929 MODULE_AUTHOR("Mike Chan <mike@android.com>");
930 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
931         "Latency sensitive workloads");
932 MODULE_LICENSE("GPL");