830a66442af56b0067a738744b30b3dd5d78371b
[linux-2.6.git] / arch / arm / mach-tegra / cpu-tegra3.c
1 /*
2  * arch/arm/mach-tegra/cpu-tegra3.c
3  *
4  * CPU auto-hotplug for Tegra3 CPUs
5  *
6  * Copyright (c) 2011-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/module.h>
25 #include <linux/types.h>
26 #include <linux/sched.h>
27 #include <linux/cpufreq.h>
28 #include <linux/delay.h>
29 #include <linux/err.h>
30 #include <linux/io.h>
31 #include <linux/cpu.h>
32 #include <linux/clk.h>
33 #include <linux/debugfs.h>
34 #include <linux/seq_file.h>
35 #include <linux/pm_qos.h>
36
37 #include "pm.h"
38 #include "cpu-tegra.h"
39 #include "clock.h"
40
41 #define INITIAL_STATE           TEGRA_HP_DISABLED
42 #define UP2G0_DELAY_MS          70
43 #define UP2Gn_DELAY_MS          100
44 #define DOWN_DELAY_MS           500
45
46 static struct mutex *tegra3_cpu_lock;
47
48 static struct workqueue_struct *hotplug_wq;
49 static struct delayed_work hotplug_work;
50
51 static bool no_lp;
52 module_param(no_lp, bool, 0644);
53
54 static unsigned long up2gn_delay;
55 static unsigned long up2g0_delay;
56 static unsigned long down_delay;
57 module_param(up2gn_delay, ulong, 0644);
58 module_param(up2g0_delay, ulong, 0644);
59 module_param(down_delay, ulong, 0644);
60
61 static unsigned int idle_top_freq;
62 static unsigned int idle_bottom_freq;
63 module_param(idle_top_freq, uint, 0644);
64 module_param(idle_bottom_freq, uint, 0644);
65
66 static int mp_overhead = 10;
67 module_param(mp_overhead, int, 0644);
68
69 static int balance_level = 60;
70 module_param(balance_level, int, 0644);
71
72 static struct clk *cpu_clk;
73 static struct clk *cpu_g_clk;
74 static struct clk *cpu_lp_clk;
75
76 static unsigned long last_change_time;
77
78 static struct {
79         cputime64_t time_up_total;
80         u64 last_update;
81         unsigned int up_down_count;
82 } hp_stats[CONFIG_NR_CPUS + 1]; /* Append LP CPU entry at the end */
83
84 static void hp_init_stats(void)
85 {
86         int i;
87         u64 cur_jiffies = get_jiffies_64();
88
89         for (i = 0; i <= CONFIG_NR_CPUS; i++) {
90                 hp_stats[i].time_up_total = 0;
91                 hp_stats[i].last_update = cur_jiffies;
92
93                 hp_stats[i].up_down_count = 0;
94                 if (is_lp_cluster()) {
95                         if (i == CONFIG_NR_CPUS)
96                                 hp_stats[i].up_down_count = 1;
97                 } else {
98                         if ((i < nr_cpu_ids) && cpu_online(i))
99                                 hp_stats[i].up_down_count = 1;
100                 }
101         }
102
103 }
104
105 static void hp_stats_update(unsigned int cpu, bool up)
106 {
107         u64 cur_jiffies = get_jiffies_64();
108         bool was_up = hp_stats[cpu].up_down_count & 0x1;
109
110         if (was_up)
111                 hp_stats[cpu].time_up_total =
112                         hp_stats[cpu].time_up_total +
113                         (cur_jiffies - hp_stats[cpu].last_update);
114
115         if (was_up != up) {
116                 hp_stats[cpu].up_down_count++;
117                 if ((hp_stats[cpu].up_down_count & 0x1) != up) {
118                         /* FIXME: sysfs user space CPU control breaks stats */
119                         pr_err("tegra hotplug stats out of sync with %s CPU%d",
120                                (cpu < CONFIG_NR_CPUS) ? "G" : "LP",
121                                (cpu < CONFIG_NR_CPUS) ?  cpu : 0);
122                         hp_stats[cpu].up_down_count ^=  0x1;
123                 }
124         }
125         hp_stats[cpu].last_update = cur_jiffies;
126 }
127
128
129 enum {
130         TEGRA_HP_DISABLED = 0,
131         TEGRA_HP_IDLE,
132         TEGRA_HP_DOWN,
133         TEGRA_HP_UP,
134 };
135 static int hp_state;
136
137 static int hp_state_set(const char *arg, const struct kernel_param *kp)
138 {
139         int ret = 0;
140         int old_state;
141
142         if (!tegra3_cpu_lock)
143                 return ret;
144
145         mutex_lock(tegra3_cpu_lock);
146
147         old_state = hp_state;
148         ret = param_set_bool(arg, kp);  /* set idle or disabled only */
149
150         if (ret == 0) {
151                 if ((hp_state == TEGRA_HP_DISABLED) &&
152                     (old_state != TEGRA_HP_DISABLED)) {
153                         mutex_unlock(tegra3_cpu_lock);
154                         cancel_delayed_work_sync(&hotplug_work);
155                         mutex_lock(tegra3_cpu_lock);
156                         pr_info("Tegra auto-hotplug disabled\n");
157                 } else if (hp_state != TEGRA_HP_DISABLED) {
158                         if (old_state == TEGRA_HP_DISABLED) {
159                                 pr_info("Tegra auto-hotplug enabled\n");
160                                 hp_init_stats();
161                         }
162                         /* catch-up with governor target speed */
163                         tegra_cpu_set_speed_cap(NULL);
164                 }
165         } else
166                 pr_warn("%s: unable to set tegra hotplug state %s\n",
167                                 __func__, arg);
168
169         mutex_unlock(tegra3_cpu_lock);
170         return ret;
171 }
172
173 static int hp_state_get(char *buffer, const struct kernel_param *kp)
174 {
175         return param_get_int(buffer, kp);
176 }
177
178 static struct kernel_param_ops tegra_hp_state_ops = {
179         .set = hp_state_set,
180         .get = hp_state_get,
181 };
182 module_param_cb(auto_hotplug, &tegra_hp_state_ops, &hp_state, 0644);
183
184
185 enum {
186         TEGRA_CPU_SPEED_BALANCED,
187         TEGRA_CPU_SPEED_BIASED,
188         TEGRA_CPU_SPEED_SKEWED,
189 };
190
191 #define NR_FSHIFT       2
192 static unsigned int nr_run_thresholds[] = {
193 /*      1,  2,  3,  4 - on-line cpus target */
194         5,  9, 10, UINT_MAX /* avg run threads * 4 (e.g., 9 = 2.25 threads) */
195 };
196 static unsigned int nr_run_hysteresis = 2;      /* 0.5 thread */
197 static unsigned int nr_run_last;
198
199 static noinline int tegra_cpu_speed_balance(void)
200 {
201         unsigned long highest_speed = tegra_cpu_highest_speed();
202         unsigned long balanced_speed = highest_speed * balance_level / 100;
203         unsigned long skewed_speed = balanced_speed / 2;
204         unsigned int nr_cpus = num_online_cpus();
205         unsigned int max_cpus = pm_qos_request(PM_QOS_MAX_ONLINE_CPUS) ? : 4;
206         unsigned int min_cpus = pm_qos_request(PM_QOS_MIN_ONLINE_CPUS);
207         unsigned int avg_nr_run = avg_nr_running();
208         unsigned int nr_run;
209
210         /* Evaluate:
211          * - distribution of freq targets for already on-lined CPUs
212          * - average number of runnable threads
213          * - effective MIPS available within EDP frequency limits,
214          * and return:
215          * TEGRA_CPU_SPEED_BALANCED to bring one more CPU core on-line
216          * TEGRA_CPU_SPEED_BIASED to keep CPU core composition unchanged
217          * TEGRA_CPU_SPEED_SKEWED to remove CPU core off-line
218          */
219         for (nr_run = 1; nr_run < ARRAY_SIZE(nr_run_thresholds); nr_run++) {
220                 unsigned int nr_threshold = nr_run_thresholds[nr_run - 1];
221                 if (nr_run_last <= nr_run)
222                         nr_threshold += nr_run_hysteresis;
223                 if (avg_nr_run <= (nr_threshold << (FSHIFT - NR_FSHIFT)))
224                         break;
225         }
226         nr_run_last = nr_run;
227
228         if (((tegra_count_slow_cpus(skewed_speed) >= 2) ||
229              (nr_run < nr_cpus) ||
230              tegra_cpu_edp_favor_down(nr_cpus, mp_overhead) ||
231              (highest_speed <= idle_bottom_freq) || (nr_cpus > max_cpus)) &&
232             (nr_cpus > min_cpus))
233                 return TEGRA_CPU_SPEED_SKEWED;
234
235         if (((tegra_count_slow_cpus(balanced_speed) >= 1) ||
236              (nr_run <= nr_cpus) ||
237              (!tegra_cpu_edp_favor_up(nr_cpus, mp_overhead)) ||
238              (highest_speed <= idle_bottom_freq) || (nr_cpus == max_cpus)) &&
239             (nr_cpus >= min_cpus))
240                 return TEGRA_CPU_SPEED_BIASED;
241
242         return TEGRA_CPU_SPEED_BALANCED;
243 }
244
245 static void __cpuinit tegra_auto_hotplug_work_func(struct work_struct *work)
246 {
247         bool up = false;
248         unsigned int cpu = nr_cpu_ids;
249         unsigned long now = jiffies;
250
251         mutex_lock(tegra3_cpu_lock);
252
253         switch (hp_state) {
254         case TEGRA_HP_DISABLED:
255         case TEGRA_HP_IDLE:
256                 break;
257         case TEGRA_HP_DOWN:
258                 cpu = tegra_get_slowest_cpu_n();
259                 if (cpu < nr_cpu_ids) {
260                         up = false;
261                 } else if (!is_lp_cluster() && !no_lp &&
262                            !pm_qos_request(PM_QOS_MIN_ONLINE_CPUS) &&
263                            ((now - last_change_time) >= down_delay)) {
264                         if(!clk_set_parent(cpu_clk, cpu_lp_clk)) {
265                                 hp_stats_update(CONFIG_NR_CPUS, true);
266                                 hp_stats_update(0, false);
267                                 /* catch-up with governor target speed */
268                                 tegra_cpu_set_speed_cap(NULL);
269                                 break;
270                         }
271                 }
272                 queue_delayed_work(
273                         hotplug_wq, &hotplug_work, up2gn_delay);
274                 break;
275         case TEGRA_HP_UP:
276                 if (is_lp_cluster() && !no_lp) {
277                         if(!clk_set_parent(cpu_clk, cpu_g_clk)) {
278                                 last_change_time = now;
279                                 hp_stats_update(CONFIG_NR_CPUS, false);
280                                 hp_stats_update(0, true);
281                                 /* catch-up with governor target speed */
282                                 tegra_cpu_set_speed_cap(NULL);
283                         }
284                 } else {
285                         switch (tegra_cpu_speed_balance()) {
286                         /* cpu speed is up and balanced - one more on-line */
287                         case TEGRA_CPU_SPEED_BALANCED:
288                                 cpu = cpumask_next_zero(0, cpu_online_mask);
289                                 if (cpu < nr_cpu_ids)
290                                         up = true;
291                                 break;
292                         /* cpu speed is up, but skewed - remove one core */
293                         case TEGRA_CPU_SPEED_SKEWED:
294                                 cpu = tegra_get_slowest_cpu_n();
295                                 if (cpu < nr_cpu_ids)
296                                         up = false;
297                                 break;
298                         /* cpu speed is up, but under-utilized - do nothing */
299                         case TEGRA_CPU_SPEED_BIASED:
300                         default:
301                                 break;
302                         }
303                 }
304                 queue_delayed_work(
305                         hotplug_wq, &hotplug_work, up2gn_delay);
306                 break;
307         default:
308                 pr_err("%s: invalid tegra hotplug state %d\n",
309                        __func__, hp_state);
310         }
311
312         if (!up && ((now - last_change_time) < down_delay))
313                         cpu = nr_cpu_ids;
314
315         if (cpu < nr_cpu_ids) {
316                 last_change_time = now;
317                 hp_stats_update(cpu, up);
318         }
319         mutex_unlock(tegra3_cpu_lock);
320
321         if (cpu < nr_cpu_ids) {
322                 if (up)
323                         cpu_up(cpu);
324                 else
325                         cpu_down(cpu);
326         }
327 }
328
329 static int min_cpus_notify(struct notifier_block *nb, unsigned long n, void *p)
330 {
331         mutex_lock(tegra3_cpu_lock);
332
333         if ((n >= 1) && is_lp_cluster()) {
334                 /* make sure cpu rate is within g-mode range before switching */
335                 unsigned int speed = max((unsigned long)tegra_getspeed(0),
336                         clk_get_min_rate(cpu_g_clk) / 1000);
337                 tegra_update_cpu_speed(speed);
338
339                 if (!clk_set_parent(cpu_clk, cpu_g_clk)) {
340                         last_change_time = jiffies;
341                         hp_stats_update(CONFIG_NR_CPUS, false);
342                         hp_stats_update(0, true);
343                 }
344         }
345         /* update governor state machine */
346         tegra_cpu_set_speed_cap(NULL);
347         mutex_unlock(tegra3_cpu_lock);
348         return NOTIFY_OK;
349 }
350
351 static struct notifier_block min_cpus_notifier = {
352         .notifier_call = min_cpus_notify,
353 };
354
355 void tegra_auto_hotplug_governor(unsigned int cpu_freq, bool suspend)
356 {
357         unsigned long up_delay, top_freq, bottom_freq;
358
359         if (!is_g_cluster_present())
360                 return;
361
362         if (hp_state == TEGRA_HP_DISABLED)
363                 return;
364
365         if (suspend) {
366                 hp_state = TEGRA_HP_IDLE;
367
368                 /* Switch to G-mode if suspend rate is high enough */
369                 if (is_lp_cluster() && (cpu_freq >= idle_bottom_freq)) {
370                         if (!clk_set_parent(cpu_clk, cpu_g_clk)) {
371                                 hp_stats_update(CONFIG_NR_CPUS, false);
372                                 hp_stats_update(0, true);
373                         }
374                 }
375                 return;
376         }
377
378         if (is_lp_cluster()) {
379                 up_delay = up2g0_delay;
380                 top_freq = idle_top_freq;
381                 bottom_freq = 0;
382         } else {
383                 up_delay = up2gn_delay;
384                 top_freq = idle_bottom_freq;
385                 bottom_freq = idle_bottom_freq;
386         }
387
388         if (pm_qos_request(PM_QOS_MIN_ONLINE_CPUS) >= 2) {
389                 if (hp_state != TEGRA_HP_UP) {
390                         hp_state = TEGRA_HP_UP;
391                         queue_delayed_work(
392                                 hotplug_wq, &hotplug_work, up_delay);
393                 }
394                 return;
395         }
396
397         switch (hp_state) {
398         case TEGRA_HP_IDLE:
399                 if (cpu_freq > top_freq) {
400                         hp_state = TEGRA_HP_UP;
401                         queue_delayed_work(
402                                 hotplug_wq, &hotplug_work, up_delay);
403                 } else if (cpu_freq <= bottom_freq) {
404                         hp_state = TEGRA_HP_DOWN;
405                         queue_delayed_work(
406                                 hotplug_wq, &hotplug_work, up_delay);
407                 }
408                 break;
409         case TEGRA_HP_DOWN:
410                 if (cpu_freq > top_freq) {
411                         hp_state = TEGRA_HP_UP;
412                         queue_delayed_work(
413                                 hotplug_wq, &hotplug_work, up_delay);
414                 } else if (cpu_freq > bottom_freq) {
415                         hp_state = TEGRA_HP_IDLE;
416                 }
417                 break;
418         case TEGRA_HP_UP:
419                 if (cpu_freq <= bottom_freq) {
420                         hp_state = TEGRA_HP_DOWN;
421                         queue_delayed_work(
422                                 hotplug_wq, &hotplug_work, up_delay);
423                 } else if (cpu_freq <= top_freq) {
424                         hp_state = TEGRA_HP_IDLE;
425                 }
426                 break;
427         default:
428                 pr_err("%s: invalid tegra hotplug state %d\n",
429                        __func__, hp_state);
430                 BUG();
431         }
432 }
433
434 int __cpuinit tegra_auto_hotplug_init(struct mutex *cpu_lock)
435 {
436         /*
437          * Not bound to the issuer CPU (=> high-priority), has rescue worker
438          * task, single-threaded, freezable.
439          */
440         hotplug_wq = alloc_workqueue(
441                 "cpu-tegra3", WQ_UNBOUND | WQ_RESCUER | WQ_FREEZABLE, 1);
442         if (!hotplug_wq)
443                 return -ENOMEM;
444         INIT_DELAYED_WORK(&hotplug_work, tegra_auto_hotplug_work_func);
445
446         cpu_clk = clk_get_sys(NULL, "cpu");
447         cpu_g_clk = clk_get_sys(NULL, "cpu_g");
448         cpu_lp_clk = clk_get_sys(NULL, "cpu_lp");
449         if (IS_ERR(cpu_clk) || IS_ERR(cpu_g_clk) || IS_ERR(cpu_lp_clk))
450                 return -ENOENT;
451
452         idle_top_freq = clk_get_max_rate(cpu_lp_clk) / 1000;
453         idle_bottom_freq = clk_get_min_rate(cpu_g_clk) / 1000;
454
455         up2g0_delay = msecs_to_jiffies(UP2G0_DELAY_MS);
456         up2gn_delay = msecs_to_jiffies(UP2Gn_DELAY_MS);
457         down_delay = msecs_to_jiffies(DOWN_DELAY_MS);
458
459         tegra3_cpu_lock = cpu_lock;
460         hp_state = INITIAL_STATE;
461         hp_init_stats();
462         pr_info("Tegra auto-hotplug initialized: %s\n",
463                 (hp_state == TEGRA_HP_DISABLED) ? "disabled" : "enabled");
464
465         if (pm_qos_add_notifier(PM_QOS_MIN_ONLINE_CPUS, &min_cpus_notifier))
466                 pr_err("%s: Failed to register min cpus PM QoS notifier\n",
467                         __func__);
468
469         return 0;
470 }
471
472 #ifdef CONFIG_DEBUG_FS
473
474 static struct dentry *hp_debugfs_root;
475
476 struct pm_qos_request min_cpu_req;
477 struct pm_qos_request max_cpu_req;
478
479 static int hp_stats_show(struct seq_file *s, void *data)
480 {
481         int i;
482         u64 cur_jiffies = get_jiffies_64();
483
484         mutex_lock(tegra3_cpu_lock);
485         if (hp_state != TEGRA_HP_DISABLED) {
486                 for (i = 0; i <= CONFIG_NR_CPUS; i++) {
487                         bool was_up = (hp_stats[i].up_down_count & 0x1);
488                         hp_stats_update(i, was_up);
489                 }
490         }
491         mutex_unlock(tegra3_cpu_lock);
492
493         seq_printf(s, "%-15s ", "cpu:");
494         for (i = 0; i < CONFIG_NR_CPUS; i++) {
495                 seq_printf(s, "G%-9d ", i);
496         }
497         seq_printf(s, "LP\n");
498
499         seq_printf(s, "%-15s ", "transitions:");
500         for (i = 0; i <= CONFIG_NR_CPUS; i++) {
501                 seq_printf(s, "%-10u ", hp_stats[i].up_down_count);
502         }
503         seq_printf(s, "\n");
504
505         seq_printf(s, "%-15s ", "time plugged:");
506         for (i = 0; i <= CONFIG_NR_CPUS; i++) {
507                 seq_printf(s, "%-10llu ",
508                            cputime64_to_clock_t(hp_stats[i].time_up_total));
509         }
510         seq_printf(s, "\n");
511
512         seq_printf(s, "%-15s %llu\n", "time-stamp:",
513                    cputime64_to_clock_t(cur_jiffies));
514
515         return 0;
516 }
517
518 static int hp_stats_open(struct inode *inode, struct file *file)
519 {
520         return single_open(file, hp_stats_show, inode->i_private);
521 }
522
523 static const struct file_operations hp_stats_fops = {
524         .open           = hp_stats_open,
525         .read           = seq_read,
526         .llseek         = seq_lseek,
527         .release        = single_release,
528 };
529
530 static int min_cpus_get(void *data, u64 *val)
531 {
532         *val = pm_qos_request(PM_QOS_MIN_ONLINE_CPUS);
533         return 0;
534 }
535 static int min_cpus_set(void *data, u64 val)
536 {
537         pm_qos_update_request(&min_cpu_req, (s32)val);
538         return 0;
539 }
540 DEFINE_SIMPLE_ATTRIBUTE(min_cpus_fops, min_cpus_get, min_cpus_set, "%llu\n");
541
542 static int max_cpus_get(void *data, u64 *val)
543 {
544         *val = pm_qos_request(PM_QOS_MAX_ONLINE_CPUS);
545         return 0;
546 }
547 static int max_cpus_set(void *data, u64 val)
548 {
549         pm_qos_update_request(&max_cpu_req, (s32)val);
550         return 0;
551 }
552 DEFINE_SIMPLE_ATTRIBUTE(max_cpus_fops, max_cpus_get, max_cpus_set, "%llu\n");
553
554 static int __init tegra_auto_hotplug_debug_init(void)
555 {
556         if (!tegra3_cpu_lock)
557                 return -ENOENT;
558
559         hp_debugfs_root = debugfs_create_dir("tegra_hotplug", NULL);
560         if (!hp_debugfs_root)
561                 return -ENOMEM;
562
563         pm_qos_add_request(&min_cpu_req, PM_QOS_MIN_ONLINE_CPUS,
564                            PM_QOS_DEFAULT_VALUE);
565         pm_qos_add_request(&max_cpu_req, PM_QOS_MAX_ONLINE_CPUS,
566                            PM_QOS_DEFAULT_VALUE);
567
568         if (!debugfs_create_file(
569                 "min_cpus", S_IRUGO, hp_debugfs_root, NULL, &min_cpus_fops))
570                 goto err_out;
571
572         if (!debugfs_create_file(
573                 "max_cpus", S_IRUGO, hp_debugfs_root, NULL, &max_cpus_fops))
574                 goto err_out;
575
576         if (!debugfs_create_file(
577                 "stats", S_IRUGO, hp_debugfs_root, NULL, &hp_stats_fops))
578                 goto err_out;
579
580         return 0;
581
582 err_out:
583         debugfs_remove_recursive(hp_debugfs_root);
584         pm_qos_remove_request(&min_cpu_req);
585         pm_qos_remove_request(&max_cpu_req);
586         return -ENOMEM;
587 }
588
589 late_initcall(tegra_auto_hotplug_debug_init);
590 #endif
591
592 void tegra_auto_hotplug_exit(void)
593 {
594         destroy_workqueue(hotplug_wq);
595 #ifdef CONFIG_DEBUG_FS
596         debugfs_remove_recursive(hp_debugfs_root);
597         pm_qos_remove_request(&min_cpu_req);
598         pm_qos_remove_request(&max_cpu_req);
599 #endif
600 }