ARM: tegra12: set CPU rate to 2.2GHz for sku 0x87
[linux-3.10.git] / arch / arm / mach-tegra / cpuidle-t3.c
1 /*
2  * arch/arm/mach-tegra/cpuidle-t3.c
3  *
4  * CPU idle driver for Tegra3 CPUs
5  *
6  * Copyright (c) 2010-2013, NVIDIA Corporation.   All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/cpu.h>
25 #include <linux/cpuidle.h>
26 #include <linux/debugfs.h>
27 #include <linux/delay.h>
28 #include <linux/hrtimer.h>
29 #include <linux/init.h>
30 #include <linux/interrupt.h>
31 #include <linux/irq.h>
32 #include <linux/io.h>
33 #include <linux/ratelimit.h>
34 #include <linux/sched.h>
35 #include <linux/seq_file.h>
36 #include <linux/slab.h>
37 #include <linux/smp.h>
38 #include <linux/suspend.h>
39 #include <linux/tick.h>
40 #include <linux/clk.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/module.h>
43 #include <linux/tegra-soc.h>
44 #include <linux/irqchip/tegra.h>
45
46 #include <asm/cacheflush.h>
47 #include <asm/localtimer.h>
48 #include <asm/suspend.h>
49 #include <asm/smp_twd.h>
50 #include <asm/cputype.h>
51
52 #include <mach/irqs.h>
53
54 #include <trace/events/power.h>
55
56 #include "clock.h"
57 #include "cpuidle.h"
58 #include "dvfs.h"
59 #include "iomap.h"
60 #include "pm.h"
61 #include "reset.h"
62 #include "sleep.h"
63 #include "timer.h"
64
65 #define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS \
66         (IO_ADDRESS(TEGRA_CLK_RESET_BASE) + 0x470)
67 #define PMC_POWERGATE_STATUS \
68         (IO_ADDRESS(TEGRA_PMC_BASE) + 0x038)
69
70 #ifdef CONFIG_SMP
71 static s64 tegra_cpu_wake_by_time[4] = {
72         LLONG_MAX, LLONG_MAX, LLONG_MAX, LLONG_MAX };
73 #endif
74
75 static bool lp2_0_in_idle = true;
76 module_param(lp2_0_in_idle, bool, 0644);
77
78 static bool lp2_n_in_idle = true;
79 module_param(lp2_n_in_idle, bool, 0644);
80
81 static struct clk *cpu_clk_for_dvfs;
82 static struct clk *twd_clk;
83
84 static int lp2_exit_latencies[5];
85
86 static struct {
87         unsigned int cpu_ready_count[5];
88         unsigned int tear_down_count[5];
89         unsigned long long cpu_wants_lp2_time[5];
90         unsigned long long in_lp2_time[5];
91         unsigned int lp2_count;
92         unsigned int lp2_completed_count;
93         unsigned int lp2_count_bin[32];
94         unsigned int lp2_completed_count_bin[32];
95         unsigned int lp2_int_count[NR_IRQS];
96         unsigned int last_lp2_int_count[NR_IRQS];
97 } idle_stats;
98
99 static inline unsigned int time_to_bin(unsigned int time)
100 {
101         return fls(time);
102 }
103
104 static inline void tegra_irq_unmask(int irq)
105 {
106         struct irq_data *data = irq_get_irq_data(irq);
107         data->chip->irq_unmask(data);
108 }
109
110 static inline unsigned int cpu_number(unsigned int n)
111 {
112         return is_lp_cluster() ? 4 : n;
113 }
114
115 void tegra3_cpu_idle_stats_lp2_ready(unsigned int cpu)
116 {
117         idle_stats.cpu_ready_count[cpu_number(cpu)]++;
118 }
119
120 void tegra3_cpu_idle_stats_lp2_time(unsigned int cpu, s64 us)
121 {
122         idle_stats.cpu_wants_lp2_time[cpu_number(cpu)] += us;
123 }
124
125 /* Allow rail off only if all secondary CPUs are power gated, and no
126    rail update is in progress */
127 static bool tegra_rail_off_is_allowed(void)
128 {
129         u32 rst = readl(CLK_RST_CONTROLLER_CPU_CMPLX_STATUS);
130         u32 pg = readl(PMC_POWERGATE_STATUS) >> 8;
131
132         if (((rst & 0xE) != 0xE) || ((pg & 0xE) != 0))
133                 return false;
134
135         if (tegra_dvfs_rail_updating(cpu_clk_for_dvfs))
136                 return false;
137
138         return true;
139 }
140
141 bool tegra3_lp2_is_allowed(struct cpuidle_device *dev,
142         struct cpuidle_state *state)
143 {
144         s64 request;
145
146         if (!tegra_all_cpus_booted)
147                 return false;
148
149         if ((!lp2_0_in_idle && !dev->cpu) || (!lp2_n_in_idle && dev->cpu))
150                 return false;
151
152 #ifndef CONFIG_TEGRA_RAIL_OFF_MULTIPLE_CPUS
153         /* FIXME: All CPU's entering LP2 is not working.
154          * Don't let CPU0 enter LP2 when any secondary CPU is online.
155          */
156         if ((dev->cpu == 0) && (num_online_cpus() > 1))
157                 return false;
158 #endif
159         if ((dev->cpu == 0)  && (!tegra_rail_off_is_allowed()))
160                 return false;
161
162         request = ktime_to_us(tick_nohz_get_sleep_length());
163         if (state->exit_latency != lp2_exit_latencies[cpu_number(dev->cpu)]) {
164                 /* possible on the 1st entry after cluster switch*/
165                 state->exit_latency = lp2_exit_latencies[cpu_number(dev->cpu)];
166                 tegra_pd_update_target_residency(state);
167         }
168         if (request < state->target_residency) {
169                 /* Not enough time left to enter LP2 */
170                 return false;
171         }
172
173         return true;
174 }
175
176 static inline void tegra3_lp2_restore_affinity(void)
177 {
178 #ifdef CONFIG_SMP
179         /* Disable the distributor. */
180         tegra_gic_dist_disable();
181
182         /* Restore the other CPU's interrupt affinity. */
183         tegra_gic_restore_affinity();
184
185         /* Re-enable the distributor. */
186         tegra_gic_dist_enable();
187 #endif
188 }
189
190 static bool tegra_cpu_cluster_power_down(struct cpuidle_device *dev,
191                            struct cpuidle_state *state, s64 request)
192 {
193         ktime_t entry_time;
194         ktime_t exit_time;
195         bool sleep_completed = false;
196         bool multi_cpu_entry = false;
197         int bin;
198         s64 sleep_time;
199
200         /* LP2 entry time */
201         entry_time = ktime_get();
202
203         if (request < state->target_residency) {
204                 /* Not enough time left to enter LP2 */
205                 cpu_do_idle();
206                 return false;
207         }
208
209 #ifdef CONFIG_SMP
210         multi_cpu_entry = !is_lp_cluster() && (num_online_cpus() > 1);
211         if (multi_cpu_entry) {
212                 s64 wake_time;
213                 unsigned int i;
214
215                 /* Disable the distributor -- this is the only way to
216                    prevent the other CPUs from responding to interrupts
217                    and potentially fiddling with the distributor
218                    registers while we're fiddling with them. */
219                 tegra_gic_dist_disable();
220
221                 /* Did an interrupt come in for another CPU before we
222                    could disable the distributor? */
223                 if (!tegra_rail_off_is_allowed()) {
224                         /* Yes, re-enable the distributor and clock gating. */
225                         tegra_gic_dist_enable();
226                         cpu_do_idle();
227                         return false;
228                 }
229
230                 /* LP2 initial targeted wake time */
231                 wake_time = ktime_to_us(entry_time) + request;
232
233                 /* CPU0 must wake up before any of the other CPUs. */
234                 smp_rmb();
235                 for (i = 1; i < CONFIG_NR_CPUS; i++)
236                         wake_time = min_t(s64, wake_time,
237                                 tegra_cpu_wake_by_time[i]);
238
239                 /* LP2 actual targeted wake time */
240                 request = wake_time - ktime_to_us(entry_time);
241                 BUG_ON(wake_time < 0LL);
242
243                 if (request < state->target_residency) {
244                         /* Not enough time left to enter LP2 */
245                         tegra_gic_dist_enable();
246                         cpu_do_idle();
247                         return false;
248                 }
249
250                 /* Cancel LP2 wake timers for all secondary CPUs */
251                 tegra_pd_timer_cancel_secondary();
252
253                 /* Save and disable the affinity setting for the other
254                    CPUs and route all interrupts to CPU0. */
255                 tegra_gic_disable_affinity();
256
257                 /* Re-enable the distributor. */
258                 tegra_gic_dist_enable();
259         }
260 #endif
261         cpu_pm_enter();
262
263         sleep_time = request -
264                 lp2_exit_latencies[cpu_number(dev->cpu)];
265
266         bin = time_to_bin((u32)request / 1000);
267         idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
268         idle_stats.lp2_count++;
269         idle_stats.lp2_count_bin[bin]++;
270
271         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
272         if (!is_lp_cluster())
273                 tegra_dvfs_rail_off(tegra_cpu_rail, entry_time);
274
275         if (tegra_idle_power_down_last(sleep_time, 0) == 0)
276                 sleep_completed = true;
277         else {
278                 int irq = tegra_gic_pending_interrupt();
279                 idle_stats.lp2_int_count[irq]++;
280         }
281
282         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
283         exit_time = ktime_get();
284         if (!is_lp_cluster())
285                 tegra_dvfs_rail_on(tegra_cpu_rail, exit_time);
286
287         idle_stats.in_lp2_time[cpu_number(dev->cpu)] +=
288                 ktime_to_us(ktime_sub(exit_time, entry_time));
289
290         if (multi_cpu_entry)
291                 tegra3_lp2_restore_affinity();
292
293         if (sleep_completed) {
294                 /*
295                  * Stayed in LP2 for the full time until the next tick,
296                  * adjust the exit latency based on measurement
297                  */
298                 int offset = ktime_to_us(ktime_sub(exit_time, entry_time))
299                         - request;
300                 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
301                         offset / 16;
302                 latency = clamp(latency, 0, 10000);
303                 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
304                 state->exit_latency = latency;          /* for idle governor */
305                 smp_wmb();
306
307                 idle_stats.lp2_completed_count++;
308                 idle_stats.lp2_completed_count_bin[bin]++;
309
310                 pr_debug("%lld %lld %d %d\n", request,
311                         ktime_to_us(ktime_sub(exit_time, entry_time)),
312                         offset, bin);
313         }
314
315         cpu_pm_exit();
316
317         return true;
318 }
319
320 static bool tegra_cpu_core_power_down(struct cpuidle_device *dev,
321                            struct cpuidle_state *state, s64 request)
322 {
323 #ifdef CONFIG_SMP
324         s64 sleep_time;
325         ktime_t entry_time;
326         struct tegra_twd_context twd_context;
327         bool sleep_completed = false;
328         struct tick_sched *ts = tick_get_tick_sched(dev->cpu);
329 #if defined(CONFIG_TEGRA_LP2_CPU_TIMER)
330         void __iomem *twd_base = IO_ADDRESS(TEGRA_ARM_PERIF_BASE + 0x600);
331 #endif
332
333         if (!tegra_twd_get_state(&twd_context)) {
334                 unsigned long twd_rate = clk_get_rate(twd_clk);
335
336                 if ((twd_context.twd_ctrl & TWD_TIMER_CONTROL_ENABLE) &&
337                     (twd_context.twd_ctrl & TWD_TIMER_CONTROL_IT_ENABLE)) {
338                         request = div_u64((u64)twd_context.twd_cnt * 1000000,
339                                           twd_rate);
340 #ifdef CONFIG_TEGRA_LP2_CPU_TIMER
341                         if (request >= state->target_residency) {
342                                 twd_context.twd_cnt -= state->exit_latency *
343                                         (twd_rate / 1000000);
344                                 writel(twd_context.twd_cnt,
345                                         twd_base + TWD_TIMER_COUNTER);
346                         }
347 #endif
348                 }
349         }
350
351         if (!tegra_is_cpu_wake_timer_ready(dev->cpu) ||
352             (request < state->target_residency) ||
353             (!ts) || (ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
354                 /*
355                  * Not enough time left to enter LP2, or wake timer not ready
356                  */
357                 cpu_do_idle();
358                 return false;
359         }
360
361         cpu_pm_enter();
362
363 #if !defined(CONFIG_TEGRA_LP2_CPU_TIMER)
364         sleep_time = request - state->exit_latency;
365         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
366         tegra_twd_suspend(&twd_context);
367         tegra_pd_set_trigger(sleep_time);
368 #endif
369         idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
370
371         entry_time = ktime_get();
372
373         /* Save time this CPU must be awakened by. */
374         tegra_cpu_wake_by_time[dev->cpu] = ktime_to_us(entry_time) + request;
375         smp_wmb();
376
377         cpu_suspend(0, tegra3_sleep_cpu_secondary_finish);
378
379         tegra_cpu_wake_by_time[dev->cpu] = LLONG_MAX;
380
381 #ifdef CONFIG_TEGRA_LP2_CPU_TIMER
382         if (!tegra_twd_get_state(&twd_context))
383                 sleep_completed = (twd_context.twd_cnt == 0);
384 #else
385         sleep_completed = !tegra_pd_timer_remain();
386         tegra_pd_set_trigger(0);
387         tegra_twd_resume(&twd_context);
388         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
389 #endif
390         sleep_time = ktime_to_us(ktime_sub(ktime_get(), entry_time));
391         idle_stats.in_lp2_time[cpu_number(dev->cpu)] += sleep_time;
392         if (sleep_completed) {
393                 /*
394                  * Stayed in LP2 for the full time until timer expires,
395                  * adjust the exit latency based on measurement
396                  */
397                 int offset = sleep_time - request;
398                 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
399                         offset / 16;
400                 latency = clamp(latency, 0, 10000);
401                 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
402                 state->exit_latency = latency;          /* for idle governor */
403                 smp_wmb();
404         }
405 #endif
406         cpu_pm_exit();
407
408         return true;
409 }
410
411 bool tegra3_idle_lp2(struct cpuidle_device *dev,
412                            struct cpuidle_state *state)
413 {
414         s64 request = ktime_to_us(tick_nohz_get_sleep_length());
415         bool last_cpu = tegra_set_cpu_in_pd(dev->cpu);
416         bool entered_lp2;
417
418         if ((dev->cpu == 0) && last_cpu)
419                 entered_lp2 = tegra_cpu_cluster_power_down(dev, state, request);
420         else if (dev->cpu)
421                 entered_lp2 = tegra_cpu_core_power_down(dev, state, request);
422         else {
423                 cpu_do_idle();
424                 entered_lp2 = false;
425         }
426
427         tegra_clear_cpu_in_pd(dev->cpu);
428
429         return entered_lp2;
430 }
431
432 #ifdef CONFIG_DEBUG_FS
433 int tegra3_lp2_debug_show(struct seq_file *s, void *data)
434 {
435         int bin;
436         int i;
437         seq_printf(s, "                                    cpu0     cpu1     cpu2     cpu3     cpulp\n");
438         seq_printf(s, "-----------------------------------------------------------------------------\n");
439         seq_printf(s, "cpu ready:                      %8u %8u %8u %8u %8u\n",
440                 idle_stats.cpu_ready_count[0],
441                 idle_stats.cpu_ready_count[1],
442                 idle_stats.cpu_ready_count[2],
443                 idle_stats.cpu_ready_count[3],
444                 idle_stats.cpu_ready_count[4]);
445         seq_printf(s, "tear down:                      %8u %8u %8u %8u %8u\n",
446                 idle_stats.tear_down_count[0],
447                 idle_stats.tear_down_count[1],
448                 idle_stats.tear_down_count[2],
449                 idle_stats.tear_down_count[3],
450                 idle_stats.tear_down_count[4]);
451         seq_printf(s, "lp2:            %8u\n", idle_stats.lp2_count);
452         seq_printf(s, "lp2 completed:  %8u %7u%%\n",
453                 idle_stats.lp2_completed_count,
454                 idle_stats.lp2_completed_count * 100 /
455                         (idle_stats.lp2_count ?: 1));
456
457         seq_printf(s, "\n");
458         seq_printf(s, "cpu ready time:                 %8llu %8llu %8llu %8llu %8llu ms\n",
459                 div64_u64(idle_stats.cpu_wants_lp2_time[0], 1000),
460                 div64_u64(idle_stats.cpu_wants_lp2_time[1], 1000),
461                 div64_u64(idle_stats.cpu_wants_lp2_time[2], 1000),
462                 div64_u64(idle_stats.cpu_wants_lp2_time[3], 1000),
463                 div64_u64(idle_stats.cpu_wants_lp2_time[4], 1000));
464
465         seq_printf(s, "lp2 time:                       %8llu %8llu %8llu %8llu %8llu ms\n",
466                 div64_u64(idle_stats.in_lp2_time[0], 1000),
467                 div64_u64(idle_stats.in_lp2_time[1], 1000),
468                 div64_u64(idle_stats.in_lp2_time[2], 1000),
469                 div64_u64(idle_stats.in_lp2_time[3], 1000),
470                 div64_u64(idle_stats.in_lp2_time[4], 1000));
471
472         seq_printf(s, "lp2 %%:                         %7d%% %7d%% %7d%% %7d%% %7d%%\n",
473                 (int)(idle_stats.cpu_wants_lp2_time[0] ?
474                         div64_u64(idle_stats.in_lp2_time[0] * 100,
475                         idle_stats.cpu_wants_lp2_time[0]) : 0),
476                 (int)(idle_stats.cpu_wants_lp2_time[1] ?
477                         div64_u64(idle_stats.in_lp2_time[1] * 100,
478                         idle_stats.cpu_wants_lp2_time[1]) : 0),
479                 (int)(idle_stats.cpu_wants_lp2_time[2] ?
480                         div64_u64(idle_stats.in_lp2_time[2] * 100,
481                         idle_stats.cpu_wants_lp2_time[2]) : 0),
482                 (int)(idle_stats.cpu_wants_lp2_time[3] ?
483                         div64_u64(idle_stats.in_lp2_time[3] * 100,
484                         idle_stats.cpu_wants_lp2_time[3]) : 0),
485                 (int)(idle_stats.cpu_wants_lp2_time[4] ?
486                         div64_u64(idle_stats.in_lp2_time[4] * 100,
487                         idle_stats.cpu_wants_lp2_time[4]) : 0));
488         seq_printf(s, "\n");
489
490         seq_printf(s, "%19s %8s %8s %8s\n", "", "lp2", "comp", "%");
491         seq_printf(s, "-------------------------------------------------\n");
492         for (bin = 0; bin < 32; bin++) {
493                 if (idle_stats.lp2_count_bin[bin] == 0)
494                         continue;
495                 seq_printf(s, "%6u - %6u ms: %8u %8u %7u%%\n",
496                         1 << (bin - 1), 1 << bin,
497                         idle_stats.lp2_count_bin[bin],
498                         idle_stats.lp2_completed_count_bin[bin],
499                         idle_stats.lp2_completed_count_bin[bin] * 100 /
500                                 idle_stats.lp2_count_bin[bin]);
501         }
502
503         seq_printf(s, "\n");
504         seq_printf(s, "%3s %20s %6s %10s\n",
505                 "int", "name", "count", "last count");
506         seq_printf(s, "--------------------------------------------\n");
507         for (i = 0; i < NR_IRQS; i++) {
508                 if (idle_stats.lp2_int_count[i] == 0)
509                         continue;
510                 seq_printf(s, "%3d %20s %6d %10d\n",
511                         i, irq_to_desc(i)->action ?
512                                 irq_to_desc(i)->action->name ?: "???" : "???",
513                         idle_stats.lp2_int_count[i],
514                         idle_stats.lp2_int_count[i] -
515                                 idle_stats.last_lp2_int_count[i]);
516                 idle_stats.last_lp2_int_count[i] = idle_stats.lp2_int_count[i];
517         };
518         return 0;
519 }
520 #endif
521
522 int __init tegra3_cpuidle_init_soc(struct tegra_cpuidle_ops *idle_ops)
523 {
524         int i;
525         struct tegra_cpuidle_ops ops = {
526                 tegra3_idle_lp2,
527                 tegra3_cpu_idle_stats_lp2_ready,
528                 tegra3_cpu_idle_stats_lp2_time,
529                 tegra3_lp2_is_allowed,
530 #ifdef CONFIG_DEBUG_FS
531                 tegra3_lp2_debug_show
532 #endif
533         };
534
535         cpu_clk_for_dvfs = tegra_get_clock_by_name("cpu_g");
536         twd_clk = tegra_get_clock_by_name("twd");
537
538         for (i = 0; i < ARRAY_SIZE(lp2_exit_latencies); i++)
539                 lp2_exit_latencies[i] = tegra_pg_exit_latency;
540
541         *idle_ops = ops;
542
543         return 0;
544 }