2cfa129e9a51a80943c18b4c0083af36a9f6dcd8
[linux-3.10.git] / arch / arm / mach-tegra / cpuidle-t3.c
1 /*
2  * arch/arm/mach-tegra/cpuidle-t3.c
3  *
4  * CPU idle driver for Tegra3 CPUs
5  *
6  * Copyright (c) 2010-2011, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/cpu.h>
25 #include <linux/cpuidle.h>
26 #include <linux/debugfs.h>
27 #include <linux/delay.h>
28 #include <linux/hrtimer.h>
29 #include <linux/init.h>
30 #include <linux/interrupt.h>
31 #include <linux/irq.h>
32 #include <linux/io.h>
33 #include <linux/ratelimit.h>
34 #include <linux/sched.h>
35 #include <linux/seq_file.h>
36 #include <linux/slab.h>
37 #include <linux/smp.h>
38 #include <linux/suspend.h>
39 #include <linux/tick.h>
40 #include <linux/cpu_pm.h>
41 #include <linux/clk.h>
42 #include <linux/moduleparam.h>
43
44 #include <asm/cacheflush.h>
45 #include <asm/hardware/gic.h>
46 #include <asm/localtimer.h>
47 #include <asm/suspend.h>
48 #include <asm/smp_twd.h>
49
50 #include <mach/iomap.h>
51 #include <mach/irqs.h>
52
53 #include <trace/events/power.h>
54
55 #include "clock.h"
56 #include "cpuidle.h"
57 #include "dvfs.h"
58 #include "fuse.h"
59 #include "gic.h"
60 #include "pm.h"
61 #include "reset.h"
62 #include "sleep.h"
63 #include "timer.h"
64
65 #define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS \
66         (IO_ADDRESS(TEGRA_CLK_RESET_BASE) + 0x470)
67 #define PMC_POWERGATE_STATUS \
68         (IO_ADDRESS(TEGRA_PMC_BASE) + 0x038)
69
70 #ifdef CONFIG_SMP
71 static s64 tegra_cpu_wake_by_time[4] = {
72         LLONG_MAX, LLONG_MAX, LLONG_MAX, LLONG_MAX };
73 #endif
74
75 static bool lp2_0_in_idle = true;
76 module_param(lp2_0_in_idle, bool, 0644);
77
78 static bool lp2_n_in_idle = true;
79 module_param(lp2_n_in_idle, bool, 0644);
80
81 static struct clk *cpu_clk_for_dvfs;
82 static struct clk *twd_clk;
83
84 static int lp2_exit_latencies[5];
85
86 static struct {
87         unsigned int cpu_ready_count[5];
88         unsigned int tear_down_count[5];
89         unsigned long long cpu_wants_lp2_time[5];
90         unsigned long long in_lp2_time[5];
91         unsigned int lp2_count;
92         unsigned int lp2_completed_count;
93         unsigned int lp2_count_bin[32];
94         unsigned int lp2_completed_count_bin[32];
95         unsigned int lp2_int_count[NR_IRQS];
96         unsigned int last_lp2_int_count[NR_IRQS];
97 } idle_stats;
98
99 static inline unsigned int time_to_bin(unsigned int time)
100 {
101         return fls(time);
102 }
103
104 static inline void tegra_irq_unmask(int irq)
105 {
106         struct irq_data *data = irq_get_irq_data(irq);
107         data->chip->irq_unmask(data);
108 }
109
110 static inline unsigned int cpu_number(unsigned int n)
111 {
112         return is_lp_cluster() ? 4 : n;
113 }
114
115 void tegra3_cpu_idle_stats_lp2_ready(unsigned int cpu)
116 {
117         idle_stats.cpu_ready_count[cpu_number(cpu)]++;
118 }
119
120 void tegra3_cpu_idle_stats_lp2_time(unsigned int cpu, s64 us)
121 {
122         idle_stats.cpu_wants_lp2_time[cpu_number(cpu)] += us;
123 }
124
125 /* Allow rail off only if all secondary CPUs are power gated, and no
126    rail update is in progress */
127 static bool tegra3_rail_off_is_allowed(void)
128 {
129         u32 rst = readl(CLK_RST_CONTROLLER_CPU_CMPLX_STATUS);
130         u32 pg = readl(PMC_POWERGATE_STATUS) >> 8;
131
132         if (((rst & 0xE) != 0xE) || ((pg & 0xE) != 0))
133                 return false;
134
135         if (tegra_dvfs_rail_updating(cpu_clk_for_dvfs))
136                 return false;
137
138         return true;
139 }
140
141 bool tegra3_lp2_is_allowed(struct cpuidle_device *dev,
142         struct cpuidle_state *state)
143 {
144         s64 request;
145
146         if (!tegra_all_cpus_booted)
147                 return false;
148
149         if ((!lp2_0_in_idle && !dev->cpu) || (!lp2_n_in_idle && dev->cpu))
150                 return false;
151
152         /* On A01, LP2 on slave CPU's cause ranhdom CPU hangs.
153          * Refer to Bug 804085.
154          */
155         if ((tegra_revision == TEGRA_REVISION_A01) &&
156                 num_online_cpus() > 1)
157                 return false;
158
159 #ifndef CONFIG_TEGRA_RAIL_OFF_MULTIPLE_CPUS
160         /* FIXME: All CPU's entering LP2 is not working.
161          * Don't let CPU0 enter LP2 when any secondary CPU is online.
162          */
163         if ((dev->cpu == 0) && (num_online_cpus() > 1))
164                 return false;
165 #endif
166         if ((dev->cpu == 0)  && (!tegra3_rail_off_is_allowed()))
167                 return false;
168
169         request = ktime_to_us(tick_nohz_get_sleep_length());
170         if (state->exit_latency != lp2_exit_latencies[cpu_number(dev->cpu)]) {
171                 /* possible on the 1st entry after cluster switch*/
172                 state->exit_latency = lp2_exit_latencies[cpu_number(dev->cpu)];
173                 tegra_lp2_update_target_residency(state);
174         }
175         if (request < state->target_residency) {
176                 /* Not enough time left to enter LP2 */
177                 return false;
178         }
179
180         return true;
181 }
182
183 static inline void tegra3_lp2_restore_affinity(void)
184 {
185 #ifdef CONFIG_SMP
186         /* Disable the distributor. */
187         tegra_gic_dist_disable();
188
189         /* Restore the other CPU's interrupt affinity. */
190         tegra_gic_restore_affinity();
191
192         /* Re-enable the distributor. */
193         tegra_gic_dist_enable();
194 #endif
195 }
196
197 static bool tegra3_idle_enter_lp2_cpu_0(struct cpuidle_device *dev,
198                            struct cpuidle_state *state, s64 request)
199 {
200         ktime_t entry_time;
201         ktime_t exit_time;
202         bool sleep_completed = false;
203         bool multi_cpu_entry = false;
204         int bin;
205         s64 sleep_time;
206
207         /* LP2 entry time */
208         entry_time = ktime_get();
209
210         if (request < state->target_residency) {
211                 /* Not enough time left to enter LP2 */
212                 tegra_cpu_wfi();
213                 return false;
214         }
215
216 #ifdef CONFIG_SMP
217         multi_cpu_entry = !is_lp_cluster() && (num_online_cpus() > 1);
218         if (multi_cpu_entry) {
219                 s64 wake_time;
220                 unsigned int i;
221
222                 /* Disable the distributor -- this is the only way to
223                    prevent the other CPUs from responding to interrupts
224                    and potentially fiddling with the distributor
225                    registers while we're fiddling with them. */
226                 tegra_gic_dist_disable();
227
228                 /* Did an interrupt come in for another CPU before we
229                    could disable the distributor? */
230                 if (!tegra3_rail_off_is_allowed()) {
231                         /* Yes, re-enable the distributor and LP3. */
232                         tegra_gic_dist_enable();
233                         tegra_cpu_wfi();
234                         return false;
235                 }
236
237                 /* LP2 initial targeted wake time */
238                 wake_time = ktime_to_us(entry_time) + request;
239
240                 /* CPU0 must wake up before any of the other CPUs. */
241                 smp_rmb();
242                 for (i = 1; i < CONFIG_NR_CPUS; i++)
243                         wake_time = min_t(s64, wake_time,
244                                 tegra_cpu_wake_by_time[i]);
245
246                 /* LP2 actual targeted wake time */
247                 request = wake_time - ktime_to_us(entry_time);
248                 BUG_ON(wake_time < 0LL);
249
250                 if (request < state->target_residency) {
251                         /* Not enough time left to enter LP2 */
252                         tegra_gic_dist_enable();
253                         tegra_cpu_wfi();
254                         return false;
255                 }
256
257                 /* Cancel LP2 wake timers for all secondary CPUs */
258                 tegra_lp2_timer_cancel_secondary();
259
260                 /* Save and disable the affinity setting for the other
261                    CPUs and route all interrupts to CPU0. */
262                 tegra_gic_disable_affinity();
263
264                 /* Re-enable the distributor. */
265                 tegra_gic_dist_enable();
266         }
267 #endif
268
269         sleep_time = request -
270                 lp2_exit_latencies[cpu_number(dev->cpu)];
271
272         bin = time_to_bin((u32)request / 1000);
273         idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
274         idle_stats.lp2_count++;
275         idle_stats.lp2_count_bin[bin]++;
276
277         trace_power_start(POWER_CSTATE, 2, dev->cpu);
278         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
279         if (!is_lp_cluster())
280                 tegra_dvfs_rail_off(tegra_cpu_rail, entry_time);
281
282         if (tegra_idle_lp2_last(sleep_time, 0) == 0)
283                 sleep_completed = true;
284         else {
285                 int irq = tegra_gic_pending_interrupt();
286                 idle_stats.lp2_int_count[irq]++;
287         }
288
289         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
290         exit_time = ktime_get();
291         if (!is_lp_cluster())
292                 tegra_dvfs_rail_on(tegra_cpu_rail, exit_time);
293         idle_stats.in_lp2_time[cpu_number(dev->cpu)] +=
294                 ktime_to_us(ktime_sub(exit_time, entry_time));
295
296         if (multi_cpu_entry)
297                 tegra3_lp2_restore_affinity();
298
299         if (sleep_completed) {
300                 /*
301                  * Stayed in LP2 for the full time until the next tick,
302                  * adjust the exit latency based on measurement
303                  */
304                 int offset = ktime_to_us(ktime_sub(exit_time, entry_time))
305                         - request;
306                 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
307                         offset / 16;
308                 latency = clamp(latency, 0, 10000);
309                 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
310                 state->exit_latency = latency;          /* for idle governor */
311                 smp_wmb();
312
313                 idle_stats.lp2_completed_count++;
314                 idle_stats.lp2_completed_count_bin[bin]++;
315
316                 pr_debug("%lld %lld %d %d\n", request,
317                         ktime_to_us(ktime_sub(exit_time, entry_time)),
318                         offset, bin);
319         }
320
321         return true;
322 }
323
324 #ifndef CONFIG_TRUSTED_FOUNDATIONS
325 static unsigned int g_diag_reg;
326
327 static void save_cpu_arch_register(void)
328 {
329         /* read diagnostic register */
330         asm("mrc p15, 0, %0, c15, c0, 1" : "=r"(g_diag_reg) : : "cc");
331 }
332
333 static void restore_cpu_arch_register(void)
334 {
335         /* write diagnostic register */
336         asm("mcr p15, 0, %0, c15, c0, 1" : : "r"(g_diag_reg) : "cc");
337 }
338 #endif
339
340 static bool tegra3_idle_enter_lp2_cpu_n(struct cpuidle_device *dev,
341                            struct cpuidle_state *state, s64 request)
342 {
343 #ifdef CONFIG_SMP
344         s64 sleep_time;
345         ktime_t entry_time;
346         struct tegra_twd_context twd_context;
347         bool sleep_completed = false;
348         struct tick_sched *ts = tick_get_tick_sched(dev->cpu);
349
350         if (!tegra_twd_get_state(&twd_context)) {
351                 unsigned long twd_rate = clk_get_rate(twd_clk);
352
353                 if ((twd_context.twd_ctrl & TWD_TIMER_CONTROL_ENABLE) &&
354                     (twd_context.twd_ctrl & TWD_TIMER_CONTROL_IT_ENABLE)) {
355                         request = div_u64((u64)twd_context.twd_cnt * 1000000,
356                                           twd_rate);
357 #ifdef CONFIG_TEGRA_LP2_ARM_TWD
358                         if (request >= state->target_residency) {
359                                 twd_context.twd_cnt -= state->exit_latency *
360                                         (twd_rate / 1000000);
361                                 writel(twd_context.twd_cnt,
362                                         twd_base + TWD_TIMER_COUNTER);
363                         }
364 #endif
365                 }
366         }
367
368         if (!tegra_is_lp2_timer_ready(dev->cpu) ||
369             (request < state->target_residency) ||
370             (!ts) || (ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
371                 /*
372                  * Not enough time left to enter LP2, or wake timer not ready
373                  */
374                 tegra_cpu_wfi();
375                 return false;
376         }
377
378 #ifndef CONFIG_TEGRA_LP2_ARM_TWD
379         sleep_time = request - state->exit_latency;
380         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
381         tegra_twd_suspend(&twd_context);
382         tegra_lp2_set_trigger(sleep_time);
383 #endif
384         idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
385
386         trace_power_start(POWER_CSTATE, 2, dev->cpu);
387
388         entry_time = ktime_get();
389
390         /* Save time this CPU must be awakened by. */
391         tegra_cpu_wake_by_time[dev->cpu] = ktime_to_us(entry_time) + request;
392         smp_wmb();
393
394 #ifndef CONFIG_TRUSTED_FOUNDATIONS
395         save_cpu_arch_register();
396 #endif
397
398         cpu_suspend(0, tegra3_sleep_cpu_secondary_finish);
399
400 #ifndef CONFIG_TRUSTED_FOUNDATIONS
401         restore_cpu_arch_register();
402 #endif
403
404         tegra_cpu_wake_by_time[dev->cpu] = LLONG_MAX;
405
406 #ifdef CONFIG_TEGRA_LP2_ARM_TWD
407         if (!tegra_twd_get_state(&twd_context))
408                 sleep_completed = (twd_context.twd_cnt == 0);
409 #else
410         sleep_completed = !tegra_lp2_timer_remain();
411         tegra_lp2_set_trigger(0);
412         tegra_twd_resume(&twd_context);
413         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
414 #endif
415         sleep_time = ktime_to_us(ktime_sub(ktime_get(), entry_time));
416         idle_stats.in_lp2_time[cpu_number(dev->cpu)] += sleep_time;
417         if (sleep_completed) {
418                 /*
419                  * Stayed in LP2 for the full time until timer expires,
420                  * adjust the exit latency based on measurement
421                  */
422                 int offset = sleep_time - request;
423                 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
424                         offset / 16;
425                 latency = clamp(latency, 0, 10000);
426                 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
427                 state->exit_latency = latency;          /* for idle governor */
428                 smp_wmb();
429         }
430 #endif
431
432         return true;
433 }
434
435 bool tegra3_idle_lp2(struct cpuidle_device *dev,
436                            struct cpuidle_state *state)
437 {
438         s64 request = ktime_to_us(tick_nohz_get_sleep_length());
439         bool last_cpu = tegra_set_cpu_in_lp2(dev->cpu);
440         bool entered_lp2;
441
442         cpu_pm_enter();
443
444         if (dev->cpu == 0) {
445                 if (last_cpu) {
446                         entered_lp2 = tegra3_idle_enter_lp2_cpu_0(dev, state, request);
447                 } else {
448                         tegra_cpu_wfi();
449                         entered_lp2 = false;
450                 }
451         } else
452                 entered_lp2 = tegra3_idle_enter_lp2_cpu_n(dev, state, request);
453
454         cpu_pm_exit();
455         tegra_clear_cpu_in_lp2(dev->cpu);
456
457         return entered_lp2;
458 }
459
460 int tegra3_cpudile_init_soc(void)
461 {
462         int i;
463
464         cpu_clk_for_dvfs = tegra_get_clock_by_name("cpu_g");
465         twd_clk = tegra_get_clock_by_name("twd");
466
467         for (i = 0; i < ARRAY_SIZE(lp2_exit_latencies); i++)
468                 lp2_exit_latencies[i] = tegra_lp2_exit_latency;
469
470         return 0;
471 }
472
473 #ifdef CONFIG_DEBUG_FS
474 int tegra3_lp2_debug_show(struct seq_file *s, void *data)
475 {
476         int bin;
477         int i;
478         seq_printf(s, "                                    cpu0     cpu1     cpu2     cpu3     cpulp\n");
479         seq_printf(s, "-----------------------------------------------------------------------------\n");
480         seq_printf(s, "cpu ready:                      %8u %8u %8u %8u %8u\n",
481                 idle_stats.cpu_ready_count[0],
482                 idle_stats.cpu_ready_count[1],
483                 idle_stats.cpu_ready_count[2],
484                 idle_stats.cpu_ready_count[3],
485                 idle_stats.cpu_ready_count[4]);
486         seq_printf(s, "tear down:                      %8u %8u %8u %8u %8u\n",
487                 idle_stats.tear_down_count[0],
488                 idle_stats.tear_down_count[1],
489                 idle_stats.tear_down_count[2],
490                 idle_stats.tear_down_count[3],
491                 idle_stats.tear_down_count[4]);
492         seq_printf(s, "lp2:            %8u\n", idle_stats.lp2_count);
493         seq_printf(s, "lp2 completed:  %8u %7u%%\n",
494                 idle_stats.lp2_completed_count,
495                 idle_stats.lp2_completed_count * 100 /
496                         (idle_stats.lp2_count ?: 1));
497
498         seq_printf(s, "\n");
499         seq_printf(s, "cpu ready time:                 %8llu %8llu %8llu %8llu %8llu ms\n",
500                 div64_u64(idle_stats.cpu_wants_lp2_time[0], 1000),
501                 div64_u64(idle_stats.cpu_wants_lp2_time[1], 1000),
502                 div64_u64(idle_stats.cpu_wants_lp2_time[2], 1000),
503                 div64_u64(idle_stats.cpu_wants_lp2_time[3], 1000),
504                 div64_u64(idle_stats.cpu_wants_lp2_time[4], 1000));
505
506         seq_printf(s, "lp2 time:                       %8llu %8llu %8llu %8llu %8llu ms\n",
507                 div64_u64(idle_stats.in_lp2_time[0], 1000),
508                 div64_u64(idle_stats.in_lp2_time[1], 1000),
509                 div64_u64(idle_stats.in_lp2_time[2], 1000),
510                 div64_u64(idle_stats.in_lp2_time[3], 1000),
511                 div64_u64(idle_stats.in_lp2_time[4], 1000));
512
513         seq_printf(s, "lp2 %%:                         %7d%% %7d%% %7d%% %7d%% %7d%%\n",
514                 (int)(idle_stats.cpu_wants_lp2_time[0] ?
515                         div64_u64(idle_stats.in_lp2_time[0] * 100,
516                         idle_stats.cpu_wants_lp2_time[0]) : 0),
517                 (int)(idle_stats.cpu_wants_lp2_time[1] ?
518                         div64_u64(idle_stats.in_lp2_time[1] * 100,
519                         idle_stats.cpu_wants_lp2_time[1]) : 0),
520                 (int)(idle_stats.cpu_wants_lp2_time[2] ?
521                         div64_u64(idle_stats.in_lp2_time[2] * 100,
522                         idle_stats.cpu_wants_lp2_time[2]) : 0),
523                 (int)(idle_stats.cpu_wants_lp2_time[3] ?
524                         div64_u64(idle_stats.in_lp2_time[3] * 100,
525                         idle_stats.cpu_wants_lp2_time[3]) : 0),
526                 (int)(idle_stats.cpu_wants_lp2_time[4] ?
527                         div64_u64(idle_stats.in_lp2_time[4] * 100,
528                         idle_stats.cpu_wants_lp2_time[4]) : 0));
529         seq_printf(s, "\n");
530
531         seq_printf(s, "%19s %8s %8s %8s\n", "", "lp2", "comp", "%");
532         seq_printf(s, "-------------------------------------------------\n");
533         for (bin = 0; bin < 32; bin++) {
534                 if (idle_stats.lp2_count_bin[bin] == 0)
535                         continue;
536                 seq_printf(s, "%6u - %6u ms: %8u %8u %7u%%\n",
537                         1 << (bin - 1), 1 << bin,
538                         idle_stats.lp2_count_bin[bin],
539                         idle_stats.lp2_completed_count_bin[bin],
540                         idle_stats.lp2_completed_count_bin[bin] * 100 /
541                                 idle_stats.lp2_count_bin[bin]);
542         }
543
544         seq_printf(s, "\n");
545         seq_printf(s, "%3s %20s %6s %10s\n",
546                 "int", "name", "count", "last count");
547         seq_printf(s, "--------------------------------------------\n");
548         for (i = 0; i < NR_IRQS; i++) {
549                 if (idle_stats.lp2_int_count[i] == 0)
550                         continue;
551                 seq_printf(s, "%3d %20s %6d %10d\n",
552                         i, irq_to_desc(i)->action ?
553                                 irq_to_desc(i)->action->name ?: "???" : "???",
554                         idle_stats.lp2_int_count[i],
555                         idle_stats.lp2_int_count[i] -
556                                 idle_stats.last_lp2_int_count[i]);
557                 idle_stats.last_lp2_int_count[i] = idle_stats.lp2_int_count[i];
558         };
559         return 0;
560 }
561 #endif