unknown changes from android-tegra-nv-3.4
[linux-3.10.git] / arch / arm / mach-tegra / cpuidle-t3.c
1 /*
2  * arch/arm/mach-tegra/cpuidle-t3.c
3  *
4  * CPU idle driver for Tegra3 CPUs
5  *
6  * Copyright (c) 2010-2011, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/cpu.h>
25 #include <linux/cpuidle.h>
26 #include <linux/debugfs.h>
27 #include <linux/delay.h>
28 #include <linux/hrtimer.h>
29 #include <linux/init.h>
30 #include <linux/interrupt.h>
31 #include <linux/irq.h>
32 #include <linux/io.h>
33 #include <linux/ratelimit.h>
34 #include <linux/sched.h>
35 #include <linux/seq_file.h>
36 #include <linux/slab.h>
37 #include <linux/smp.h>
38 #include <linux/suspend.h>
39 #include <linux/tick.h>
40 #include <linux/clk.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/module.h>
43
44 #include <asm/cacheflush.h>
45 #include <asm/hardware/gic.h>
46 #include <asm/localtimer.h>
47 #include <asm/suspend.h>
48 #include <asm/smp_twd.h>
49
50 #include <mach/iomap.h>
51 #include <mach/irqs.h>
52 #include <mach/hardware.h>
53
54 #include <trace/events/power.h>
55
56 #include "clock.h"
57 #include "cpuidle.h"
58 #include "dvfs.h"
59 #include "fuse.h"
60 #include "gic.h"
61 #include "pm.h"
62 #include "reset.h"
63 #include "sleep.h"
64 #include "timer.h"
65 #include "fuse.h"
66
67 #define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS \
68         (IO_ADDRESS(TEGRA_CLK_RESET_BASE) + 0x470)
69 #define PMC_POWERGATE_STATUS \
70         (IO_ADDRESS(TEGRA_PMC_BASE) + 0x038)
71
72 #ifdef CONFIG_SMP
73 static s64 tegra_cpu_wake_by_time[4] = {
74         LLONG_MAX, LLONG_MAX, LLONG_MAX, LLONG_MAX };
75 #endif
76
77 static bool lp2_0_in_idle = true;
78 module_param(lp2_0_in_idle, bool, 0644);
79
80 static bool lp2_n_in_idle = true;
81 module_param(lp2_n_in_idle, bool, 0644);
82
83 static struct clk *cpu_clk_for_dvfs;
84 static struct clk *twd_clk;
85
86 static int lp2_exit_latencies[5];
87
88 static struct {
89         unsigned int cpu_ready_count[5];
90         unsigned int tear_down_count[5];
91         unsigned long long cpu_wants_lp2_time[5];
92         unsigned long long in_lp2_time[5];
93         unsigned int lp2_count;
94         unsigned int lp2_completed_count;
95         unsigned int lp2_count_bin[32];
96         unsigned int lp2_completed_count_bin[32];
97         unsigned int lp2_int_count[NR_IRQS];
98         unsigned int last_lp2_int_count[NR_IRQS];
99 } idle_stats;
100
101 static inline unsigned int time_to_bin(unsigned int time)
102 {
103         return fls(time);
104 }
105
106 static inline void tegra_irq_unmask(int irq)
107 {
108         struct irq_data *data = irq_get_irq_data(irq);
109         data->chip->irq_unmask(data);
110 }
111
112 static inline unsigned int cpu_number(unsigned int n)
113 {
114         return is_lp_cluster() ? 4 : n;
115 }
116
117 void tegra3_cpu_idle_stats_lp2_ready(unsigned int cpu)
118 {
119         idle_stats.cpu_ready_count[cpu_number(cpu)]++;
120 }
121
122 void tegra3_cpu_idle_stats_lp2_time(unsigned int cpu, s64 us)
123 {
124         idle_stats.cpu_wants_lp2_time[cpu_number(cpu)] += us;
125 }
126
127 /* Allow rail off only if all secondary CPUs are power gated, and no
128    rail update is in progress */
129 static bool tegra3_rail_off_is_allowed(void)
130 {
131         u32 rst = readl(CLK_RST_CONTROLLER_CPU_CMPLX_STATUS);
132         u32 pg = readl(PMC_POWERGATE_STATUS) >> 8;
133
134         if (((rst & 0xE) != 0xE) || ((pg & 0xE) != 0))
135                 return false;
136
137         if (tegra_dvfs_rail_updating(cpu_clk_for_dvfs))
138                 return false;
139
140         return true;
141 }
142
143 bool tegra3_lp2_is_allowed(struct cpuidle_device *dev,
144         struct cpuidle_state *state)
145 {
146         s64 request;
147
148         if (!tegra_all_cpus_booted)
149                 return false;
150
151         if ((!lp2_0_in_idle && !dev->cpu) || (!lp2_n_in_idle && dev->cpu))
152                 return false;
153
154         /* On A01, LP2 on slave CPU's cause ranhdom CPU hangs.
155          * Refer to Bug 804085.
156          */
157         if ((tegra_revision == TEGRA_REVISION_A01) &&
158                 num_online_cpus() > 1)
159                 return false;
160
161 #ifndef CONFIG_TEGRA_RAIL_OFF_MULTIPLE_CPUS
162         /* FIXME: All CPU's entering LP2 is not working.
163          * Don't let CPU0 enter LP2 when any secondary CPU is online.
164          */
165         if ((dev->cpu == 0) && (num_online_cpus() > 1))
166                 return false;
167 #endif
168         if ((dev->cpu == 0)  && (!tegra3_rail_off_is_allowed()))
169                 return false;
170
171         request = ktime_to_us(tick_nohz_get_sleep_length());
172         if (state->exit_latency != lp2_exit_latencies[cpu_number(dev->cpu)]) {
173                 /* possible on the 1st entry after cluster switch*/
174                 state->exit_latency = lp2_exit_latencies[cpu_number(dev->cpu)];
175                 tegra_lp2_update_target_residency(state);
176         }
177         if (request < state->target_residency) {
178                 /* Not enough time left to enter LP2 */
179                 return false;
180         }
181
182         return true;
183 }
184
185 static inline void tegra3_lp2_restore_affinity(void)
186 {
187 #ifdef CONFIG_SMP
188         /* Disable the distributor. */
189         tegra_gic_dist_disable();
190
191         /* Restore the other CPU's interrupt affinity. */
192         tegra_gic_restore_affinity();
193
194         /* Re-enable the distributor. */
195         tegra_gic_dist_enable();
196 #endif
197 }
198
199 static bool tegra3_idle_enter_lp2_cpu_0(struct cpuidle_device *dev,
200                            struct cpuidle_state *state, s64 request)
201 {
202         ktime_t entry_time;
203         ktime_t exit_time;
204         bool sleep_completed = false;
205         bool multi_cpu_entry = false;
206         int bin;
207         s64 sleep_time;
208
209         /* LP2 entry time */
210         entry_time = ktime_get();
211
212         if (request < state->target_residency) {
213                 /* Not enough time left to enter LP2 */
214                 tegra_cpu_wfi();
215                 return false;
216         }
217
218 #ifdef CONFIG_SMP
219         multi_cpu_entry = !is_lp_cluster() && (num_online_cpus() > 1);
220         if (multi_cpu_entry) {
221                 s64 wake_time;
222                 unsigned int i;
223
224                 /* Disable the distributor -- this is the only way to
225                    prevent the other CPUs from responding to interrupts
226                    and potentially fiddling with the distributor
227                    registers while we're fiddling with them. */
228                 tegra_gic_dist_disable();
229
230                 /* Did an interrupt come in for another CPU before we
231                    could disable the distributor? */
232                 if (!tegra3_rail_off_is_allowed()) {
233                         /* Yes, re-enable the distributor and LP3. */
234                         tegra_gic_dist_enable();
235                         tegra_cpu_wfi();
236                         return false;
237                 }
238
239                 /* LP2 initial targeted wake time */
240                 wake_time = ktime_to_us(entry_time) + request;
241
242                 /* CPU0 must wake up before any of the other CPUs. */
243                 smp_rmb();
244                 for (i = 1; i < CONFIG_NR_CPUS; i++)
245                         wake_time = min_t(s64, wake_time,
246                                 tegra_cpu_wake_by_time[i]);
247
248                 /* LP2 actual targeted wake time */
249                 request = wake_time - ktime_to_us(entry_time);
250                 BUG_ON(wake_time < 0LL);
251
252                 if (request < state->target_residency) {
253                         /* Not enough time left to enter LP2 */
254                         tegra_gic_dist_enable();
255                         tegra_cpu_wfi();
256                         return false;
257                 }
258
259                 /* Cancel LP2 wake timers for all secondary CPUs */
260                 tegra_lp2_timer_cancel_secondary();
261
262                 /* Save and disable the affinity setting for the other
263                    CPUs and route all interrupts to CPU0. */
264                 tegra_gic_disable_affinity();
265
266                 /* Re-enable the distributor. */
267                 tegra_gic_dist_enable();
268         }
269 #endif
270
271         sleep_time = request -
272                 lp2_exit_latencies[cpu_number(dev->cpu)];
273
274         bin = time_to_bin((u32)request / 1000);
275         idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
276         idle_stats.lp2_count++;
277         idle_stats.lp2_count_bin[bin]++;
278
279         trace_power_start(POWER_CSTATE, 2, dev->cpu);
280         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
281         if (!is_lp_cluster())
282                 tegra_dvfs_rail_off(tegra_cpu_rail, entry_time);
283
284         if (tegra_idle_lp2_last(sleep_time, 0) == 0)
285                 sleep_completed = true;
286         else {
287                 int irq = tegra_gic_pending_interrupt();
288                 idle_stats.lp2_int_count[irq]++;
289         }
290
291         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
292         exit_time = ktime_get();
293         if (!is_lp_cluster())
294                 tegra_dvfs_rail_on(tegra_cpu_rail, exit_time);
295         idle_stats.in_lp2_time[cpu_number(dev->cpu)] +=
296                 ktime_to_us(ktime_sub(exit_time, entry_time));
297
298         if (multi_cpu_entry)
299                 tegra3_lp2_restore_affinity();
300
301         if (sleep_completed) {
302                 /*
303                  * Stayed in LP2 for the full time until the next tick,
304                  * adjust the exit latency based on measurement
305                  */
306                 int offset = ktime_to_us(ktime_sub(exit_time, entry_time))
307                         - request;
308                 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
309                         offset / 16;
310                 latency = clamp(latency, 0, 10000);
311                 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
312                 state->exit_latency = latency;          /* for idle governor */
313                 smp_wmb();
314
315                 idle_stats.lp2_completed_count++;
316                 idle_stats.lp2_completed_count_bin[bin]++;
317
318                 pr_debug("%lld %lld %d %d\n", request,
319                         ktime_to_us(ktime_sub(exit_time, entry_time)),
320                         offset, bin);
321         }
322
323         return true;
324 }
325
326 #ifndef CONFIG_TRUSTED_FOUNDATIONS
327 static unsigned int g_diag_reg;
328
329 static void save_cpu_arch_register(void)
330 {
331         /* read diagnostic register */
332         asm("mrc p15, 0, %0, c15, c0, 1" : "=r"(g_diag_reg) : : "cc");
333 }
334
335 static void restore_cpu_arch_register(void)
336 {
337         /* write diagnostic register */
338         asm("mcr p15, 0, %0, c15, c0, 1" : : "r"(g_diag_reg) : "cc");
339 }
340 #endif
341
342 static bool tegra3_idle_enter_lp2_cpu_n(struct cpuidle_device *dev,
343                            struct cpuidle_state *state, s64 request)
344 {
345 #ifdef CONFIG_SMP
346         s64 sleep_time;
347         ktime_t entry_time;
348         struct tegra_twd_context twd_context;
349         bool sleep_completed = false;
350         struct tick_sched *ts = tick_get_tick_sched(dev->cpu);
351         void __iomem *twd_base = IO_ADDRESS(TEGRA_ARM_PERIF_BASE + 0x600);
352
353         if (!tegra_twd_get_state(&twd_context)) {
354                 unsigned long twd_rate = clk_get_rate(twd_clk);
355
356                 if ((twd_context.twd_ctrl & TWD_TIMER_CONTROL_ENABLE) &&
357                     (twd_context.twd_ctrl & TWD_TIMER_CONTROL_IT_ENABLE)) {
358                         request = div_u64((u64)twd_context.twd_cnt * 1000000,
359                                           twd_rate);
360 #ifdef CONFIG_TEGRA_LP2_ARM_TWD
361                         if (request >= state->target_residency) {
362                                 twd_context.twd_cnt -= state->exit_latency *
363                                         (twd_rate / 1000000);
364                                 writel(twd_context.twd_cnt,
365                                         twd_base + TWD_TIMER_COUNTER);
366                         }
367 #endif
368                 }
369         }
370
371         if (!tegra_is_lp2_timer_ready(dev->cpu) ||
372             (request < state->target_residency) ||
373             (!ts) || (ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
374                 /*
375                  * Not enough time left to enter LP2, or wake timer not ready
376                  */
377                 tegra_cpu_wfi();
378                 return false;
379         }
380
381 #ifndef CONFIG_TEGRA_LP2_ARM_TWD
382         sleep_time = request - state->exit_latency;
383         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
384         tegra_twd_suspend(&twd_context);
385         tegra_lp2_set_trigger(sleep_time);
386 #endif
387         idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
388
389         trace_power_start(POWER_CSTATE, 2, dev->cpu);
390
391         entry_time = ktime_get();
392
393         /* Save time this CPU must be awakened by. */
394         tegra_cpu_wake_by_time[dev->cpu] = ktime_to_us(entry_time) + request;
395         smp_wmb();
396
397 #ifndef CONFIG_TRUSTED_FOUNDATIONS
398         save_cpu_arch_register();
399 #endif
400
401         cpu_suspend(0, tegra3_sleep_cpu_secondary_finish);
402
403 #ifndef CONFIG_TRUSTED_FOUNDATIONS
404         restore_cpu_arch_register();
405 #endif
406
407         tegra_cpu_wake_by_time[dev->cpu] = LLONG_MAX;
408
409 #ifdef CONFIG_TEGRA_LP2_ARM_TWD
410         if (!tegra_twd_get_state(&twd_context))
411                 sleep_completed = (twd_context.twd_cnt == 0);
412 #else
413         sleep_completed = !tegra_lp2_timer_remain();
414         tegra_lp2_set_trigger(0);
415         tegra_twd_resume(&twd_context);
416         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
417 #endif
418         sleep_time = ktime_to_us(ktime_sub(ktime_get(), entry_time));
419         idle_stats.in_lp2_time[cpu_number(dev->cpu)] += sleep_time;
420         if (sleep_completed) {
421                 /*
422                  * Stayed in LP2 for the full time until timer expires,
423                  * adjust the exit latency based on measurement
424                  */
425                 int offset = sleep_time - request;
426                 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
427                         offset / 16;
428                 latency = clamp(latency, 0, 10000);
429                 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
430                 state->exit_latency = latency;          /* for idle governor */
431                 smp_wmb();
432         }
433 #endif
434
435         return true;
436 }
437
438 bool tegra3_idle_lp2(struct cpuidle_device *dev,
439                            struct cpuidle_state *state)
440 {
441         s64 request = ktime_to_us(tick_nohz_get_sleep_length());
442         bool last_cpu = tegra_set_cpu_in_lp2(dev->cpu);
443         bool entered_lp2;
444
445         cpu_pm_enter();
446
447         if (dev->cpu == 0) {
448                 if (last_cpu) {
449                         entered_lp2 = tegra3_idle_enter_lp2_cpu_0(dev, state, request);
450                 } else {
451                         tegra_cpu_wfi();
452                         entered_lp2 = false;
453                 }
454         } else
455                 entered_lp2 = tegra3_idle_enter_lp2_cpu_n(dev, state, request);
456
457         cpu_pm_exit();
458         tegra_clear_cpu_in_lp2(dev->cpu);
459
460         return entered_lp2;
461 }
462
463 int tegra3_cpudile_init_soc(void)
464 {
465         int i;
466
467         cpu_clk_for_dvfs = tegra_get_clock_by_name("cpu_g");
468         twd_clk = tegra_get_clock_by_name("twd");
469
470         for (i = 0; i < ARRAY_SIZE(lp2_exit_latencies); i++)
471                 lp2_exit_latencies[i] = tegra_lp2_exit_latency;
472
473         return 0;
474 }
475
476 #ifdef CONFIG_DEBUG_FS
477 int tegra3_lp2_debug_show(struct seq_file *s, void *data)
478 {
479         int bin;
480         int i;
481         seq_printf(s, "                                    cpu0     cpu1     cpu2     cpu3     cpulp\n");
482         seq_printf(s, "-----------------------------------------------------------------------------\n");
483         seq_printf(s, "cpu ready:                      %8u %8u %8u %8u %8u\n",
484                 idle_stats.cpu_ready_count[0],
485                 idle_stats.cpu_ready_count[1],
486                 idle_stats.cpu_ready_count[2],
487                 idle_stats.cpu_ready_count[3],
488                 idle_stats.cpu_ready_count[4]);
489         seq_printf(s, "tear down:                      %8u %8u %8u %8u %8u\n",
490                 idle_stats.tear_down_count[0],
491                 idle_stats.tear_down_count[1],
492                 idle_stats.tear_down_count[2],
493                 idle_stats.tear_down_count[3],
494                 idle_stats.tear_down_count[4]);
495         seq_printf(s, "lp2:            %8u\n", idle_stats.lp2_count);
496         seq_printf(s, "lp2 completed:  %8u %7u%%\n",
497                 idle_stats.lp2_completed_count,
498                 idle_stats.lp2_completed_count * 100 /
499                         (idle_stats.lp2_count ?: 1));
500
501         seq_printf(s, "\n");
502         seq_printf(s, "cpu ready time:                 %8llu %8llu %8llu %8llu %8llu ms\n",
503                 div64_u64(idle_stats.cpu_wants_lp2_time[0], 1000),
504                 div64_u64(idle_stats.cpu_wants_lp2_time[1], 1000),
505                 div64_u64(idle_stats.cpu_wants_lp2_time[2], 1000),
506                 div64_u64(idle_stats.cpu_wants_lp2_time[3], 1000),
507                 div64_u64(idle_stats.cpu_wants_lp2_time[4], 1000));
508
509         seq_printf(s, "lp2 time:                       %8llu %8llu %8llu %8llu %8llu ms\n",
510                 div64_u64(idle_stats.in_lp2_time[0], 1000),
511                 div64_u64(idle_stats.in_lp2_time[1], 1000),
512                 div64_u64(idle_stats.in_lp2_time[2], 1000),
513                 div64_u64(idle_stats.in_lp2_time[3], 1000),
514                 div64_u64(idle_stats.in_lp2_time[4], 1000));
515
516         seq_printf(s, "lp2 %%:                         %7d%% %7d%% %7d%% %7d%% %7d%%\n",
517                 (int)(idle_stats.cpu_wants_lp2_time[0] ?
518                         div64_u64(idle_stats.in_lp2_time[0] * 100,
519                         idle_stats.cpu_wants_lp2_time[0]) : 0),
520                 (int)(idle_stats.cpu_wants_lp2_time[1] ?
521                         div64_u64(idle_stats.in_lp2_time[1] * 100,
522                         idle_stats.cpu_wants_lp2_time[1]) : 0),
523                 (int)(idle_stats.cpu_wants_lp2_time[2] ?
524                         div64_u64(idle_stats.in_lp2_time[2] * 100,
525                         idle_stats.cpu_wants_lp2_time[2]) : 0),
526                 (int)(idle_stats.cpu_wants_lp2_time[3] ?
527                         div64_u64(idle_stats.in_lp2_time[3] * 100,
528                         idle_stats.cpu_wants_lp2_time[3]) : 0),
529                 (int)(idle_stats.cpu_wants_lp2_time[4] ?
530                         div64_u64(idle_stats.in_lp2_time[4] * 100,
531                         idle_stats.cpu_wants_lp2_time[4]) : 0));
532         seq_printf(s, "\n");
533
534         seq_printf(s, "%19s %8s %8s %8s\n", "", "lp2", "comp", "%");
535         seq_printf(s, "-------------------------------------------------\n");
536         for (bin = 0; bin < 32; bin++) {
537                 if (idle_stats.lp2_count_bin[bin] == 0)
538                         continue;
539                 seq_printf(s, "%6u - %6u ms: %8u %8u %7u%%\n",
540                         1 << (bin - 1), 1 << bin,
541                         idle_stats.lp2_count_bin[bin],
542                         idle_stats.lp2_completed_count_bin[bin],
543                         idle_stats.lp2_completed_count_bin[bin] * 100 /
544                                 idle_stats.lp2_count_bin[bin]);
545         }
546
547         seq_printf(s, "\n");
548         seq_printf(s, "%3s %20s %6s %10s\n",
549                 "int", "name", "count", "last count");
550         seq_printf(s, "--------------------------------------------\n");
551         for (i = 0; i < NR_IRQS; i++) {
552                 if (idle_stats.lp2_int_count[i] == 0)
553                         continue;
554                 seq_printf(s, "%3d %20s %6d %10d\n",
555                         i, irq_to_desc(i)->action ?
556                                 irq_to_desc(i)->action->name ?: "???" : "???",
557                         idle_stats.lp2_int_count[i],
558                         idle_stats.lp2_int_count[i] -
559                                 idle_stats.last_lp2_int_count[i]);
560                 idle_stats.last_lp2_int_count[i] = idle_stats.lp2_int_count[i];
561         };
562         return 0;
563 }
564 #endif