2 * arch/arm/mach-tegra/cpuidle-t3.c
4 * CPU idle driver for Tegra3 CPUs
6 * Copyright (c) 2010-2011, NVIDIA Corporation.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
23 #include <linux/kernel.h>
24 #include <linux/cpu.h>
25 #include <linux/cpuidle.h>
26 #include <linux/debugfs.h>
27 #include <linux/delay.h>
28 #include <linux/hrtimer.h>
29 #include <linux/init.h>
30 #include <linux/interrupt.h>
31 #include <linux/irq.h>
33 #include <linux/ratelimit.h>
34 #include <linux/sched.h>
35 #include <linux/seq_file.h>
36 #include <linux/slab.h>
37 #include <linux/smp.h>
38 #include <linux/suspend.h>
39 #include <linux/tick.h>
40 #include <linux/clk.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/module.h>
44 #include <asm/cacheflush.h>
45 #include <asm/hardware/gic.h>
46 #include <asm/localtimer.h>
47 #include <asm/suspend.h>
48 #include <asm/smp_twd.h>
49 #include <asm/cputype.h>
51 #include <mach/iomap.h>
52 #include <mach/irqs.h>
53 #include <mach/hardware.h>
55 #include <trace/events/power.h>
68 #define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS \
69 (IO_ADDRESS(TEGRA_CLK_RESET_BASE) + 0x470)
70 #define PMC_POWERGATE_STATUS \
71 (IO_ADDRESS(TEGRA_PMC_BASE) + 0x038)
73 #define ARCH_TIMER_CTRL_ENABLE (1 << 0)
74 #define ARCH_TIMER_CTRL_IT_MASK (1 << 1)
77 static s64 tegra_cpu_wake_by_time[4] = {
78 LLONG_MAX, LLONG_MAX, LLONG_MAX, LLONG_MAX };
81 static bool lp2_0_in_idle = true;
82 module_param(lp2_0_in_idle, bool, 0644);
84 static bool lp2_n_in_idle = true;
85 module_param(lp2_n_in_idle, bool, 0644);
87 static struct clk *cpu_clk_for_dvfs;
88 static struct clk *twd_clk;
90 static int lp2_exit_latencies[5];
93 unsigned int cpu_ready_count[5];
94 unsigned int tear_down_count[5];
95 unsigned long long cpu_wants_lp2_time[5];
96 unsigned long long in_lp2_time[5];
97 unsigned int lp2_count;
98 unsigned int lp2_completed_count;
99 unsigned int lp2_count_bin[32];
100 unsigned int lp2_completed_count_bin[32];
101 unsigned int lp2_int_count[NR_IRQS];
102 unsigned int last_lp2_int_count[NR_IRQS];
105 static inline unsigned int time_to_bin(unsigned int time)
110 static inline void tegra_irq_unmask(int irq)
112 struct irq_data *data = irq_get_irq_data(irq);
113 data->chip->irq_unmask(data);
116 static inline unsigned int cpu_number(unsigned int n)
118 return is_lp_cluster() ? 4 : n;
121 void tegra3_cpu_idle_stats_lp2_ready(unsigned int cpu)
123 idle_stats.cpu_ready_count[cpu_number(cpu)]++;
126 void tegra3_cpu_idle_stats_lp2_time(unsigned int cpu, s64 us)
128 idle_stats.cpu_wants_lp2_time[cpu_number(cpu)] += us;
131 /* Allow rail off only if all secondary CPUs are power gated, and no
132 rail update is in progress */
133 static bool tegra_rail_off_is_allowed(void)
135 u32 rst = readl(CLK_RST_CONTROLLER_CPU_CMPLX_STATUS);
136 u32 pg = readl(PMC_POWERGATE_STATUS) >> 8;
138 if (((rst & 0xE) != 0xE) || ((pg & 0xE) != 0))
141 if (tegra_dvfs_rail_updating(cpu_clk_for_dvfs))
147 bool tegra3_lp2_is_allowed(struct cpuidle_device *dev,
148 struct cpuidle_state *state)
152 if (!tegra_all_cpus_booted)
155 if ((!lp2_0_in_idle && !dev->cpu) || (!lp2_n_in_idle && dev->cpu))
158 /* On A01, LP2 on slave CPU's cause ranhdom CPU hangs.
159 * Refer to Bug 804085.
161 if ((tegra_revision == TEGRA_REVISION_A01) &&
162 (tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
163 num_online_cpus() > 1)
166 #ifndef CONFIG_TEGRA_RAIL_OFF_MULTIPLE_CPUS
167 /* FIXME: All CPU's entering LP2 is not working.
168 * Don't let CPU0 enter LP2 when any secondary CPU is online.
170 if ((dev->cpu == 0) && (num_online_cpus() > 1))
173 if ((dev->cpu == 0) && (!tegra_rail_off_is_allowed()))
176 request = ktime_to_us(tick_nohz_get_sleep_length());
177 if (state->exit_latency != lp2_exit_latencies[cpu_number(dev->cpu)]) {
178 /* possible on the 1st entry after cluster switch*/
179 state->exit_latency = lp2_exit_latencies[cpu_number(dev->cpu)];
180 tegra_lp2_update_target_residency(state);
182 if (request < state->target_residency) {
183 /* Not enough time left to enter LP2 */
190 static inline void tegra3_lp2_restore_affinity(void)
193 /* Disable the distributor. */
194 tegra_gic_dist_disable();
196 /* Restore the other CPU's interrupt affinity. */
197 tegra_gic_restore_affinity();
199 /* Re-enable the distributor. */
200 tegra_gic_dist_enable();
204 static bool tegra_cpu_cluster_power_down(struct cpuidle_device *dev,
205 struct cpuidle_state *state, s64 request)
209 bool sleep_completed = false;
210 bool multi_cpu_entry = false;
212 unsigned int flag = 0;
216 entry_time = ktime_get();
218 if (request < state->target_residency) {
219 /* Not enough time left to enter LP2 */
225 multi_cpu_entry = !is_lp_cluster() && (num_online_cpus() > 1);
226 if (multi_cpu_entry) {
230 /* Disable the distributor -- this is the only way to
231 prevent the other CPUs from responding to interrupts
232 and potentially fiddling with the distributor
233 registers while we're fiddling with them. */
234 tegra_gic_dist_disable();
236 /* Did an interrupt come in for another CPU before we
237 could disable the distributor? */
238 if (!tegra_rail_off_is_allowed()) {
239 /* Yes, re-enable the distributor and LP3. */
240 tegra_gic_dist_enable();
245 /* LP2 initial targeted wake time */
246 wake_time = ktime_to_us(entry_time) + request;
248 /* CPU0 must wake up before any of the other CPUs. */
250 for (i = 1; i < CONFIG_NR_CPUS; i++)
251 wake_time = min_t(s64, wake_time,
252 tegra_cpu_wake_by_time[i]);
254 /* LP2 actual targeted wake time */
255 request = wake_time - ktime_to_us(entry_time);
256 BUG_ON(wake_time < 0LL);
258 if (request < state->target_residency) {
259 /* Not enough time left to enter LP2 */
260 tegra_gic_dist_enable();
265 /* Cancel LP2 wake timers for all secondary CPUs */
266 tegra_lp2_timer_cancel_secondary();
268 /* Save and disable the affinity setting for the other
269 CPUs and route all interrupts to CPU0. */
270 tegra_gic_disable_affinity();
272 /* Re-enable the distributor. */
273 tegra_gic_dist_enable();
278 sleep_time = request -
279 lp2_exit_latencies[cpu_number(dev->cpu)];
281 bin = time_to_bin((u32)request / 1000);
282 idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
283 idle_stats.lp2_count++;
284 idle_stats.lp2_count_bin[bin]++;
286 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
287 if (!is_lp_cluster())
288 tegra_dvfs_rail_off(tegra_cpu_rail, entry_time);
290 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
291 flag = get_power_gating_partition();
293 if (tegra_idle_lp2_last(sleep_time, flag) == 0)
294 sleep_completed = true;
296 int irq = tegra_gic_pending_interrupt();
297 idle_stats.lp2_int_count[irq]++;
300 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
301 exit_time = ktime_get();
302 if (!is_lp_cluster())
303 tegra_dvfs_rail_on(tegra_cpu_rail, exit_time);
305 idle_stats.in_lp2_time[cpu_number(dev->cpu)] +=
306 ktime_to_us(ktime_sub(exit_time, entry_time));
309 tegra3_lp2_restore_affinity();
311 if (sleep_completed) {
313 * Stayed in LP2 for the full time until the next tick,
314 * adjust the exit latency based on measurement
316 int offset = ktime_to_us(ktime_sub(exit_time, entry_time))
318 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
320 latency = clamp(latency, 0, 10000);
321 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
322 state->exit_latency = latency; /* for idle governor */
325 idle_stats.lp2_completed_count++;
326 idle_stats.lp2_completed_count_bin[bin]++;
328 pr_debug("%lld %lld %d %d\n", request,
329 ktime_to_us(ktime_sub(exit_time, entry_time)),
338 #ifndef CONFIG_TRUSTED_FOUNDATIONS
339 static unsigned int g_diag_reg;
341 static void save_cpu_arch_register(void)
343 /* read diagnostic register */
344 if (((read_cpuid_id() >> 4) & 0xFFF) == 0xC09)
345 asm("mrc p15, 0, %0, c15, c0, 1" : "=r"(g_diag_reg) : : "cc");
348 static void restore_cpu_arch_register(void)
350 /* write diagnostic register */
351 if (((read_cpuid_id() >> 4) & 0xFFF) == 0xC09)
352 asm("mcr p15, 0, %0, c15, c0, 1" : : "r"(g_diag_reg) : "cc");
356 static bool tegra_cpu_core_power_down(struct cpuidle_device *dev,
357 struct cpuidle_state *state, s64 request)
362 #ifdef CONFIG_HAVE_ARM_TWD
363 struct tegra_twd_context twd_context;
365 #ifdef CONFIG_ARM_ARCH_TIMER
366 struct arch_timer_context timer_context;
368 bool sleep_completed = false;
369 struct tick_sched *ts = tick_get_tick_sched(dev->cpu);
370 #if defined(CONFIG_TEGRA_LP2_CPU_TIMER) && defined(CONFIG_HAVE_ARM_TWD)
371 void __iomem *twd_base = IO_ADDRESS(TEGRA_ARM_PERIF_BASE + 0x600);
374 #ifdef CONFIG_HAVE_ARM_TWD
375 if (!tegra_twd_get_state(&twd_context)) {
376 unsigned long twd_rate = clk_get_rate(twd_clk);
378 if ((twd_context.twd_ctrl & TWD_TIMER_CONTROL_ENABLE) &&
379 (twd_context.twd_ctrl & TWD_TIMER_CONTROL_IT_ENABLE)) {
380 request = div_u64((u64)twd_context.twd_cnt * 1000000,
382 #ifdef CONFIG_TEGRA_LP2_CPU_TIMER
383 if (request >= state->target_residency) {
384 twd_context.twd_cnt -= state->exit_latency *
385 (twd_rate / 1000000);
386 writel(twd_context.twd_cnt,
387 twd_base + TWD_TIMER_COUNTER);
393 #ifdef CONFIG_ARM_ARCH_TIMER
394 if (!arch_timer_get_state(&timer_context)) {
395 if ((timer_context.cntp_ctl & ARCH_TIMER_CTRL_ENABLE) &&
396 ~(timer_context.cntp_ctl & ARCH_TIMER_CTRL_IT_MASK)) {
397 if (timer_context.cntp_tval <= 0) {
401 request = div_u64((u64)timer_context.cntp_tval *
402 1000000, timer_context.cntfrq);
403 #ifdef CONFIG_TEGRA_LP2_CPU_TIMER
404 if (request >= state->target_residency) {
405 timer_context.cntp_tval -= state->exit_latency *
406 (timer_context.cntfrq / 1000000);
407 __asm__("mcr p15, 0, %0, c14, c2, 0\n"
410 "r"(timer_context.cntp_tval));
417 if (!tegra_is_lp2_timer_ready(dev->cpu) ||
418 (request < state->target_residency) ||
419 (!ts) || (ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
421 * Not enough time left to enter LP2, or wake timer not ready
429 #if !defined(CONFIG_TEGRA_LP2_CPU_TIMER)
430 #ifdef CONFIG_HAVE_ARM_TWD
431 sleep_time = request - state->exit_latency;
432 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
433 tegra_twd_suspend(&twd_context);
434 tegra_lp2_set_trigger(sleep_time);
436 #ifdef CONFIG_ARM_ARCH_TIMER
437 sleep_time = request - state->exit_latency;
438 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
439 arch_timer_suspend(&timer_context);
440 tegra_lp2_set_trigger(sleep_time);
443 idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
445 entry_time = ktime_get();
447 /* Save time this CPU must be awakened by. */
448 tegra_cpu_wake_by_time[dev->cpu] = ktime_to_us(entry_time) + request;
451 #ifndef CONFIG_TRUSTED_FOUNDATIONS
452 save_cpu_arch_register();
455 cpu_suspend(0, tegra3_sleep_cpu_secondary_finish);
457 #ifndef CONFIG_TRUSTED_FOUNDATIONS
458 restore_cpu_arch_register();
461 tegra_cpu_wake_by_time[dev->cpu] = LLONG_MAX;
463 #ifdef CONFIG_TEGRA_LP2_CPU_TIMER
464 #ifdef CONFIG_HAVE_ARM_TWD
465 if (!tegra_twd_get_state(&twd_context))
466 sleep_completed = (twd_context.twd_cnt == 0);
468 #ifdef CONFIG_ARM_ARCH_TIMER
469 if (!arch_timer_get_state(&timer_context))
470 sleep_completed = (timer_context.cntp_tval <= 0);
473 sleep_completed = !tegra_lp2_timer_remain();
474 tegra_lp2_set_trigger(0);
475 #ifdef CONFIG_HAVE_ARM_TWD
476 tegra_twd_resume(&twd_context);
478 #ifdef CONFIG_ARM_ARCH_TIMER
479 arch_timer_resume(&timer_context);
481 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
483 sleep_time = ktime_to_us(ktime_sub(ktime_get(), entry_time));
484 idle_stats.in_lp2_time[cpu_number(dev->cpu)] += sleep_time;
485 if (sleep_completed) {
487 * Stayed in LP2 for the full time until timer expires,
488 * adjust the exit latency based on measurement
490 int offset = sleep_time - request;
491 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
493 latency = clamp(latency, 0, 10000);
494 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
495 state->exit_latency = latency; /* for idle governor */
504 bool tegra3_idle_lp2(struct cpuidle_device *dev,
505 struct cpuidle_state *state)
507 s64 request = ktime_to_us(tick_nohz_get_sleep_length());
508 bool last_cpu = tegra_set_cpu_in_lp2(dev->cpu);
511 if ((dev->cpu == 0) && last_cpu)
512 entered_lp2 = tegra_cpu_cluster_power_down(dev, state, request);
514 entered_lp2 = tegra_cpu_core_power_down(dev, state, request);
520 tegra_clear_cpu_in_lp2(dev->cpu);
525 int tegra3_cpuidle_init_soc(void)
529 cpu_clk_for_dvfs = tegra_get_clock_by_name("cpu_g");
530 twd_clk = tegra_get_clock_by_name("twd");
532 for (i = 0; i < ARRAY_SIZE(lp2_exit_latencies); i++)
533 lp2_exit_latencies[i] = tegra_lp2_exit_latency;
538 #ifdef CONFIG_DEBUG_FS
539 int tegra3_lp2_debug_show(struct seq_file *s, void *data)
543 seq_printf(s, " cpu0 cpu1 cpu2 cpu3 cpulp\n");
544 seq_printf(s, "-----------------------------------------------------------------------------\n");
545 seq_printf(s, "cpu ready: %8u %8u %8u %8u %8u\n",
546 idle_stats.cpu_ready_count[0],
547 idle_stats.cpu_ready_count[1],
548 idle_stats.cpu_ready_count[2],
549 idle_stats.cpu_ready_count[3],
550 idle_stats.cpu_ready_count[4]);
551 seq_printf(s, "tear down: %8u %8u %8u %8u %8u\n",
552 idle_stats.tear_down_count[0],
553 idle_stats.tear_down_count[1],
554 idle_stats.tear_down_count[2],
555 idle_stats.tear_down_count[3],
556 idle_stats.tear_down_count[4]);
557 seq_printf(s, "lp2: %8u\n", idle_stats.lp2_count);
558 seq_printf(s, "lp2 completed: %8u %7u%%\n",
559 idle_stats.lp2_completed_count,
560 idle_stats.lp2_completed_count * 100 /
561 (idle_stats.lp2_count ?: 1));
564 seq_printf(s, "cpu ready time: %8llu %8llu %8llu %8llu %8llu ms\n",
565 div64_u64(idle_stats.cpu_wants_lp2_time[0], 1000),
566 div64_u64(idle_stats.cpu_wants_lp2_time[1], 1000),
567 div64_u64(idle_stats.cpu_wants_lp2_time[2], 1000),
568 div64_u64(idle_stats.cpu_wants_lp2_time[3], 1000),
569 div64_u64(idle_stats.cpu_wants_lp2_time[4], 1000));
571 seq_printf(s, "lp2 time: %8llu %8llu %8llu %8llu %8llu ms\n",
572 div64_u64(idle_stats.in_lp2_time[0], 1000),
573 div64_u64(idle_stats.in_lp2_time[1], 1000),
574 div64_u64(idle_stats.in_lp2_time[2], 1000),
575 div64_u64(idle_stats.in_lp2_time[3], 1000),
576 div64_u64(idle_stats.in_lp2_time[4], 1000));
578 seq_printf(s, "lp2 %%: %7d%% %7d%% %7d%% %7d%% %7d%%\n",
579 (int)(idle_stats.cpu_wants_lp2_time[0] ?
580 div64_u64(idle_stats.in_lp2_time[0] * 100,
581 idle_stats.cpu_wants_lp2_time[0]) : 0),
582 (int)(idle_stats.cpu_wants_lp2_time[1] ?
583 div64_u64(idle_stats.in_lp2_time[1] * 100,
584 idle_stats.cpu_wants_lp2_time[1]) : 0),
585 (int)(idle_stats.cpu_wants_lp2_time[2] ?
586 div64_u64(idle_stats.in_lp2_time[2] * 100,
587 idle_stats.cpu_wants_lp2_time[2]) : 0),
588 (int)(idle_stats.cpu_wants_lp2_time[3] ?
589 div64_u64(idle_stats.in_lp2_time[3] * 100,
590 idle_stats.cpu_wants_lp2_time[3]) : 0),
591 (int)(idle_stats.cpu_wants_lp2_time[4] ?
592 div64_u64(idle_stats.in_lp2_time[4] * 100,
593 idle_stats.cpu_wants_lp2_time[4]) : 0));
596 seq_printf(s, "%19s %8s %8s %8s\n", "", "lp2", "comp", "%");
597 seq_printf(s, "-------------------------------------------------\n");
598 for (bin = 0; bin < 32; bin++) {
599 if (idle_stats.lp2_count_bin[bin] == 0)
601 seq_printf(s, "%6u - %6u ms: %8u %8u %7u%%\n",
602 1 << (bin - 1), 1 << bin,
603 idle_stats.lp2_count_bin[bin],
604 idle_stats.lp2_completed_count_bin[bin],
605 idle_stats.lp2_completed_count_bin[bin] * 100 /
606 idle_stats.lp2_count_bin[bin]);
610 seq_printf(s, "%3s %20s %6s %10s\n",
611 "int", "name", "count", "last count");
612 seq_printf(s, "--------------------------------------------\n");
613 for (i = 0; i < NR_IRQS; i++) {
614 if (idle_stats.lp2_int_count[i] == 0)
616 seq_printf(s, "%3d %20s %6d %10d\n",
617 i, irq_to_desc(i)->action ?
618 irq_to_desc(i)->action->name ?: "???" : "???",
619 idle_stats.lp2_int_count[i],
620 idle_stats.lp2_int_count[i] -
621 idle_stats.last_lp2_int_count[i]);
622 idle_stats.last_lp2_int_count[i] = idle_stats.lp2_int_count[i];