ARM: tegra: power: Power off multiple CPUs on-line
[linux-3.10.git] / arch / arm / mach-tegra / cpuidle-t3.c
1 /*
2  * arch/arm/mach-tegra/cpuidle-t3.c
3  *
4  * CPU idle driver for Tegra3 CPUs
5  *
6  * Copyright (c) 2010-2011, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/cpu.h>
25 #include <linux/cpuidle.h>
26 #include <linux/debugfs.h>
27 #include <linux/delay.h>
28 #include <linux/hrtimer.h>
29 #include <linux/init.h>
30 #include <linux/interrupt.h>
31 #include <linux/irq.h>
32 #include <linux/io.h>
33 #include <linux/ratelimit.h>
34 #include <linux/sched.h>
35 #include <linux/seq_file.h>
36 #include <linux/slab.h>
37 #include <linux/smp.h>
38 #include <linux/suspend.h>
39 #include <linux/tick.h>
40 #include <linux/cpu_pm.h>
41 #include <linux/clk.h>
42 #include <linux/moduleparam.h>
43
44 #include <asm/cacheflush.h>
45 #include <asm/hardware/gic.h>
46 #include <asm/localtimer.h>
47 #include <asm/smp_twd.h>
48
49 #include <mach/iomap.h>
50 #include <mach/irqs.h>
51
52 #include <trace/events/power.h>
53
54 #include "clock.h"
55 #include "cpuidle.h"
56 #include "dvfs.h"
57 #include "fuse.h"
58 #include "gic.h"
59 #include "pm.h"
60 #include "reset.h"
61 #include "sleep.h"
62 #include "timer.h"
63
64 #define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS \
65         (IO_ADDRESS(TEGRA_CLK_RESET_BASE) + 0x470)
66 #define PMC_POWERGATE_STATUS \
67         (IO_ADDRESS(TEGRA_PMC_BASE) + 0x038)
68
69 #ifdef CONFIG_SMP
70 static s64 tegra_cpu_wake_by_time[4] = {
71         LLONG_MAX, LLONG_MAX, LLONG_MAX, LLONG_MAX };
72 #endif
73
74 static bool lp2_0_in_idle = true;
75 module_param(lp2_0_in_idle, bool, 0644);
76
77 static bool lp2_n_in_idle = true;
78 module_param(lp2_n_in_idle, bool, 0644);
79
80 static struct clk *cpu_clk_for_dvfs;
81 static struct clk *twd_clk;
82
83 static int lp2_exit_latencies[5];
84
85 static struct {
86         unsigned int cpu_ready_count[5];
87         unsigned int tear_down_count[5];
88         unsigned long long cpu_wants_lp2_time[5];
89         unsigned long long in_lp2_time[5];
90         unsigned int lp2_count;
91         unsigned int lp2_completed_count;
92         unsigned int lp2_count_bin[32];
93         unsigned int lp2_completed_count_bin[32];
94         unsigned int lp2_int_count[NR_IRQS];
95         unsigned int last_lp2_int_count[NR_IRQS];
96 } idle_stats;
97
98 static inline unsigned int time_to_bin(unsigned int time)
99 {
100         return fls(time);
101 }
102
103 static inline void tegra_irq_unmask(int irq)
104 {
105         struct irq_data *data = irq_get_irq_data(irq);
106         data->chip->irq_unmask(data);
107 }
108
109 static inline unsigned int cpu_number(unsigned int n)
110 {
111         return is_lp_cluster() ? 4 : n;
112 }
113
114 void tegra3_cpu_idle_stats_lp2_ready(unsigned int cpu)
115 {
116         idle_stats.cpu_ready_count[cpu_number(cpu)]++;
117 }
118
119 void tegra3_cpu_idle_stats_lp2_time(unsigned int cpu, s64 us)
120 {
121         idle_stats.cpu_wants_lp2_time[cpu_number(cpu)] += us;
122 }
123
124 /* Allow rail off only if all secondary CPUs are power gated, and no
125    rail update is in progress */
126 static bool tegra3_rail_off_is_allowed(void)
127 {
128         u32 rst = readl(CLK_RST_CONTROLLER_CPU_CMPLX_STATUS);
129         u32 pg = readl(PMC_POWERGATE_STATUS) >> 8;
130
131         if (((rst & 0xE) != 0xE) || ((pg & 0xE) != 0))
132                 return false;
133
134         if (tegra_dvfs_rail_updating(cpu_clk_for_dvfs))
135                 return false;
136
137         return true;
138 }
139
140 bool tegra3_lp2_is_allowed(struct cpuidle_device *dev,
141         struct cpuidle_state *state)
142 {
143         s64 request;
144
145         if (!tegra_all_cpus_booted)
146                 return false;
147
148         if ((!lp2_0_in_idle && !dev->cpu) || (!lp2_n_in_idle && dev->cpu))
149                 return false;
150
151         /* On A01, LP2 on slave CPU's cause ranhdom CPU hangs.
152          * Refer to Bug 804085.
153          */
154         if ((tegra_revision == TEGRA_REVISION_A01) &&
155                 num_online_cpus() > 1)
156                 return false;
157
158 #ifndef CONFIG_TEGRA_RAIL_OFF_MULTIPLE_CPUS
159         /* FIXME: All CPU's entering LP2 is not working.
160          * Don't let CPU0 enter LP2 when any secondary CPU is online.
161          */
162         if ((dev->cpu == 0) && (num_online_cpus() > 1))
163                 return false;
164 #endif
165         if ((dev->cpu == 0)  && (!tegra3_rail_off_is_allowed()))
166                 return false;
167
168         request = ktime_to_us(tick_nohz_get_sleep_length());
169         if (state->exit_latency != lp2_exit_latencies[cpu_number(dev->cpu)]) {
170                 /* possible on the 1st entry after cluster switch*/
171                 state->exit_latency = lp2_exit_latencies[cpu_number(dev->cpu)];
172                 tegra_lp2_update_target_residency(state);
173         }
174         if (request < state->target_residency) {
175                 /* Not enough time left to enter LP2 */
176                 return false;
177         }
178
179         return true;
180 }
181
182 static inline void tegra3_lp3_fall_back(struct cpuidle_device *dev)
183 {
184         tegra_cpu_wfi();
185         /* fall back here from LP2 path - tell cpuidle governor */
186         dev->last_state = &dev->states[0];
187 }
188
189 static inline void tegra3_lp2_restore_affinity(void)
190 {
191 #ifdef CONFIG_SMP
192         /* Disable the distributor. */
193         tegra_gic_dist_disable();
194
195         /* Restore the other CPU's interrupt affinity. */
196         tegra_gic_restore_affinity();
197
198         /* Re-enable the distributor. */
199         tegra_gic_dist_enable();
200 #endif
201 }
202
203 static void tegra3_idle_enter_lp2_cpu_0(struct cpuidle_device *dev,
204                            struct cpuidle_state *state, s64 request)
205 {
206         ktime_t entry_time;
207         ktime_t exit_time;
208         bool sleep_completed = false;
209         bool multi_cpu_entry = false;
210         int bin;
211         s64 sleep_time;
212
213         /* LP2 entry time */
214         entry_time = ktime_get();
215
216         if (request < state->target_residency) {
217                 /* Not enough time left to enter LP2 */
218                 tegra3_lp3_fall_back(dev);
219                 return;
220         }
221
222 #ifdef CONFIG_SMP
223         multi_cpu_entry = !is_lp_cluster() && (num_online_cpus() > 1);
224         if (multi_cpu_entry) {
225                 s64 wake_time;
226                 unsigned int i;
227
228                 /* Disable the distributor -- this is the only way to
229                    prevent the other CPUs from responding to interrupts
230                    and potentially fiddling with the distributor
231                    registers while we're fiddling with them. */
232                 tegra_gic_dist_disable();
233
234                 /* Did an interrupt come in for another CPU before we
235                    could disable the distributor? */
236                 if (!tegra3_rail_off_is_allowed()) {
237                         /* Yes, re-enable the distributor and LP3. */
238                         tegra_gic_dist_enable();
239                         tegra3_lp3_fall_back(dev);
240                         return;
241                 }
242
243                 /* LP2 initial targeted wake time */
244                 wake_time = ktime_to_us(entry_time) + request;
245
246                 /* CPU0 must wake up before any of the other CPUs. */
247                 smp_rmb();
248                 for (i = 1; i < CONFIG_NR_CPUS; i++)
249                         wake_time = min_t(s64, wake_time,
250                                 tegra_cpu_wake_by_time[i]);
251
252                 /* LP2 actual targeted wake time */
253                 request = wake_time - ktime_to_us(entry_time);
254                 BUG_ON(wake_time < 0LL);
255
256                 if (request < state->target_residency) {
257                         /* Not enough time left to enter LP2 */
258                         tegra_gic_dist_enable();
259                         tegra3_lp3_fall_back(dev);
260                         return;
261                 }
262
263                 /* Cancel LP2 wake timers for all secondary CPUs */
264                 tegra_lp2_timer_cancel_secondary();
265
266                 /* Save and disable the affinity setting for the other
267                    CPUs and route all interrupts to CPU0. */
268                 tegra_gic_disable_affinity();
269
270                 /* Re-enable the distributor. */
271                 tegra_gic_dist_enable();
272         }
273 #endif
274
275         sleep_time = request -
276                 lp2_exit_latencies[cpu_number(dev->cpu)];
277
278         bin = time_to_bin((u32)request / 1000);
279         idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
280         idle_stats.lp2_count++;
281         idle_stats.lp2_count_bin[bin]++;
282
283         trace_power_start(POWER_CSTATE, 2, dev->cpu);
284         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
285         if (!is_lp_cluster())
286                 tegra_dvfs_rail_off(tegra_cpu_rail, entry_time);
287
288         if (tegra_idle_lp2_last(sleep_time, 0) == 0)
289                 sleep_completed = true;
290         else {
291                 int irq = tegra_gic_pending_interrupt();
292                 idle_stats.lp2_int_count[irq]++;
293         }
294
295         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
296         exit_time = ktime_get();
297         if (!is_lp_cluster())
298                 tegra_dvfs_rail_on(tegra_cpu_rail, exit_time);
299         idle_stats.in_lp2_time[cpu_number(dev->cpu)] +=
300                 ktime_to_us(ktime_sub(exit_time, entry_time));
301
302         if (multi_cpu_entry)
303                 tegra3_lp2_restore_affinity();
304
305         if (sleep_completed) {
306                 /*
307                  * Stayed in LP2 for the full time until the next tick,
308                  * adjust the exit latency based on measurement
309                  */
310                 int offset = ktime_to_us(ktime_sub(exit_time, entry_time))
311                         - request;
312                 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
313                         offset / 16;
314                 latency = clamp(latency, 0, 10000);
315                 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
316                 state->exit_latency = latency;          /* for idle governor */
317                 smp_wmb();
318
319                 idle_stats.lp2_completed_count++;
320                 idle_stats.lp2_completed_count_bin[bin]++;
321
322                 pr_debug("%lld %lld %d %d\n", request,
323                         ktime_to_us(ktime_sub(exit_time, entry_time)),
324                         offset, bin);
325         }
326 }
327
328 static void tegra3_idle_enter_lp2_cpu_n(struct cpuidle_device *dev,
329                            struct cpuidle_state *state, s64 request)
330 {
331 #ifdef CONFIG_SMP
332         s64 sleep_time;
333         ktime_t entry_time;
334         struct tegra_twd_context twd_context;
335         bool sleep_completed = false;
336
337         if (!tegra_twd_get_state(&twd_context)) {
338                 unsigned long twd_rate = clk_get_rate(twd_clk);
339
340                 if ((twd_context.twd_ctrl & TWD_TIMER_CONTROL_ENABLE) &&
341                     (twd_context.twd_ctrl & TWD_TIMER_CONTROL_IT_ENABLE)) {
342                         request = div_u64((u64)twd_context.twd_cnt * 1000000,
343                                           twd_rate);
344 #ifdef CONFIG_TEGRA_LP2_ARM_TWD
345                         if (request >= state->target_residency) {
346                                 twd_context.twd_cnt -= state->exit_latency *
347                                         (twd_rate / 1000000);
348                                 writel(twd_context.twd_cnt,
349                                         twd_base + TWD_TIMER_COUNTER);
350                         }
351 #endif
352                 }
353         }
354
355         if (!tegra_is_lp2_timer_ready(dev->cpu) ||
356             (request < state->target_residency)) {
357                 /*
358                  * Not enough time left to enter LP2, or wake timer not ready
359                  */
360                 tegra3_lp3_fall_back(dev);
361                 return;
362         }
363
364 #ifndef CONFIG_TEGRA_LP2_ARM_TWD
365         sleep_time = request - state->exit_latency;
366         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
367         tegra_twd_suspend(&twd_context);
368         tegra_lp2_set_trigger(sleep_time);
369 #endif
370         idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
371
372         trace_power_start(POWER_CSTATE, 2, dev->cpu);
373
374         entry_time = ktime_get();
375
376         /* Save time this CPU must be awakened by. */
377         tegra_cpu_wake_by_time[dev->cpu] = ktime_to_us(entry_time) + request;
378         smp_wmb();
379
380         tegra3_sleep_cpu_secondary(PLAT_PHYS_OFFSET - PAGE_OFFSET);
381
382         tegra_cpu_wake_by_time[dev->cpu] = LLONG_MAX;
383
384 #ifdef CONFIG_TEGRA_LP2_ARM_TWD
385         if (!tegra_twd_get_state(&twd_context))
386                 sleep_completed = (twd_context.twd_cnt == 0);
387 #else
388         sleep_completed = !tegra_lp2_timer_remain();
389         tegra_lp2_set_trigger(0);
390         tegra_twd_resume(&twd_context);
391         clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
392 #endif
393         sleep_time = ktime_to_us(ktime_sub(ktime_get(), entry_time));
394         idle_stats.in_lp2_time[cpu_number(dev->cpu)] += sleep_time;
395         if (sleep_completed) {
396                 /*
397                  * Stayed in LP2 for the full time until timer expires,
398                  * adjust the exit latency based on measurement
399                  */
400                 int offset = sleep_time - request;
401                 int latency = lp2_exit_latencies[cpu_number(dev->cpu)] +
402                         offset / 16;
403                 latency = clamp(latency, 0, 10000);
404                 lp2_exit_latencies[cpu_number(dev->cpu)] = latency;
405                 state->exit_latency = latency;          /* for idle governor */
406                 smp_wmb();
407         }
408 #endif
409 }
410
411 void tegra3_idle_lp2(struct cpuidle_device *dev,
412                            struct cpuidle_state *state)
413 {
414         s64 request = ktime_to_us(tick_nohz_get_sleep_length());
415         bool last_cpu = tegra_set_cpu_in_lp2(dev->cpu);
416
417         cpu_pm_enter();
418
419         if (dev->cpu == 0) {
420                 if (last_cpu)
421                         tegra3_idle_enter_lp2_cpu_0(dev, state, request);
422                 else
423                         tegra3_lp3_fall_back(dev);
424         } else
425                 tegra3_idle_enter_lp2_cpu_n(dev, state, request);
426
427         cpu_pm_exit();
428         tegra_clear_cpu_in_lp2(dev->cpu);
429 }
430
431 int tegra3_cpudile_init_soc(void)
432 {
433         int i;
434
435         cpu_clk_for_dvfs = tegra_get_clock_by_name("cpu_g");
436         twd_clk = tegra_get_clock_by_name("twd");
437
438         for (i = 0; i < ARRAY_SIZE(lp2_exit_latencies); i++)
439                 lp2_exit_latencies[i] = tegra_lp2_exit_latency;
440
441         return 0;
442 }
443
444 #ifdef CONFIG_DEBUG_FS
445 int tegra3_lp2_debug_show(struct seq_file *s, void *data)
446 {
447         int bin;
448         int i;
449         seq_printf(s, "                                    cpu0     cpu1     cpu2     cpu3     cpulp\n");
450         seq_printf(s, "-----------------------------------------------------------------------------\n");
451         seq_printf(s, "cpu ready:                      %8u %8u %8u %8u %8u\n",
452                 idle_stats.cpu_ready_count[0],
453                 idle_stats.cpu_ready_count[1],
454                 idle_stats.cpu_ready_count[2],
455                 idle_stats.cpu_ready_count[3],
456                 idle_stats.cpu_ready_count[4]);
457         seq_printf(s, "tear down:                      %8u %8u %8u %8u %8u\n",
458                 idle_stats.tear_down_count[0],
459                 idle_stats.tear_down_count[1],
460                 idle_stats.tear_down_count[2],
461                 idle_stats.tear_down_count[3],
462                 idle_stats.tear_down_count[4]);
463         seq_printf(s, "lp2:            %8u\n", idle_stats.lp2_count);
464         seq_printf(s, "lp2 completed:  %8u %7u%%\n",
465                 idle_stats.lp2_completed_count,
466                 idle_stats.lp2_completed_count * 100 /
467                         (idle_stats.lp2_count ?: 1));
468
469         seq_printf(s, "\n");
470         seq_printf(s, "cpu ready time:                 %8llu %8llu %8llu %8llu %8llu ms\n",
471                 div64_u64(idle_stats.cpu_wants_lp2_time[0], 1000),
472                 div64_u64(idle_stats.cpu_wants_lp2_time[1], 1000),
473                 div64_u64(idle_stats.cpu_wants_lp2_time[2], 1000),
474                 div64_u64(idle_stats.cpu_wants_lp2_time[3], 1000),
475                 div64_u64(idle_stats.cpu_wants_lp2_time[4], 1000));
476
477         seq_printf(s, "lp2 time:                       %8llu %8llu %8llu %8llu %8llu ms\n",
478                 div64_u64(idle_stats.in_lp2_time[0], 1000),
479                 div64_u64(idle_stats.in_lp2_time[1], 1000),
480                 div64_u64(idle_stats.in_lp2_time[2], 1000),
481                 div64_u64(idle_stats.in_lp2_time[3], 1000),
482                 div64_u64(idle_stats.in_lp2_time[4], 1000));
483
484         seq_printf(s, "lp2 %%:                         %7d%% %7d%% %7d%% %7d%% %7d%%\n",
485                 (int)(idle_stats.cpu_wants_lp2_time[0] ?
486                         div64_u64(idle_stats.in_lp2_time[0] * 100,
487                         idle_stats.cpu_wants_lp2_time[0]) : 0),
488                 (int)(idle_stats.cpu_wants_lp2_time[1] ?
489                         div64_u64(idle_stats.in_lp2_time[1] * 100,
490                         idle_stats.cpu_wants_lp2_time[1]) : 0),
491                 (int)(idle_stats.cpu_wants_lp2_time[2] ?
492                         div64_u64(idle_stats.in_lp2_time[2] * 100,
493                         idle_stats.cpu_wants_lp2_time[2]) : 0),
494                 (int)(idle_stats.cpu_wants_lp2_time[3] ?
495                         div64_u64(idle_stats.in_lp2_time[3] * 100,
496                         idle_stats.cpu_wants_lp2_time[3]) : 0),
497                 (int)(idle_stats.cpu_wants_lp2_time[4] ?
498                         div64_u64(idle_stats.in_lp2_time[4] * 100,
499                         idle_stats.cpu_wants_lp2_time[4]) : 0));
500         seq_printf(s, "\n");
501
502         seq_printf(s, "%19s %8s %8s %8s\n", "", "lp2", "comp", "%");
503         seq_printf(s, "-------------------------------------------------\n");
504         for (bin = 0; bin < 32; bin++) {
505                 if (idle_stats.lp2_count_bin[bin] == 0)
506                         continue;
507                 seq_printf(s, "%6u - %6u ms: %8u %8u %7u%%\n",
508                         1 << (bin - 1), 1 << bin,
509                         idle_stats.lp2_count_bin[bin],
510                         idle_stats.lp2_completed_count_bin[bin],
511                         idle_stats.lp2_completed_count_bin[bin] * 100 /
512                                 idle_stats.lp2_count_bin[bin]);
513         }
514
515         seq_printf(s, "\n");
516         seq_printf(s, "%3s %20s %6s %10s\n",
517                 "int", "name", "count", "last count");
518         seq_printf(s, "--------------------------------------------\n");
519         for (i = 0; i < NR_IRQS; i++) {
520                 if (idle_stats.lp2_int_count[i] == 0)
521                         continue;
522                 seq_printf(s, "%3d %20s %6d %10d\n",
523                         i, irq_to_desc(i)->action ?
524                                 irq_to_desc(i)->action->name ?: "???" : "???",
525                         idle_stats.lp2_int_count[i],
526                         idle_stats.lp2_int_count[i] -
527                                 idle_stats.last_lp2_int_count[i]);
528                 idle_stats.last_lp2_int_count[i] = idle_stats.lp2_int_count[i];
529         };
530         return 0;
531 }
532 #endif