d8cf8c299a0add757855cc355bc417d3ab81afa8
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2013, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/cpumask.h>
32 #include <linux/delay.h>
33 #include <linux/cpu_pm.h>
34 #include <linux/err.h>
35 #include <linux/debugfs.h>
36 #include <linux/delay.h>
37 #include <linux/suspend.h>
38 #include <linux/slab.h>
39 #include <linux/serial_reg.h>
40 #include <linux/seq_file.h>
41 #include <linux/uaccess.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/cpu_pm.h>
44 #include <linux/clk/tegra.h>
45 #include <linux/export.h>
46 #include <linux/vmalloc.h>
47 #include <linux/memblock.h>
48 #include <linux/console.h>
49 #include <linux/tegra_audio.h>
50 #include <linux/clk/tegra.h>
51 #include <linux/tegra-powergate.h>
52 #include <linux/tegra-soc.h>
53 #include <linux/tegra-timer.h>
54 #include <linux/tegra-cpuidle.h>
55 #include <linux/irqchip/tegra.h>
56 #include <linux/tegra-pm.h>
57
58 #include <trace/events/power.h>
59 #include <trace/events/nvsecurity.h>
60
61 #include <asm/cacheflush.h>
62 #include <asm/idmap.h>
63 #include <asm/localtimer.h>
64 #include <asm/pgalloc.h>
65 #include <asm/pgtable.h>
66 #include <asm/tlbflush.h>
67 #include <asm/suspend.h>
68 #include <asm/smp_plat.h>
69 #include <asm/rodata.h>
70
71 #include <mach/irqs.h>
72 #include <mach/tegra_smmu.h>
73 #include <mach/pm_domains.h>
74
75 #include "board.h"
76 #include "clock.h"
77 #include "common.h"
78 #include "iomap.h"
79 #include "pm.h"
80 #include "reset.h"
81 #include "pmc.h"
82 #include "sleep.h"
83 #include "dvfs.h"
84 #include "cpu-tegra.h"
85 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
86 #include "tegra14_scratch.h"
87 #endif
88
89 struct suspend_context {
90         /*
91          * The next 7 values are referenced by offset in __restart_plls
92          * in headsmp-t2.S, and should not be moved
93          */
94         u32 pllx_misc;
95         u32 pllx_base;
96         u32 pllp_misc;
97         u32 pllp_base;
98         u32 pllp_outa;
99         u32 pllp_outb;
100         u32 pll_timeout;
101
102         u32 cpu_burst;
103         u32 clk_csite_src;
104         u32 cclk_divider;
105
106         u32 mc[3];
107         u8 uart[5];
108
109         struct tegra_twd_context twd;
110 };
111 #define PMC_CTRL                0x0
112
113 #ifdef CONFIG_PM_SLEEP
114 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
115 static pgd_t *tegra_pgd;
116 static DEFINE_SPINLOCK(tegra_lp2_lock);
117 static cpumask_t tegra_in_lp2;
118 static cpumask_t *iram_cpu_lp2_mask;
119 static unsigned long *iram_cpu_lp1_mask;
120 static unsigned long *iram_mc_clk_mask;
121 static u8 *iram_save;
122 static unsigned long iram_save_size;
123 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
124 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
125 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
126 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
127 static void __iomem *tert_ictlr = IO_ADDRESS(TEGRA_TERTIARY_ICTLR_BASE);
128 #endif
129 static void __iomem *tmrus_reg_base = IO_ADDRESS(TEGRA_TMR1_BASE);
130 static int tegra_last_pclk;
131 static u64 resume_time;
132 static u64 resume_entry_time;
133 static u64 suspend_time;
134 static u64 suspend_entry_time;
135 #endif
136
137 static RAW_NOTIFIER_HEAD(tegra_pm_chain_head);
138
139 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
140 static void update_pmc_registers(unsigned long rate);
141 #endif
142
143 struct suspend_context tegra_sctx;
144 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
145 extern struct device *get_se_device(void);
146 extern int se_suspend(struct device *dev, bool pooling);
147 extern struct device *get_smmu_device(void);
148 extern int tegra_smmu_resume(struct device *dev);
149 extern int tegra_smmu_suspend(struct device *dev);
150 #endif
151
152 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
153 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
154 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
155 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
156 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
157 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
158 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
159 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
160 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
161
162 #define TEGRA_DPAD_ORIDE_SYS_CLK_REQ    (1 << 21)
163
164 #define PMC_CTRL                0x0
165 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
166 #define PMC_WAKE_MASK           0xc
167 #define PMC_WAKE_LEVEL          0x10
168 #define PMC_DPAD_ORIDE          0x1C
169 #define PMC_WAKE_DELAY          0xe0
170 #define PMC_DPD_SAMPLE          0x20
171 #if defined(CONFIG_ARCH_TEGRA_14x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
172 #define PMC_DPD_ENABLE          0x24
173 #endif
174 #define PMC_IO_DPD_REQ          0x1B8
175 #define PMC_IO_DPD2_REQ         0x1C0
176
177
178 #define PMC_WAKE_STATUS         0x14
179 #define PMC_SW_WAKE_STATUS      0x18
180 #define PMC_COREPWRGOOD_TIMER   0x3c
181 #define PMC_CPUPWRGOOD_TIMER    0xc8
182 #define PMC_CPUPWROFF_TIMER     0xcc
183 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
184
185 #define PMC_PWRGATE_TOGGLE      0x30
186 #define PWRGATE_TOGGLE_START    (1 << 8)
187 #define UN_PWRGATE_CPU          \
188         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
189
190 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
191 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
192 #endif
193
194 #define CLK_RESET_CCLK_BURST    0x20
195 #define CLK_RESET_CCLK_DIVIDER  0x24
196 #define CLK_RESET_PLLC_BASE     0x80
197 #define CLK_RESET_PLLM_BASE     0x90
198 #define CLK_RESET_PLLX_BASE     0xe0
199 #define CLK_RESET_PLLX_MISC     0xe4
200 #define CLK_RESET_PLLP_BASE     0xa0
201 #define CLK_RESET_PLLP_OUTA     0xa4
202 #define CLK_RESET_PLLP_OUTB     0xa8
203 #define CLK_RESET_PLLP_MISC     0xac
204
205 #define CLK_RESET_SOURCE_CSITE  0x1d4
206
207 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
208 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
209 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
210 #define CLK_RESET_CCLK_IDLE_POLICY         1
211 #define CLK_RESET_CCLK_RUN_POLICY          2
212 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
213 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
214
215 #define EMC_MRW_0               0x0e8
216 #define EMC_MRW_DEV_SELECTN     30
217 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
218
219 #define MC_SECURITY_START       0x6c
220 #define MC_SECURITY_SIZE        0x70
221 #define MC_SECURITY_CFG2        0x7c
222
223 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
224 static struct clk *tegra_dfll;
225 #endif
226 static struct clk *tegra_pclk;
227 static struct clk *tegra_clk_m;
228 static struct tegra_suspend_platform_data *pdata;
229 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
230
231 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
232 static unsigned long
233                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
234 struct tegra_cluster_switch_time_stats {
235         unsigned long sum;
236         unsigned long avg;
237         unsigned long exp_avg;
238         unsigned long max;
239         int cnt;
240 };
241
242 static struct tegra_cluster_switch_time_stats lp2g_stats;
243 static struct tegra_cluster_switch_time_stats g2lp_stats;
244
245 void tegra_cluster_switch_time(unsigned int flags, int id)
246 {
247         unsigned long t;
248         struct tegra_cluster_switch_time_stats *stats;
249
250         if (!(flags & TEGRA_POWER_CLUSTER_MASK) ||
251             (id >= tegra_cluster_switch_time_id_max))
252                 return;
253
254         tegra_cluster_switch_times[id] = tegra_read_usec_raw();
255         wmb();
256         if (id != tegra_cluster_switch_time_id_end)
257                 return;
258
259         stats = flags & TEGRA_POWER_CLUSTER_G ? &lp2g_stats : &g2lp_stats;
260
261         t = tegra_cluster_switch_times[tegra_cluster_switch_time_id_end] -
262                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_start];
263         if (stats->max < t)
264                 stats->max = t;
265
266         stats->sum += t;
267         stats->cnt++;
268         if (stats->cnt < CLUSTER_SWITCH_AVG_SAMPLES)
269                 return;
270
271         stats->avg = stats->sum;
272         stats->cnt = stats->sum = 0;
273         if (!stats->exp_avg) {
274                 stats->exp_avg = stats->avg;    /* 1st window sample */
275                 return;
276         }
277         stats->exp_avg = (stats->exp_avg * (CLUSTER_SWITCH_AVG_SAMPLES - 1) +
278                           stats->avg) >> CLUSTER_SWITCH_TIME_AVG_SHIFT;
279 }
280 #endif
281
282 int tegra_register_pm_notifier(struct notifier_block *nb)
283 {
284         return raw_notifier_chain_register(&tegra_pm_chain_head, nb);
285 }
286 EXPORT_SYMBOL(tegra_register_pm_notifier);
287
288 int tegra_unregister_pm_notifier(struct notifier_block *nb)
289 {
290         return raw_notifier_chain_unregister(&tegra_pm_chain_head, nb);
291 }
292 EXPORT_SYMBOL(tegra_unregister_pm_notifier);
293
294 static int tegra_pm_notifier_call_chain(unsigned int val)
295 {
296         int ret = raw_notifier_call_chain(&tegra_pm_chain_head, val, NULL);
297
298         return notifier_to_errno(ret);
299 }
300
301 #ifdef CONFIG_PM_SLEEP
302 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
303         [TEGRA_SUSPEND_NONE]    = "none",
304         [TEGRA_SUSPEND_LP2]     = "lp2",
305         [TEGRA_SUSPEND_LP1]     = "lp1",
306         [TEGRA_SUSPEND_LP0]     = "lp0",
307 };
308
309 void tegra_log_resume_time(void)
310 {
311         u64 resume_end_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
312
313         if (resume_entry_time > resume_end_time)
314                 resume_end_time |= 1ull<<32;
315         resume_time = resume_end_time - resume_entry_time;
316 }
317
318 void tegra_log_suspend_time(void)
319 {
320         suspend_entry_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
321 }
322
323 static void tegra_get_suspend_time(void)
324 {
325         u64 suspend_end_time;
326         suspend_end_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
327
328         if (suspend_entry_time > suspend_end_time)
329                 suspend_end_time |= 1ull<<32;
330         suspend_time = suspend_end_time - suspend_entry_time;
331 }
332
333 unsigned long tegra_cpu_power_good_time(void)
334 {
335         if (WARN_ON_ONCE(!pdata))
336                 return 5000;
337
338         return pdata->cpu_timer;
339 }
340
341 unsigned long tegra_cpu_power_off_time(void)
342 {
343         if (WARN_ON_ONCE(!pdata))
344                 return 5000;
345
346         return pdata->cpu_off_timer;
347 }
348
349 unsigned long tegra_cpu_lp2_min_residency(void)
350 {
351         if (WARN_ON_ONCE(!pdata))
352                 return 2000;
353
354         return pdata->cpu_lp2_min_residency;
355 }
356
357 #define TEGRA_MIN_RESIDENCY_MCLK_STOP   20000
358
359 unsigned long tegra_mc_clk_stop_min_residency(void)
360 {
361         return pdata && pdata->min_residency_mclk_stop
362                         ? pdata->min_residency_mclk_stop
363                         : TEGRA_MIN_RESIDENCY_MCLK_STOP;
364 }
365
366 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
367 #define TEGRA_MIN_RESIDENCY_VMIN_FMIN   2000
368 #define TEGRA_MIN_RESIDENCY_NCPU_SLOW   2000
369 #define TEGRA_MIN_RESIDENCY_NCPU_FAST   13000
370 #define TEGRA_MIN_RESIDENCY_CRAIL       20000
371
372 unsigned long tegra_min_residency_vmin_fmin(void)
373 {
374         return pdata && pdata->min_residency_vmin_fmin
375                         ? pdata->min_residency_vmin_fmin
376                         : TEGRA_MIN_RESIDENCY_VMIN_FMIN;
377 }
378
379 unsigned long tegra_min_residency_ncpu()
380 {
381         if (is_lp_cluster()) {
382                 return pdata && pdata->min_residency_ncpu_slow
383                         ? pdata->min_residency_ncpu_slow
384                         : TEGRA_MIN_RESIDENCY_NCPU_SLOW;
385         } else
386                 return pdata && pdata->min_residency_ncpu_fast
387                         ? pdata->min_residency_ncpu_fast
388                         : TEGRA_MIN_RESIDENCY_NCPU_FAST;
389 }
390
391 unsigned long tegra_min_residency_crail(void)
392 {
393         return pdata && pdata->min_residency_crail
394                         ? pdata->min_residency_crail
395                         : TEGRA_MIN_RESIDENCY_CRAIL;
396 }
397
398 bool tegra_crail_can_start_early(void)
399 {
400         return pdata && pdata->crail_up_early;
401 }
402 #endif
403
404 static void suspend_cpu_dfll_mode(unsigned int flags)
405 {
406 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
407         /* If DFLL is used as CPU clock source go to open loop mode */
408         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
409                 if (!is_lp_cluster() && tegra_dfll &&
410                     tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
411                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
412         }
413
414         /* Suspend dfll bypass (safe rail down) on LP or if DFLL is Not used */
415         if (pdata && pdata->suspend_dfll_bypass &&
416             (!tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail) || is_lp_cluster()))
417                 pdata->suspend_dfll_bypass();
418 #endif
419 }
420
421 static void resume_cpu_dfll_mode(unsigned int flags)
422 {
423 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
424         /* If DFLL is Not used and resume on G restore bypass mode */
425         if (pdata && pdata->resume_dfll_bypass && !is_lp_cluster() &&
426             tegra_is_clk_initialized(tegra_dfll) &&
427             !tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
428                 pdata->resume_dfll_bypass();
429
430         /* If DFLL is used as CPU clock source restore closed loop mode */
431         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
432                 if (!is_lp_cluster() && tegra_dfll &&
433                     tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
434                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
435         }
436 #endif
437 }
438
439 /*
440  * create_suspend_pgtable
441  *
442  * Creates a page table with identity mappings of physical memory and IRAM
443  * for use when the MMU is off, in addition to all the regular kernel mappings.
444  */
445 static __init int create_suspend_pgtable(void)
446 {
447         tegra_pgd = pgd_alloc(&init_mm);
448         if (!tegra_pgd)
449                 return -ENOMEM;
450
451         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
452         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
453                 high_memory, 0);
454         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
455                 IO_IRAM_VIRT + SECTION_SIZE, 0);
456
457 #if defined(CONFIG_ARM_LPAE)
458         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK);
459 #else
460         /* inner/outer write-back/write-allocate, sharable */
461         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
462 #endif
463
464         return 0;
465 }
466
467 /* ensures that sufficient time is passed for a register write to
468  * serialize into the 32KHz domain */
469 static void pmc_32kwritel(u32 val, unsigned long offs)
470 {
471         writel(val, pmc + offs);
472         udelay(130);
473 }
474
475 #if !defined(CONFIG_OF) || !defined(CONFIG_COMMON_CLK)
476 static void set_power_timers(unsigned long us_on, unsigned long us_off,
477                              long rate)
478 {
479         static unsigned long last_us_off = 0;
480         unsigned long long ticks;
481         unsigned long long pclk;
482
483         if (WARN_ON_ONCE(rate <= 0))
484                 pclk = 100000000;
485         else
486                 pclk = rate;
487
488         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
489                 ticks = (us_on * pclk) + 999999ull;
490                 do_div(ticks, 1000000);
491                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
492
493                 ticks = (us_off * pclk) + 999999ull;
494                 do_div(ticks, 1000000);
495                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
496                 wmb();
497         }
498         tegra_last_pclk = pclk;
499         last_us_off = us_off;
500 }
501 #endif
502
503 void tegra_limit_cpu_power_timers(unsigned long us_on, unsigned long us_off)
504 {
505         /* make sure power timers would not exceed specified limits */
506         set_power_timers(us_on, us_off, clk_get_min_rate(tegra_pclk));
507 }
508
509 void (*tegra_tear_down_cpu)(void);
510
511 /*
512  * restore_cpu_complex
513  *
514  * restores cpu clock setting, clears flow controller
515  *
516  * Always called on CPU 0.
517  */
518 static void restore_cpu_complex(u32 mode)
519 {
520         int cpu = cpu_logical_map(smp_processor_id());
521         unsigned int reg;
522 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
523         unsigned int policy;
524 #endif
525
526 /*
527  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
528  * or restored by common clock suspend/resume procedures. Hence, we don't
529  * need it here.
530  */
531 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
532         /* Is CPU complex already running on PLLX? */
533         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
534         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
535
536         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
537                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
538         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
539                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
540         else
541                 BUG();
542
543         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
544                 /* restore PLLX settings if CPU is on different PLL */
545                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
546                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
547
548                 /* wait for PLL stabilization if PLLX was enabled */
549                 if (tegra_sctx.pllx_base & (1<<30)) {
550 #if USE_PLL_LOCK_BITS
551                         /* Enable lock detector */
552                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
553                         reg |= 1<<18;
554                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
555                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
556                                  (1<<27)))
557                                 cpu_relax();
558
559                         udelay(PLL_POST_LOCK_DELAY);
560 #else
561                         udelay(300);
562 #endif
563                 }
564         }
565
566         /* Restore original burst policy setting for calls resulting from CPU
567            LP2 in idle or system suspend; keep cluster switch prolog setting
568            intact. */
569         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
570                 writel(tegra_sctx.cclk_divider, clk_rst +
571                        CLK_RESET_CCLK_DIVIDER);
572                 writel(tegra_sctx.cpu_burst, clk_rst +
573                        CLK_RESET_CCLK_BURST);
574         }
575 #endif
576         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
577
578         /* Do not power-gate CPU 0 when flow controlled */
579         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
580         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
581         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
582         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
583         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
584         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
585         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
586
587         /* If an immedidate cluster switch is being perfomed, restore the
588            local timer registers. For calls resulting from CPU LP2 in
589            idle or system suspend, the local timer was shut down and
590            timekeeping switched over to the global system timer. In this
591            case keep local timer disabled, and restore only periodic load. */
592 #ifdef CONFIG_HAVE_ARM_TWD
593         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
594                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
595                 tegra_sctx.twd.twd_ctrl = 0;
596         }
597         tegra_twd_resume(&tegra_sctx.twd);
598 #endif
599 }
600
601 /*
602  * suspend_cpu_complex
603  *
604  * saves pll state for use by restart_plls, prepares flow controller for
605  * transition to suspend state
606  *
607  * Must always be called on cpu 0.
608  */
609 static void suspend_cpu_complex(u32 mode)
610 {
611         int cpu = cpu_logical_map(smp_processor_id());
612         unsigned int reg;
613         int i;
614
615         BUG_ON(cpu != 0);
616
617         /* switch coresite to clk_m, save off original source */
618         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
619         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
620
621         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
622         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
623         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
624         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
625         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
626         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
627         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
628         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
629
630 #ifdef CONFIG_HAVE_ARM_TWD
631         tegra_twd_suspend(&tegra_sctx.twd);
632 #endif
633
634         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
635         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
636         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
637         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
638         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
639 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
640         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
641 #else
642         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
643 #endif
644         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
645         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
646
647         for (i = 0; i < num_possible_cpus(); i++) {
648                 if (i == cpu)
649                         continue;
650                 reg = readl(FLOW_CTRL_CPU_CSR(i));
651                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
652                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
653                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
654         }
655
656         tegra_gic_cpu_disable(true);
657 }
658
659 void tegra_clear_cpu_in_pd(int cpu)
660 {
661         spin_lock(&tegra_lp2_lock);
662         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
663         cpumask_clear_cpu(cpu, &tegra_in_lp2);
664
665         /* Update the IRAM copy used by the reset handler. The IRAM copy
666            can't use used directly by cpumask_clear_cpu() because it uses
667            LDREX/STREX which requires the addressed location to be inner
668            cacheable and sharable which IRAM isn't. */
669         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
670         dsb();
671
672         spin_unlock(&tegra_lp2_lock);
673 }
674
675 bool tegra_set_cpu_in_pd(int cpu)
676 {
677         bool last_cpu = false;
678
679         spin_lock(&tegra_lp2_lock);
680         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
681         cpumask_set_cpu(cpu, &tegra_in_lp2);
682
683         /* Update the IRAM copy used by the reset handler. The IRAM copy
684            can't use used directly by cpumask_set_cpu() because it uses
685            LDREX/STREX which requires the addressed location to be inner
686            cacheable and sharable which IRAM isn't. */
687         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
688         dsb();
689
690         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
691                 last_cpu = true;
692 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
693         else if (cpu == 1)
694                 tegra2_cpu_set_resettable_soon();
695 #endif
696
697         spin_unlock(&tegra_lp2_lock);
698         return last_cpu;
699 }
700
701 static void tegra_sleep_core(enum tegra_suspend_mode mode,
702                              unsigned long v2p)
703 {
704 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
705         outer_flush_range(__pa(&tegra_resume_timestamps_start),
706                           __pa(&tegra_resume_timestamps_end));
707
708         if (mode == TEGRA_SUSPEND_LP0) {
709                 trace_smc_sleep_core(NVSEC_SMC_START);
710
711                 tegra_generic_smc(0x84000001, ((1 << 16) | (1 << 24) | 1),
712                                   virt_to_phys(tegra_resume));
713         } else {
714                 trace_smc_sleep_core(NVSEC_SMC_START);
715
716                 tegra_generic_smc(0x84000001, ((1 << 16) | 2),
717                                   (TEGRA_RESET_HANDLER_BASE +
718                                    tegra_cpu_reset_handler_offset));
719         }
720
721         trace_smc_sleep_core(NVSEC_SMC_DONE);
722 #endif
723         tegra_get_suspend_time();
724 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
725         cpu_suspend(v2p, tegra2_sleep_core_finish);
726 #else
727         cpu_suspend(v2p, tegra3_sleep_core_finish);
728 #endif
729 }
730
731 static inline void tegra_sleep_cpu(unsigned long v2p)
732 {
733         cpu_suspend(v2p, tegra_sleep_cpu_finish);
734 }
735
736 static inline void tegra_stop_mc_clk(unsigned long v2p)
737 {
738 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
739         outer_flush_range(__pa(&tegra_resume_timestamps_start),
740                           __pa(&tegra_resume_timestamps_end));
741         trace_smc_sleep_core(NVSEC_SMC_START);
742
743         tegra_generic_smc(0x84000001, ((1 << 16) | 3),
744                           (TEGRA_RESET_HANDLER_BASE +
745                            tegra_cpu_reset_handler_offset));
746
747         trace_smc_sleep_core(NVSEC_SMC_DONE);
748 #endif
749         cpu_suspend(v2p, tegra3_stop_mc_clk_finish);
750 }
751
752 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
753                                         unsigned int flags)
754 {
755         u32 reg;
756         unsigned int remain;
757
758         /* Only the last cpu down does the final suspend steps */
759         reg = readl(pmc + PMC_CTRL);
760         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
761         if (pdata->combined_req)
762                 reg &= ~TEGRA_POWER_PWRREQ_OE;
763         else
764                 reg |= TEGRA_POWER_PWRREQ_OE;
765
766         reg &= ~TEGRA_POWER_EFFECT_LP0;
767         writel(reg, pmc + PMC_CTRL);
768
769         /*
770          * We can use clk_get_rate_all_locked() here, because all other cpus
771          * are in LP2 state and irqs are disabled
772          */
773         suspend_cpu_dfll_mode(flags);
774         if (flags & TEGRA_POWER_CLUSTER_MASK) {
775 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
776                 set_power_timers(pdata->cpu_timer, 2);
777 #else
778                 set_power_timers(pdata->cpu_timer, 2,
779                         clk_get_rate_all_locked(tegra_pclk));
780 #endif
781                 if (flags & TEGRA_POWER_CLUSTER_G) {
782                         /*
783                          * To reduce the vdd_cpu up latency when LP->G
784                          * transition. Before the transition, enable
785                          * the vdd_cpu rail.
786                          */
787                         if (!tegra_crail_can_start_early() && is_lp_cluster()) {
788 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
789                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
790                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
791                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
792 #else
793                                 writel(UN_PWRGATE_CPU,
794                                        pmc + PMC_PWRGATE_TOGGLE);
795 #endif
796                         }
797                 }
798                 tegra_cluster_switch_prolog(flags);
799         } else {
800 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
801                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
802 #else
803                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
804                         clk_get_rate_all_locked(tegra_pclk));
805 #endif
806 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
807                 reg = readl(FLOW_CTRL_CPU_CSR(0));
808                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
809                 if (is_lp_cluster()) {
810                         /* for LP cluster, there is no option for rail gating */
811                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
812                                                 TEGRA_POWER_CLUSTER_PART_MASK)
813                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
814                         else if (flags)
815                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
816                 }
817                 else {
818                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
819                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
820                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
821                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
822                 }
823                 writel(reg, FLOW_CTRL_CPU_CSR(0));
824 #endif
825         }
826
827         if (sleep_time)
828                 tegra_pd_set_trigger(sleep_time);
829
830         cpu_cluster_pm_enter();
831         suspend_cpu_complex(flags);
832         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
833 #if defined(CONFIG_CACHE_L2X0)
834 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
835         flush_cache_all();
836         outer_disable();
837 #elif !defined(CONFIG_ARCH_TEGRA_14x_SOC)
838         tegra_resume_l2_init = 1;
839         __cpuc_flush_dcache_area(&tegra_resume_l2_init, sizeof(unsigned long));
840         outer_flush_range(__pa(&tegra_resume_l2_init),
841                           __pa(&tegra_resume_l2_init) + sizeof(unsigned long));
842 #endif
843 #endif
844
845         /* T148: Check for mem_req and mem_req_soon only if it is
846          * MC clock stop state.
847          */
848         if (flags & TEGRA_POWER_STOP_MC_CLK) {
849 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
850                 u32 val;
851
852                 /* Check if mem_req or mem_req_soon is asserted or if voice
853                  * call is active call, if yes then we skip SDRAM
854                  * self-refresh and just do CPU power-gating.
855                  */
856                 val = readl(pmc + PMC_IPC_STS);
857                 if ((val & (PMC_IPC_STS_MEM_REQ | PMC_IPC_STS_MEM_REQ_SOON)) ||
858                         tegra_is_voice_call_active()) {
859
860                         /* Reset LP1 and MC clock mask if we skipping SDRAM
861                          * self-refresh.
862                          */
863                         *iram_cpu_lp1_mask = 0;
864                         *iram_mc_clk_mask = 0;
865                         writel(0, pmc + PMC_SCRATCH41);
866
867                         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
868                 } else {
869                         /* Clear mem_sts since SDRAM will not be accessible
870                          * to BBC in this state.
871                          */
872                         val = PMC_IPC_CLR_MEM_STS;
873                         writel(val, pmc + PMC_IPC_CLR);
874
875                         tegra_stop_mc_clk(PHYS_OFFSET - PAGE_OFFSET);
876                 }
877 #else
878                 /* If it is not T148 then we do not have to
879                  * check mem_req and mem_req_soon.
880                  */
881                 tegra_stop_mc_clk(PHYS_OFFSET - PAGE_OFFSET);
882 #endif
883         } else {
884                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
885         }
886
887 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
888         tegra_init_cache(true);
889 #elif defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
890         tegra_init_cache(false);
891 #endif
892
893 #if defined(CONFIG_TRUSTED_FOUNDATIONS)
894 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
895         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
896         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
897 #endif
898 #endif
899
900         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
901         restore_cpu_complex(flags);
902         cpu_cluster_pm_exit();
903
904         remain = tegra_pd_timer_remain();
905         if (sleep_time)
906                 tegra_pd_set_trigger(0);
907
908         if (flags & TEGRA_POWER_CLUSTER_MASK) {
909                 tegra_cluster_switch_epilog(flags);
910         }
911         resume_cpu_dfll_mode(flags);
912         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
913
914 #if INSTRUMENT_CLUSTER_SWITCH
915         if (flags & TEGRA_POWER_CLUSTER_MASK) {
916                 pr_debug("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
917                         is_lp_cluster() ? "G=>LP" : "LP=>G",
918                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
919                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
920                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
921                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
922                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
923                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
924                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
925                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
926         }
927 #endif
928         return remain;
929 }
930
931 void tegra_mc_clk_prepare(void)
932 {
933         /* copy the reset vector and SDRAM shutdown code into IRAM */
934         memcpy(iram_save, iram_code, iram_save_size);
935         memcpy(iram_code, tegra_iram_start(), iram_save_size);
936
937         *iram_cpu_lp1_mask = 1;
938         *iram_mc_clk_mask = 1;
939
940         __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
941         wmb();
942 }
943
944 void tegra_mc_clk_finish(void)
945 {
946         /* restore IRAM */
947         memcpy(iram_code, iram_save, iram_save_size);
948         *iram_cpu_lp1_mask = 0;
949         *iram_mc_clk_mask = 0;
950         writel(0, pmc + PMC_SCRATCH41);
951 }
952
953 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
954 int tegra_is_lp1_suspend_mode(void)
955 {
956         return (current_suspend_mode == TEGRA_SUSPEND_LP1);
957 }
958 #endif
959
960 static int tegra_common_suspend(void)
961 {
962         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
963
964         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
965         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
966         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
967
968 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
969         if (pdata && pdata->lp1_lowvolt_support) {
970                 u32 lp1_core_lowvolt =
971                         (tegra_is_voice_call_active() ||
972                         tegra_dvfs_rail_get_thermal_floor(tegra_core_rail)) ?
973                         pdata->lp1_core_volt_low_cold << 8 :
974                         pdata->lp1_core_volt_low << 8;
975
976                 lp1_core_lowvolt |= pdata->core_reg_addr;
977                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
978         }
979 #endif
980
981         /* copy the reset vector and SDRAM shutdown code into IRAM */
982         memcpy(iram_save, iram_code, iram_save_size);
983         memcpy(iram_code, tegra_iram_start(), iram_save_size);
984
985         return 0;
986 }
987
988 static void tegra_common_resume(void)
989 {
990         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
991 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
992         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
993 #endif
994
995 #if defined(CONFIG_ARCH_TEGRA_14x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
996         /* Clear DPD Enable */
997         writel(0x0, pmc + PMC_DPD_ENABLE);
998 #endif
999
1000         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
1001         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
1002         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
1003 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1004         /* trigger emc mode write */
1005         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
1006         /* clear scratch registers shared by suspend and the reset pen */
1007         writel(0x0, pmc + PMC_SCRATCH39);
1008 #endif
1009         writel(0x0, pmc + PMC_SCRATCH41);
1010
1011         /* restore IRAM */
1012         memcpy(iram_code, iram_save, iram_save_size);
1013 }
1014
1015 static int tegra_suspend_prepare_late(void)
1016 {
1017 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1018         disable_irq(INT_SYS_STATS_MON);
1019 #endif
1020         return 0;
1021 }
1022
1023 static void tegra_suspend_wake(void)
1024 {
1025 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1026         enable_irq(INT_SYS_STATS_MON);
1027 #endif
1028 }
1029
1030 static void tegra_pm_set(enum tegra_suspend_mode mode)
1031 {
1032         u32 reg, boot_flag;
1033         unsigned long rate = 32768;
1034
1035         reg = readl(pmc + PMC_CTRL);
1036         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1037         if (pdata->combined_req)
1038                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1039         else
1040                 reg |= TEGRA_POWER_PWRREQ_OE;
1041         reg &= ~TEGRA_POWER_EFFECT_LP0;
1042
1043         switch (mode) {
1044         case TEGRA_SUSPEND_LP0:
1045 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
1046                 rate = clk_get_rate_all_locked(tegra_pclk);
1047 #endif
1048                 if (pdata->combined_req) {
1049                         reg |= TEGRA_POWER_PWRREQ_OE;
1050                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
1051                 }
1052
1053 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1054                 /*
1055                  * LP0 boots through the AVP, which then resumes the AVP to
1056                  * the address in scratch 39, and the cpu to the address in
1057                  * scratch 41 to tegra_resume
1058                  */
1059                 writel(0x0, pmc + PMC_SCRATCH39);
1060 #endif
1061
1062                 /* Enable DPD sample to trigger sampling pads data and direction
1063                  * in which pad will be driven during lp0 mode*/
1064                 writel(0x1, pmc + PMC_DPD_SAMPLE);
1065 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
1066 #if defined(CONFIG_ARCH_TEGRA_11x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
1067                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
1068 #else
1069                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ);
1070 #endif
1071                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
1072 #endif
1073
1074 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1075                 /* this is needed only for T11x, not for other chips */
1076                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
1077 #endif
1078
1079                 /* Set warmboot flag */
1080                 boot_flag = readl(pmc + PMC_SCRATCH0);
1081                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
1082
1083                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
1084
1085                 reg |= TEGRA_POWER_EFFECT_LP0;
1086                 /* No break here. LP0 code falls through to write SCRATCH41 */
1087         case TEGRA_SUSPEND_LP1:
1088                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
1089                 wmb();
1090                 rate = clk_get_rate(tegra_clk_m);
1091                 break;
1092         case TEGRA_SUSPEND_LP2:
1093                 rate = clk_get_rate(tegra_pclk);
1094                 break;
1095         case TEGRA_SUSPEND_NONE:
1096                 return;
1097         default:
1098                 BUG();
1099         }
1100
1101 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
1102         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
1103 #else
1104         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
1105 #endif
1106
1107         pmc_32kwritel(reg, PMC_CTRL);
1108 }
1109
1110 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
1111         [TEGRA_SUSPEND_NONE] = "none",
1112         [TEGRA_SUSPEND_LP2] = "LP2",
1113         [TEGRA_SUSPEND_LP1] = "LP1",
1114         [TEGRA_SUSPEND_LP0] = "LP0",
1115 };
1116
1117 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
1118 static int save_se_context(void)
1119 {
1120         struct device *smmu_dev, *se_dev;
1121         int ret = 0;
1122
1123         smmu_dev = get_smmu_device();
1124         if (!smmu_dev) {
1125                 pr_info("Failed to get smmu device\n");
1126                 goto save_fail;
1127         }
1128
1129         se_dev = get_se_device();
1130         if (!se_dev) {
1131                 pr_info("Failed to get SE device \n");
1132                 goto save_fail;
1133         }
1134
1135         /* smmu resume needs to be called
1136          * for se_suspend() operation */
1137         ret = tegra_smmu_resume(smmu_dev);
1138         if (ret) {
1139                 pr_info("Failed to resume smmu device\n");
1140                 goto save_fail;
1141         }
1142
1143         ret = se_suspend(se_dev, true);
1144         if (ret) {
1145                 pr_info("Failed to suspend SE device\n");
1146                 goto save_fail;
1147         }
1148
1149         ret = tegra_smmu_suspend(smmu_dev);
1150         if (ret) {
1151                 pr_info("Failed to suspend smmu device\n");
1152                 goto save_fail;
1153         }
1154
1155 save_fail:
1156         return ret;
1157 }
1158 #endif
1159
1160 static int tegra_suspend_enter(suspend_state_t state)
1161 {
1162         int ret = 0;
1163         ktime_t delta;
1164         struct timespec ts_entry, ts_exit;
1165
1166         if (pdata && pdata->board_suspend)
1167                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
1168
1169         read_persistent_clock(&ts_entry);
1170
1171         ret = tegra_suspend_dram(current_suspend_mode, 0);
1172         if (ret) {
1173                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
1174                 goto abort_suspend;
1175         }
1176
1177 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
1178         ret = save_se_context();
1179         if (ret) {
1180                 pr_info("Failed to save SE context\n");
1181                 goto abort_suspend;
1182         }
1183 #endif
1184
1185         read_persistent_clock(&ts_exit);
1186
1187         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
1188                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
1189
1190                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
1191                 tegra_dvfs_rail_pause(tegra_gpu_rail, delta, false);
1192                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1193                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
1194                 else
1195                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
1196         }
1197
1198 abort_suspend:
1199         if (pdata && pdata->board_resume)
1200                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
1201
1202         return ret;
1203 }
1204
1205 static void tegra_suspend_check_pwr_stats(void)
1206 {
1207         /* cpus and l2 are powered off later */
1208         unsigned long pwrgate_partid_mask =
1209 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
1210                 (1 << TEGRA_POWERGATE_HEG)      |
1211                 (1 << TEGRA_POWERGATE_SATA)     |
1212                 (1 << TEGRA_POWERGATE_3D1)      |
1213 #endif
1214                 (1 << TEGRA_POWERGATE_3D)       |
1215                 (1 << TEGRA_POWERGATE_VENC)     |
1216                 (1 << TEGRA_POWERGATE_PCIE)     |
1217                 (1 << TEGRA_POWERGATE_VDEC)     |
1218                 (1 << TEGRA_POWERGATE_MPE);
1219
1220         int partid;
1221
1222         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
1223                 if ((1 << partid) & pwrgate_partid_mask)
1224                         if (tegra_powergate_is_powered(partid))
1225                                 pr_debug("partition %s is left on before suspend\n",
1226                                         tegra_powergate_get_name(partid));
1227
1228         return;
1229 }
1230
1231 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1232 /* This is the opposite of the LP1BB related PMC setup that occurs
1233  * during suspend.
1234  */
1235 static void tegra_disable_lp1bb_interrupt(void)
1236 {
1237         unsigned reg;
1238         /* mem_req = 0 was set as an interrupt during LP1BB entry.
1239          * It has to be disabled now
1240          */
1241         reg = readl(pmc + PMC_CTRL2);
1242         reg &= ~(PMC_CTRL2_WAKE_DET_EN);
1243         pmc_32kwritel(reg, PMC_CTRL2);
1244
1245         /* Program mem_req NOT to be a wake event */
1246         reg = readl(pmc + PMC_WAKE2_MASK);
1247         reg &= ~(PMC_WAKE2_BB_MEM_REQ);
1248         pmc_32kwritel(reg, PMC_WAKE2_MASK);
1249
1250         reg = PMC_WAKE2_BB_MEM_REQ;
1251         pmc_32kwritel(reg, PMC_WAKE2_STATUS);
1252
1253         /* Set up the LIC to NOT accept pmc_wake events as interrupts */
1254         reg = TRI_ICTLR_PMC_WAKE_INT;
1255         writel(reg, tert_ictlr + TRI_ICTLR_CPU_IER_CLR);
1256 }
1257 #endif
1258
1259 static void tegra_suspend_powergate_control(int partid, bool turn_off)
1260 {
1261         if (turn_off)
1262                 tegra_powergate_partition(partid);
1263         else
1264                 tegra_unpowergate_partition(partid);
1265 }
1266
1267 #ifdef CONFIG_TEGRA_LP0_IN_IDLE
1268 int tegra_enter_lp0(unsigned long sleep_time)
1269 {
1270         int err = 0;
1271
1272         /* This state is managed by power domains, hence no voice call expected if
1273          * we are entering this state */
1274
1275         tegra_rtc_set_trigger(sleep_time);
1276
1277         tegra_actmon_save();
1278
1279         tegra_dma_save();
1280
1281         tegra_smmu_save();
1282
1283         err = syscore_save();
1284         if (err) {
1285                 tegra_smmu_restore();
1286                 tegra_dma_restore();
1287                 tegra_rtc_set_trigger(0);
1288                 return err;
1289         }
1290
1291         tegra_suspend_dram(TEGRA_SUSPEND_LP0, 0);
1292
1293         syscore_restore();
1294
1295         tegra_smmu_restore();
1296
1297         tegra_dma_restore();
1298
1299         tegra_actmon_restore();
1300
1301         tegra_rtc_set_trigger(0);
1302
1303         return 0;
1304 }
1305 #endif
1306
1307 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
1308 {
1309         int err = 0;
1310         u32 scratch37 = 0xDEADBEEF;
1311         u32 reg;
1312
1313 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1314         u32 enter_state = 0;
1315 #endif
1316         bool tegra_suspend_vde_powergated = false;
1317
1318         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
1319                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
1320                 err = -ENXIO;
1321                 goto fail;
1322         }
1323
1324 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1325         update_pmc_registers(tegra_lp1bb_emc_min_rate_get());
1326 #endif
1327
1328         if (tegra_is_voice_call_active()) {
1329                 /* backup the current value of scratch37 */
1330                 scratch37 = readl(pmc + PMC_SCRATCH37);
1331
1332                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
1333                 reg = TEGRA_POWER_LP1_AUDIO;
1334                 pmc_32kwritel(reg, PMC_SCRATCH37);
1335         }
1336
1337         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
1338                 pr_info("LP0 not used due to unsupported wakeup events\n");
1339                 mode = TEGRA_SUSPEND_LP1;
1340         }
1341
1342         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
1343                 tegra_suspend_check_pwr_stats();
1344
1345         /* turn off VDE partition in LP1 */
1346         if (mode == TEGRA_SUSPEND_LP1 &&
1347                 tegra_powergate_is_powered(TEGRA_POWERGATE_VDEC)) {
1348                 pr_info("turning off partition %s in LP1\n",
1349                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
1350                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, true);
1351                 tegra_suspend_vde_powergated = true;
1352         }
1353
1354         tegra_common_suspend();
1355
1356         tegra_pm_set(mode);
1357
1358         if (pdata && pdata->board_suspend)
1359                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
1360
1361         local_fiq_disable();
1362
1363 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1364         tegra_smp_save_power_mask();
1365 #endif
1366
1367         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
1368
1369         if (mode == TEGRA_SUSPEND_LP0) {
1370 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
1371                 reg = readl(pmc + PMC_SCRATCH4);
1372                 if (is_lp_cluster())
1373                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
1374                 else
1375                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
1376                 pmc_32kwritel(reg, PMC_SCRATCH4);
1377 #endif
1378                 tegra_tsc_suspend();
1379                 tegra_lp0_suspend_mc();
1380                 tegra_cpu_reset_handler_save();
1381                 tegra_tsc_wait_for_suspend();
1382                 if (!tegra_cpu_is_asim())
1383                         tegra_smp_clear_power_mask();
1384         }
1385
1386 #if !defined(CONFIG_ARCH_TEGRA_14x_SOC)
1387         if (mode == TEGRA_SUSPEND_LP1)
1388 #endif
1389                 *iram_cpu_lp1_mask = 1;
1390
1391         suspend_cpu_complex(flags);
1392
1393 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
1394         /* In case of LP0/1, program external power gating accordinly */
1395         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
1396                 reg = readl(FLOW_CTRL_CPU_CSR(0));
1397                 if (is_lp_cluster())
1398                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
1399                 else
1400                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
1401                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
1402         }
1403 #endif
1404
1405         flush_cache_all();
1406         outer_disable();
1407
1408         if (mode == TEGRA_SUSPEND_LP2)
1409                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
1410         else
1411                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
1412
1413         resume_entry_time = 0;
1414         if (mode != TEGRA_SUSPEND_LP0)
1415                 resume_entry_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
1416
1417         tegra_init_cache(true);
1418
1419 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1420         reg = readl(pmc + PMC_LP_STATE_SCRATCH_REG);
1421         enter_state = (reg >> PMC_LP_STATE_BIT_OFFSET) & PMC_LP_STATE_BIT_MASK;
1422         /* If we actually had entered in either LP1 or LP1BB,
1423          * restore power mask and disable mem_req interrupt PMC
1424          */
1425         if (enter_state) {
1426                 pr_debug("Exited state is LP1/LP1BB\n");
1427                 tegra_disable_lp1bb_interrupt();
1428                 tegra_smp_restore_power_mask();
1429         }
1430 #endif
1431
1432 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
1433 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
1434         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
1435         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
1436 #endif
1437
1438         if (mode == TEGRA_SUSPEND_LP0) {
1439                 trace_secureos_init(tegra_resume_entry_time,
1440                         NVSEC_SUSPEND_EXIT_DONE);
1441         }
1442 #endif
1443
1444         if (mode == TEGRA_SUSPEND_LP0) {
1445
1446                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
1447                 * Otherwise it is creating issue in cluster switch after LP0 *
1448 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1449                 reg = readl(pmc+PMC_CTRL);
1450                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
1451                 pmc_32kwritel(reg, PMC_CTRL);
1452 #endif
1453                 */
1454
1455                 tegra_tsc_resume();
1456                 tegra_cpu_reset_handler_restore();
1457                 tegra_lp0_resume_mc();
1458                 tegra_tsc_wait_for_resume();
1459         }
1460
1461 #if !defined(CONFIG_ARCH_TEGRA_14x_SOC)
1462         if (mode == TEGRA_SUSPEND_LP1)
1463 #endif
1464                 *iram_cpu_lp1_mask = 0;
1465
1466         /* if scratch37 was clobbered during LP1, restore it */
1467         if (scratch37 != 0xDEADBEEF)
1468                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1469
1470         restore_cpu_complex(flags);
1471
1472         /* for platforms where the core & CPU power requests are
1473          * combined as a single request to the PMU, transition out
1474          * of LP0 state by temporarily enabling both requests
1475          */
1476         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1477                 reg = readl(pmc + PMC_CTRL);
1478                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1479                 pmc_32kwritel(reg, PMC_CTRL);
1480                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1481                 pmc_32kwritel(reg, PMC_CTRL);
1482         }
1483
1484         if (pdata && pdata->board_resume)
1485                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1486
1487         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1488
1489         local_fiq_enable();
1490
1491         tegra_common_resume();
1492
1493         /* turn on VDE partition in LP1 */
1494         if (mode == TEGRA_SUSPEND_LP1 && tegra_suspend_vde_powergated) {
1495                 pr_info("turning on partition %s in LP1\n",
1496                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
1497                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, false);
1498         }
1499
1500 fail:
1501         return err;
1502 }
1503
1504 /*
1505  * Function pointers to optional board specific function
1506  */
1507 void (*tegra_deep_sleep)(int);
1508 EXPORT_SYMBOL(tegra_deep_sleep);
1509
1510 static int tegra_suspend_prepare(void)
1511 {
1512         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1513                 tegra_deep_sleep(1);
1514         return 0;
1515 }
1516
1517 static void tegra_suspend_finish(void)
1518 {
1519         if (pdata && pdata->cpu_resume_boost) {
1520                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1521                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1522                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1523         }
1524
1525         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1526                 tegra_deep_sleep(0);
1527 }
1528
1529 static const struct platform_suspend_ops tegra_suspend_ops = {
1530         .valid          = suspend_valid_only_mem,
1531         .prepare        = tegra_suspend_prepare,
1532         .finish         = tegra_suspend_finish,
1533         .prepare_late   = tegra_suspend_prepare_late,
1534         .wake           = tegra_suspend_wake,
1535         .enter          = tegra_suspend_enter,
1536 };
1537
1538 static ssize_t suspend_mode_show(struct kobject *kobj,
1539                                         struct kobj_attribute *attr, char *buf)
1540 {
1541         char *start = buf;
1542         char *end = buf + PAGE_SIZE;
1543
1544         start += scnprintf(start, end - start, "%s ", \
1545                                 tegra_suspend_name[current_suspend_mode]);
1546         start += scnprintf(start, end - start, "\n");
1547
1548         return start - buf;
1549 }
1550
1551 static ssize_t suspend_mode_store(struct kobject *kobj,
1552                                         struct kobj_attribute *attr,
1553                                         const char *buf, size_t n)
1554 {
1555         int len;
1556         const char *name_ptr;
1557         enum tegra_suspend_mode new_mode;
1558
1559         name_ptr = buf;
1560         while (*name_ptr && !isspace(*name_ptr))
1561                 name_ptr++;
1562         len = name_ptr - buf;
1563         if (!len)
1564                 goto bad_name;
1565         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1566         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1567                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1568                 pr_info("Illegal tegra suspend state: %s\n", buf);
1569                 goto bad_name;
1570         }
1571
1572         for (new_mode = TEGRA_SUSPEND_NONE; \
1573                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1574                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1575                         current_suspend_mode = new_mode;
1576                         break;
1577                 }
1578         }
1579
1580 bad_name:
1581         return n;
1582 }
1583
1584 static struct kobj_attribute suspend_mode_attribute =
1585         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1586
1587 static ssize_t suspend_resume_time_show(struct kobject *kobj,
1588                                         struct kobj_attribute *attr,
1589                                         char *buf)
1590 {
1591         return sprintf(buf, "%ums\n", ((u32)resume_time / 1000));
1592 }
1593
1594 static struct kobj_attribute suspend_resume_time_attribute =
1595         __ATTR(resume_time, 0444, suspend_resume_time_show, 0);
1596
1597 static ssize_t suspend_time_show(struct kobject *kobj,
1598                                         struct kobj_attribute *attr,
1599                                         char *buf)
1600 {
1601         return sprintf(buf, "%ums\n", ((u32)suspend_time / 1000));
1602 }
1603
1604 static struct kobj_attribute suspend_time_attribute =
1605         __ATTR(suspend_time, 0444, suspend_time_show, 0);
1606
1607 static struct kobject *suspend_kobj;
1608
1609 static int tegra_pm_enter_suspend(void)
1610 {
1611         suspend_cpu_dfll_mode(0);
1612         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1613                 tegra_lp0_cpu_mode(true);
1614         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1615         return 0;
1616 }
1617
1618 static void tegra_pm_enter_resume(void)
1619 {
1620         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1621         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1622                 tegra_lp0_cpu_mode(false);
1623         resume_cpu_dfll_mode(0);
1624 }
1625
1626 static void tegra_pm_enter_shutdown(void)
1627 {
1628         suspend_cpu_dfll_mode(0);
1629         pr_info("Shutting down tegra ...\n");
1630 }
1631
1632 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1633         .suspend = tegra_pm_enter_suspend,
1634         .resume = tegra_pm_enter_resume,
1635         .save = tegra_pm_enter_suspend,
1636         .restore = tegra_pm_enter_resume,
1637         .shutdown = tegra_pm_enter_shutdown,
1638 };
1639
1640 static __init int tegra_pm_enter_syscore_init(void)
1641 {
1642         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1643         return 0;
1644 }
1645 subsys_initcall(tegra_pm_enter_syscore_init);
1646 #endif
1647
1648 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1649 {
1650         u32 reg;
1651         u32 mode;
1652         struct pmc_pm_data *pm_dat;
1653         bool is_board_pdata = true;
1654
1655 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1656         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1657         BUG_ON(IS_ERR(tegra_dfll));
1658 #endif
1659         tegra_pclk = clk_get_sys(NULL, "pclk");
1660         BUG_ON(IS_ERR(tegra_pclk));
1661         tegra_clk_m = clk_get_sys(NULL, "clk_m");
1662         BUG_ON(IS_ERR(tegra_clk_m));
1663
1664         /* create the pdata from DT information */
1665         pm_dat = tegra_get_pm_data();
1666         if (pm_dat) {
1667                 pr_err("PMC dt information non-NULL %s\n", __func__);
1668                 is_board_pdata = false;
1669                 pdata = kzalloc(sizeof(struct tegra_suspend_platform_data),
1670                         GFP_KERNEL);
1671                 if (pm_dat->combined_req != plat->combined_req) {
1672                         pr_err("PMC DT attribute combined_req=%d, board value=%d\n",
1673                                 pm_dat->combined_req, plat->combined_req);
1674                         pdata->combined_req = plat->combined_req;
1675                 } else {
1676                         pdata->combined_req = pm_dat->combined_req;
1677                 }
1678                 if (pm_dat->sysclkreq_high != plat->sysclkreq_high) {
1679                         pr_err("PMC DT attribute sysclkreq_high=%d, board value=%d\n",
1680                                 pm_dat->sysclkreq_high, plat->sysclkreq_high);
1681                         pdata->sysclkreq_high = plat->sysclkreq_high;
1682                 } else {
1683                         pdata->sysclkreq_high = pm_dat->sysclkreq_high;
1684                 }
1685                 if (pm_dat->corereq_high != plat->corereq_high) {
1686                         pr_err("PMC DT attribute corereq_high=%d, board value=%d\n",
1687                                 pm_dat->corereq_high, plat->corereq_high);
1688                         pdata->corereq_high = plat->corereq_high;
1689                 } else {
1690                         pdata->corereq_high = pm_dat->corereq_high;
1691                 }
1692                 if (pm_dat->cpu_off_time != plat->cpu_off_timer) {
1693                         pr_err("PMC DT attribute cpu_off_timer=%d, board value=%ld\n",
1694                                 pm_dat->cpu_off_time, plat->cpu_off_timer);
1695                         pdata->cpu_off_timer = plat->cpu_off_timer;
1696                 } else {
1697                         pdata->cpu_off_timer = pm_dat->cpu_off_time;
1698                 }
1699                 if (pm_dat->cpu_good_time != plat->cpu_timer) {
1700                         pr_err("PMC DT attribute cpu_timer=%d, board value=%ld\n",
1701                                 pm_dat->cpu_good_time, plat->cpu_timer);
1702                         pdata->cpu_timer = plat->cpu_timer;
1703                 } else {
1704                         pdata->cpu_timer = pm_dat->cpu_good_time;
1705                 }
1706                 if (pm_dat->suspend_mode != plat->suspend_mode) {
1707                         pr_err("PMC DT attribute suspend_mode=%d, board value=%d\n",
1708                                 pm_dat->suspend_mode, plat->suspend_mode);
1709                         pdata->suspend_mode = plat->suspend_mode;
1710                 } else {
1711                         pdata->suspend_mode = pm_dat->suspend_mode;
1712                 }
1713                 /* FIXME: pmc_pm_data fields to be reused
1714                  *      core_osc_time, core_pmu_time, core_off_time
1715                  *      units of above fields is uSec while
1716                  *      platform data values are in ticks
1717                  */
1718                 /* FIXME: pmc_pm_data unused by downstream code
1719                  *      cpu_pwr_good_en, lp0_vec_size, lp0_vec_phy_addr
1720                  */
1721                 /* FIXME: add missing DT bindings taken from platform data */
1722                 pdata->core_timer = plat->core_timer;
1723                 pdata->core_off_timer = plat->core_off_timer;
1724                 pdata->board_suspend = plat->board_suspend;
1725                 pdata->board_resume = plat->board_resume;
1726                 pdata->sysclkreq_gpio = plat->sysclkreq_gpio;
1727                 pdata->cpu_lp2_min_residency = plat->cpu_lp2_min_residency;
1728                 pdata->cpu_resume_boost = plat->cpu_resume_boost;
1729 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
1730                 pdata->lp1_lowvolt_support = plat->lp1_lowvolt_support;
1731                 pdata->i2c_base_addr = plat->i2c_base_addr;
1732                 pdata->pmuslave_addr = plat->pmuslave_addr;
1733                 pdata->core_reg_addr = plat->core_reg_addr;
1734                 pdata->lp1_core_volt_low_cold = plat->lp1_core_volt_low_cold;
1735                 pdata->lp1_core_volt_low = plat->lp1_core_volt_low;
1736                 pdata->lp1_core_volt_high = plat->lp1_core_volt_high;
1737 #endif
1738 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
1739                 pdata->min_residency_vmin_fmin = plat->min_residency_vmin_fmin;
1740                 pdata->min_residency_ncpu_slow = plat->min_residency_ncpu_slow;
1741                 pdata->min_residency_ncpu_fast = plat->min_residency_ncpu_fast;
1742                 pdata->min_residency_crail = plat->min_residency_crail;
1743                 pdata->crail_up_early = plat->crail_up_early;
1744 #endif
1745                 pdata->min_residency_mclk_stop = plat->min_residency_mclk_stop;
1746                 pdata->usb_vbus_internal_wake = plat->usb_vbus_internal_wake;
1747                 pdata->usb_id_internal_wake = plat->usb_id_internal_wake;
1748                 pdata->suspend_dfll_bypass = plat->suspend_dfll_bypass;
1749                 pdata->resume_dfll_bypass = plat->resume_dfll_bypass;
1750         } else {
1751                 pr_err("PMC board data used in %s\n", __func__);
1752                 pdata = plat;
1753         }
1754         (void)reg;
1755         (void)mode;
1756
1757         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1758                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1759
1760 #ifndef CONFIG_PM_SLEEP
1761         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1762                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1763                            "disabling suspend\n", __func__);
1764                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1765         }
1766 #else
1767         if (create_suspend_pgtable() < 0) {
1768                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1769                                 __func__);
1770                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1771                 goto fail;
1772         }
1773
1774         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1775             (tegra_revision == TEGRA_REVISION_A01) &&
1776             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1777                 /* Tegra 3 A01 supports only LP1 */
1778                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1779                            "-- disabling LP0\n", __func__);
1780                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1781         }
1782         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1783                 tegra_lp0_vec_relocate) {
1784                 unsigned char *reloc_lp0;
1785                 unsigned long tmp;
1786                 void __iomem *orig;
1787                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1788                                         GFP_KERNEL);
1789                 WARN_ON(!reloc_lp0);
1790                 if (!reloc_lp0) {
1791                         pr_err("%s: Failed to allocate reloc_lp0\n",
1792                                 __func__);
1793                         goto out;
1794                 }
1795
1796                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1797                 WARN_ON(!orig);
1798                 if (!orig) {
1799                         pr_err("%s: Failed to map tegra_lp0_vec_start %08x\n",
1800                                 __func__, tegra_lp0_vec_start);
1801                         kfree(reloc_lp0);
1802                         goto out;
1803                 }
1804
1805                 tmp = (unsigned long) reloc_lp0;
1806                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1807                 reloc_lp0 = (unsigned char *)tmp;
1808                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1809                 iounmap(orig);
1810                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1811         }
1812
1813 out:
1814         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1815                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1816                            "provided by bootlader -- disabling LP0\n",
1817                            __func__);
1818                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1819         }
1820
1821         iram_save_size = tegra_iram_end() - tegra_iram_start();
1822
1823         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1824         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1825                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1826                        "-- LP0/LP1 unavailable\n", __func__);
1827                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1828         }
1829
1830 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
1831         if (pdata->lp1_lowvolt_support) {
1832                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1833                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1834                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1835
1836                 lp1_core_lowvolt = 0;
1837                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1838                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1839
1840                 lp1_core_highvolt = 0;
1841                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1842                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1843         }
1844 #endif
1845         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1846         /* Initialize scratch registers used for CPU LP2 synchronization */
1847         writel(0, pmc + PMC_SCRATCH37);
1848         writel(0, pmc + PMC_SCRATCH38);
1849 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1850         writel(0, pmc + PMC_SCRATCH39);
1851 #endif
1852         writel(0, pmc + PMC_SCRATCH41);
1853
1854         /* Always enable CPU power request; just normal polarity is supported */
1855         reg = readl(pmc + PMC_CTRL);
1856         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1857         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1858         pmc_32kwritel(reg, PMC_CTRL);
1859
1860         /* Configure core power request and system clock control if LP0
1861            is supported */
1862         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1863         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1864
1865         reg = readl(pmc + PMC_CTRL);
1866
1867         if (!pdata->sysclkreq_high)
1868                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1869         else
1870                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1871
1872         if (!pdata->corereq_high)
1873                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1874         else
1875                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1876
1877         /* configure output inverters while the request is tristated */
1878         pmc_32kwritel(reg, PMC_CTRL);
1879
1880         /* now enable requests */
1881         reg |= TEGRA_POWER_SYSCLK_OE;
1882         if (!pdata->combined_req)
1883                 reg |= TEGRA_POWER_PWRREQ_OE;
1884         pmc_32kwritel(reg, PMC_CTRL);
1885
1886         if (pdata->sysclkreq_gpio) {
1887                 reg = readl(pmc + PMC_DPAD_ORIDE);
1888                 reg &= ~TEGRA_DPAD_ORIDE_SYS_CLK_REQ;
1889                 pmc_32kwritel(reg, PMC_DPAD_ORIDE);
1890         }
1891
1892         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1893                 tegra_lp0_suspend_init();
1894
1895         suspend_set_ops(&tegra_suspend_ops);
1896
1897         /* Create /sys/power/suspend/type */
1898         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1899         if (suspend_kobj) {
1900                 if (sysfs_create_file(suspend_kobj, \
1901                                                 &suspend_mode_attribute.attr))
1902                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1903                                                                 __func__);
1904                 if (sysfs_create_file(suspend_kobj, \
1905                                         &suspend_resume_time_attribute.attr))
1906                         pr_err("%s: sysfs_create_file resume_time failed!\n",
1907                                                                 __func__);
1908                 if (sysfs_create_file(suspend_kobj, \
1909                                         &suspend_time_attribute.attr))
1910                         pr_err("%s: sysfs_create_file suspend_time failed!\n",
1911                                                                 __func__);
1912         }
1913
1914         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1915         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1916         iram_mc_clk_mask = tegra_mc_clk_mask;
1917
1918         /* clear io dpd settings before kernel */
1919         tegra_bl_io_dpd_cleanup();
1920
1921 fail:
1922 #endif
1923         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1924                 tegra_pd_in_idle(false);
1925
1926         current_suspend_mode = plat->suspend_mode;
1927 }
1928
1929 void tegra_lp1bb_suspend_emc_rate(unsigned long emc_min, unsigned long emc_max)
1930 {
1931         pdata->lp1bb_emc_rate_min = emc_min;
1932         pdata->lp1bb_emc_rate_max = emc_max;
1933 }
1934
1935 void tegra_lp1bb_suspend_mv_set(int mv)
1936 {
1937         if (WARN_ON_ONCE(!pdata))
1938                 return;
1939
1940         pdata->lp1bb_core_volt_min = mv;
1941 }
1942
1943 unsigned long tegra_lp1bb_emc_min_rate_get(void)
1944 {
1945         if (WARN_ON_ONCE(!pdata) || !pdata->lp1bb_emc_rate_min)
1946                 return 204000000;
1947
1948         return pdata->lp1bb_emc_rate_min;
1949 }
1950
1951 unsigned long debug_uart_port_base = 0;
1952 EXPORT_SYMBOL(debug_uart_port_base);
1953
1954 static int tegra_debug_uart_suspend(void)
1955 {
1956         void __iomem *uart;
1957         u32 lcr;
1958
1959         if (!debug_uart_port_base)
1960                 return 0;
1961
1962         uart = IO_ADDRESS(debug_uart_port_base);
1963
1964         lcr = readb(uart + UART_LCR * 4);
1965
1966         tegra_sctx.uart[0] = lcr;
1967         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1968
1969         /* DLAB = 0 */
1970         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1971
1972         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1973
1974         /* DLAB = 1 */
1975         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1976
1977         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1978         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1979
1980         writeb(lcr, uart + UART_LCR * 4);
1981
1982         return 0;
1983 }
1984
1985 static void tegra_debug_uart_resume(void)
1986 {
1987         void __iomem *uart;
1988         u32 lcr;
1989
1990         if (!debug_uart_port_base)
1991                 return;
1992
1993         uart = IO_ADDRESS(debug_uart_port_base);
1994
1995         lcr = tegra_sctx.uart[0];
1996
1997         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1998
1999         /* DLAB = 0 */
2000         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
2001
2002         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
2003                         uart + UART_FCR * 4);
2004
2005         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
2006
2007         /* DLAB = 1 */
2008         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
2009
2010         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
2011         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
2012
2013         writeb(lcr, uart + UART_LCR * 4);
2014 }
2015
2016 static struct syscore_ops tegra_debug_uart_syscore_ops = {
2017         .suspend = tegra_debug_uart_suspend,
2018         .resume = tegra_debug_uart_resume,
2019         .save = tegra_debug_uart_suspend,
2020         .restore = tegra_debug_uart_resume,
2021 };
2022
2023 struct clk *debug_uart_clk = NULL;
2024 EXPORT_SYMBOL(debug_uart_clk);
2025
2026 void tegra_console_uart_suspend(void)
2027 {
2028         if (console_suspend_enabled && debug_uart_clk)
2029                 tegra_clk_disable_unprepare(debug_uart_clk);
2030 }
2031
2032 void tegra_console_uart_resume(void)
2033 {
2034         if (console_suspend_enabled && debug_uart_clk)
2035                 tegra_clk_prepare_enable(debug_uart_clk);
2036 }
2037
2038 static int tegra_debug_uart_syscore_init(void)
2039 {
2040         register_syscore_ops(&tegra_debug_uart_syscore_ops);
2041         return 0;
2042 }
2043 arch_initcall(tegra_debug_uart_syscore_init);
2044
2045 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
2046 static inline bool pmc_write_check(int index, int bit_position)
2047 {
2048         if (pmc_write_bitmap[index] & (1 << bit_position))
2049                 return true;
2050         else
2051                 return false;
2052 }
2053
2054 static void update_pmc_registers(unsigned long rate)
2055 {
2056         u32 i, j;
2057         int instance = 1;
2058
2059         /* FIXME: convert rate to instance */
2060
2061         /* Based on index, we select that block of scratches */
2062         u32 base2 = (tegra_wb0_params_address + (instance - 1) *
2063                 tegra_wb0_params_block_size);
2064         void __iomem *base = ioremap(base2, tegra_wb0_params_block_size);
2065
2066 #define copy_dram_to_pmc(index, bit)    \
2067         pmc_32kwritel(readl(base + PMC_REGISTER_OFFSET(index, bit)), \
2068                 PMC_REGISTER_OFFSET(index, bit) + PMC_SCRATCH0)
2069
2070
2071         /* Iterate through the bitmap, and copy those registers
2072          * which are marked in the bitmap
2073          */
2074         for (i = 0, j = 0; j < ARRAY_SIZE(pmc_write_bitmap);) {
2075                 if (pmc_write_bitmap[j] == 0) {
2076                         j++;
2077                         i = 0;
2078                         continue;
2079                 }
2080
2081                 if (pmc_write_check(j, i))
2082                         copy_dram_to_pmc(j, i);
2083
2084                 if (++i > (sizeof(pmc_write_bitmap[0]) * 8)) {
2085                         i = 0;
2086                         j++;
2087                 }
2088         }
2089
2090 #undef copy_dram_to_pmc
2091         iounmap(base);
2092 }
2093 #endif
2094
2095 #ifdef CONFIG_ARM_ARCH_TIMER
2096
2097 static u32 tsc_suspend_start;
2098 static u32 tsc_resume_start;
2099
2100 #define pmc_writel(value, reg) \
2101                 writel(value, pmc + (reg))
2102 #define pmc_readl(reg) \
2103                 readl(pmc + (reg))
2104
2105 #define PMC_DPD_ENABLE                  0x24
2106 #define PMC_DPD_ENABLE_TSC_MULT_ENABLE  (1 << 1)
2107 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
2108 #define PMC_DPD_ENABLE_ON               (1 << 0)
2109 #endif
2110
2111 #define PMC_TSC_MULT                    0x2b4
2112 #define PMC_TSC_MULT_FREQ_STS           (1 << 16)
2113
2114 #define TSC_TIMEOUT_US                  32
2115
2116 void tegra_tsc_suspend(void)
2117 {
2118         if (arch_timer_initialized) {
2119                 u32 reg = pmc_readl(PMC_DPD_ENABLE);
2120                 BUG_ON(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE);
2121                 reg |= PMC_DPD_ENABLE_TSC_MULT_ENABLE;
2122                 pmc_writel(reg, PMC_DPD_ENABLE);
2123                 tsc_suspend_start = timer_readl(TIMERUS_CNTR_1US);
2124         }
2125 }
2126
2127 void tegra_tsc_resume(void)
2128 {
2129         if (arch_timer_initialized) {
2130                 u32 reg = pmc_readl(PMC_DPD_ENABLE);
2131                 BUG_ON(!(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE));
2132                 reg &= ~PMC_DPD_ENABLE_TSC_MULT_ENABLE;
2133 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
2134                 /*
2135                  * FIXME: T12x SW WAR -
2136                  * Resume ensures DPD_ENABLE is 0 when writing
2137                  * TSC_MULT_ENABLE, else PMC wake status gets reset
2138                  */
2139                 reg &= ~PMC_DPD_ENABLE_ON;
2140 #endif
2141                 pmc_writel(reg, PMC_DPD_ENABLE);
2142                 tsc_resume_start = timer_readl(TIMERUS_CNTR_1US);
2143         }
2144 }
2145
2146 void tegra_tsc_wait_for_suspend(void)
2147 {
2148         if (arch_timer_initialized) {
2149                 while ((timer_readl(TIMERUS_CNTR_1US) - tsc_suspend_start) <
2150                         TSC_TIMEOUT_US) {
2151                         if (pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS)
2152                                 break;
2153                         cpu_relax();
2154                 }
2155         }
2156 }
2157
2158 void tegra_tsc_wait_for_resume(void)
2159 {
2160         if (arch_timer_initialized) {
2161                 while ((timer_readl(TIMERUS_CNTR_1US) - tsc_resume_start) <
2162                         TSC_TIMEOUT_US) {
2163                         if (!(pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS))
2164                                 break;
2165                         cpu_relax();
2166                 }
2167         }
2168 }
2169 #endif
2170
2171 #if defined(CONFIG_DEBUG_FS) && INSTRUMENT_CLUSTER_SWITCH
2172
2173 static void cluster_switch_stats_show(
2174         struct seq_file *s, struct tegra_cluster_switch_time_stats *stats)
2175 {
2176         seq_printf(s, "%u-samples average:           %lu\n",
2177                    CLUSTER_SWITCH_AVG_SAMPLES,
2178                    stats->avg >> CLUSTER_SWITCH_TIME_AVG_SHIFT);
2179         seq_printf(s, "exponential average:          %lu\n",
2180                    stats->exp_avg >> CLUSTER_SWITCH_TIME_AVG_SHIFT);
2181         seq_printf(s, "maximum since boot:           %lu\n\n", stats->max);
2182 }
2183
2184
2185 static int tegra_cluster_switch_stats_show(struct seq_file *s, void *data)
2186 {
2187         seq_printf(s, "G=>LP cluster switch timing:  (us)\n");
2188         cluster_switch_stats_show(s, &g2lp_stats);
2189         seq_printf(s, "LP=>G cluster switch timing:  (us)\n");
2190         cluster_switch_stats_show(s, &lp2g_stats);
2191         return 0;
2192 }
2193
2194 static int tegra_cluster_switch_stats_open(
2195         struct inode *inode, struct file *file)
2196 {
2197         return single_open(file, tegra_cluster_switch_stats_show,
2198                            inode->i_private);
2199 }
2200
2201 static const struct file_operations tegra_cluster_switch_stats_ops = {
2202         .open           = tegra_cluster_switch_stats_open,
2203         .read           = seq_read,
2204         .llseek         = seq_lseek,
2205         .release        = single_release,
2206 };
2207
2208 static int __init tegra_pm_core_debug_init(void)
2209 {
2210         struct dentry *dir, *d;
2211
2212         dir = debugfs_create_dir("tegra_pm_core", NULL);
2213         if (!dir)
2214                 return -ENOMEM;
2215
2216         d = debugfs_create_file("cluster_switch_stats", S_IRUGO, dir, NULL,
2217                 &tegra_cluster_switch_stats_ops);
2218         if (!d)
2219                 return -ENOMEM;
2220
2221         return 0;
2222 }
2223
2224 late_initcall(tegra_pm_core_debug_init);
2225 #endif
2226
2227 #ifdef CONFIG_DEBUG_RODATA
2228 void set_platform_text_rw(void)
2229 {
2230 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
2231         set_memory_rw((unsigned long)tegra_generic_smc, 1);
2232 #endif
2233 }
2234 #endif