ARM: tegra12: set CPU rate to 2.2GHz for sku 0x87
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2014, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/cpumask.h>
32 #include <linux/delay.h>
33 #include <linux/cpu_pm.h>
34 #include <linux/err.h>
35 #include <linux/debugfs.h>
36 #include <linux/delay.h>
37 #include <linux/suspend.h>
38 #include <linux/slab.h>
39 #include <linux/serial_reg.h>
40 #include <linux/seq_file.h>
41 #include <linux/uaccess.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/cpu_pm.h>
44 #include <linux/clk/tegra.h>
45 #include <linux/export.h>
46 #include <linux/vmalloc.h>
47 #include <linux/memblock.h>
48 #include <linux/console.h>
49 #include <linux/tegra_audio.h>
50 #include <linux/clk/tegra.h>
51 #include <linux/tegra-powergate.h>
52 #include <linux/tegra-soc.h>
53 #include <linux/tegra-timer.h>
54 #include <linux/tegra-cpuidle.h>
55 #include <linux/irqchip/tegra.h>
56 #include <linux/tegra-pm.h>
57
58 #include <trace/events/power.h>
59 #include <trace/events/nvsecurity.h>
60
61 #include <asm/cacheflush.h>
62 #include <asm/idmap.h>
63 #include <asm/localtimer.h>
64 #include <asm/pgalloc.h>
65 #include <asm/pgtable.h>
66 #include <asm/tlbflush.h>
67 #include <asm/suspend.h>
68 #include <asm/smp_plat.h>
69 #include <asm/rodata.h>
70
71 #include <mach/irqs.h>
72 #include <mach/tegra_smmu.h>
73 #include <mach/pm_domains.h>
74
75 #include "board.h"
76 #include "clock.h"
77 #include "common.h"
78 #include "iomap.h"
79 #include "pm.h"
80 #include "reset.h"
81 #include "pmc.h"
82 #include "sleep.h"
83 #include "dvfs.h"
84 #include "cpu-tegra.h"
85 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
86 #include "tegra14_scratch.h"
87 #endif
88
89 struct suspend_context {
90         /*
91          * The next 7 values are referenced by offset in __restart_plls
92          * in headsmp-t2.S, and should not be moved
93          */
94         u32 pllx_misc;
95         u32 pllx_base;
96         u32 pllp_misc;
97         u32 pllp_base;
98         u32 pllp_outa;
99         u32 pllp_outb;
100         u32 pll_timeout;
101
102         u32 cpu_burst;
103         u32 clk_csite_src;
104         u32 cclk_divider;
105
106         u32 mc[3];
107         u8 uart[5];
108
109         struct tegra_twd_context twd;
110 };
111 #define PMC_CTRL                0x0
112
113 #ifdef CONFIG_PM_SLEEP
114 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
115 static pgd_t *tegra_pgd;
116 static DEFINE_SPINLOCK(tegra_lp2_lock);
117 static cpumask_t tegra_in_lp2;
118 static cpumask_t *iram_cpu_lp2_mask;
119 static unsigned long *iram_cpu_lp1_mask;
120 static unsigned long *iram_mc_clk_mask;
121 static u8 *iram_save;
122 static unsigned long iram_save_size;
123 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
124 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
125 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
126 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
127 static void __iomem *tert_ictlr = IO_ADDRESS(TEGRA_TERTIARY_ICTLR_BASE);
128 #endif
129 static void __iomem *tmrus_reg_base = IO_ADDRESS(TEGRA_TMR1_BASE);
130 static int tegra_last_pclk;
131 static u64 resume_time;
132 static u64 resume_entry_time;
133 static u64 suspend_time;
134 static u64 suspend_entry_time;
135 #endif
136
137 static RAW_NOTIFIER_HEAD(tegra_pm_chain_head);
138
139 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
140 static void update_pmc_registers(unsigned long rate);
141 #endif
142
143 struct suspend_context tegra_sctx;
144 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
145 extern struct device *get_se_device(void);
146 extern int se_suspend(struct device *dev, bool pooling);
147 extern struct device *get_smmu_device(void);
148 extern int tegra_smmu_resume(struct device *dev);
149 extern int tegra_smmu_suspend(struct device *dev);
150 #endif
151
152 bool tegra_is_dpd_mode;
153
154 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
155 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
156 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
157 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
158 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
159 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
160 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
161 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
162 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
163
164 #define TEGRA_DPAD_ORIDE_SYS_CLK_REQ    (1 << 21)
165
166 #define PMC_CTRL                0x0
167 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
168 #define PMC_WAKE_MASK           0xc
169 #define PMC_WAKE_LEVEL          0x10
170 #define PMC_DPAD_ORIDE          0x1C
171 #define PMC_WAKE_DELAY          0xe0
172 #define PMC_DPD_SAMPLE          0x20
173 #if defined(CONFIG_ARCH_TEGRA_14x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
174 #define PMC_DPD_ENABLE          0x24
175 #endif
176 #define PMC_IO_DPD_REQ          0x1B8
177 #define PMC_IO_DPD2_REQ         0x1C0
178
179
180 #define PMC_WAKE_STATUS         0x14
181 #define PMC_SW_WAKE_STATUS      0x18
182 #define PMC_COREPWRGOOD_TIMER   0x3c
183 #define PMC_CPUPWRGOOD_TIMER    0xc8
184 #define PMC_CPUPWROFF_TIMER     0xcc
185 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
186
187 #define PMC_PWRGATE_TOGGLE      0x30
188 #define PWRGATE_TOGGLE_START    (1 << 8)
189 #define UN_PWRGATE_CPU          \
190         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
191
192 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
193 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
194 #endif
195
196 #define CLK_RESET_CCLK_BURST    0x20
197 #define CLK_RESET_CCLK_DIVIDER  0x24
198 #define CLK_RESET_PLLC_BASE     0x80
199 #define CLK_RESET_PLLM_BASE     0x90
200 #define CLK_RESET_PLLX_BASE     0xe0
201 #define CLK_RESET_PLLX_MISC     0xe4
202 #define CLK_RESET_PLLP_BASE     0xa0
203 #define CLK_RESET_PLLP_OUTA     0xa4
204 #define CLK_RESET_PLLP_OUTB     0xa8
205 #define CLK_RESET_PLLP_MISC     0xac
206
207 #define CLK_RESET_SOURCE_CSITE  0x1d4
208
209 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
210 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
211 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
212 #define CLK_RESET_CCLK_IDLE_POLICY         1
213 #define CLK_RESET_CCLK_RUN_POLICY          2
214 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
215 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
216
217 #define EMC_MRW_0               0x0e8
218 #define EMC_MRW_DEV_SELECTN     30
219 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
220
221 #define MC_SECURITY_START       0x6c
222 #define MC_SECURITY_SIZE        0x70
223 #define MC_SECURITY_CFG2        0x7c
224
225 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
226 static struct clk *tegra_dfll;
227 #endif
228 static struct clk *tegra_pclk;
229 static struct clk *tegra_clk_m;
230 static struct tegra_suspend_platform_data *pdata;
231 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
232
233 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
234 static unsigned long
235                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
236 struct tegra_cluster_switch_time_stats {
237         unsigned long sum;
238         unsigned long avg;
239         unsigned long exp_avg;
240         unsigned long max;
241         int cnt;
242 };
243
244 static struct tegra_cluster_switch_time_stats lp2g_stats;
245 static struct tegra_cluster_switch_time_stats g2lp_stats;
246
247 void tegra_cluster_switch_time(unsigned int flags, int id)
248 {
249         unsigned long t;
250         struct tegra_cluster_switch_time_stats *stats;
251
252         if (!(flags & TEGRA_POWER_CLUSTER_MASK) ||
253             (id >= tegra_cluster_switch_time_id_max))
254                 return;
255
256         tegra_cluster_switch_times[id] = tegra_read_usec_raw();
257         wmb();
258         if (id != tegra_cluster_switch_time_id_end)
259                 return;
260
261         stats = flags & TEGRA_POWER_CLUSTER_G ? &lp2g_stats : &g2lp_stats;
262
263         t = tegra_cluster_switch_times[tegra_cluster_switch_time_id_end] -
264                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_start];
265         if (stats->max < t)
266                 stats->max = t;
267
268         stats->sum += t;
269         stats->cnt++;
270         if (stats->cnt < CLUSTER_SWITCH_AVG_SAMPLES)
271                 return;
272
273         stats->avg = stats->sum;
274         stats->cnt = stats->sum = 0;
275         if (!stats->exp_avg) {
276                 stats->exp_avg = stats->avg;    /* 1st window sample */
277                 return;
278         }
279         stats->exp_avg = (stats->exp_avg * (CLUSTER_SWITCH_AVG_SAMPLES - 1) +
280                           stats->avg) >> CLUSTER_SWITCH_TIME_AVG_SHIFT;
281 }
282 #endif
283
284 int tegra_register_pm_notifier(struct notifier_block *nb)
285 {
286         return raw_notifier_chain_register(&tegra_pm_chain_head, nb);
287 }
288 EXPORT_SYMBOL(tegra_register_pm_notifier);
289
290 int tegra_unregister_pm_notifier(struct notifier_block *nb)
291 {
292         return raw_notifier_chain_unregister(&tegra_pm_chain_head, nb);
293 }
294 EXPORT_SYMBOL(tegra_unregister_pm_notifier);
295
296 #ifdef CONFIG_PM_SLEEP
297 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
298         [TEGRA_SUSPEND_NONE]    = "none",
299         [TEGRA_SUSPEND_LP2]     = "lp2",
300         [TEGRA_SUSPEND_LP1]     = "lp1",
301         [TEGRA_SUSPEND_LP0]     = "lp0",
302 };
303
304 void tegra_log_resume_time(void)
305 {
306         u64 resume_end_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
307
308         if (resume_entry_time > resume_end_time)
309                 resume_end_time |= 1ull<<32;
310         resume_time = resume_end_time - resume_entry_time;
311 }
312
313 void tegra_log_suspend_time(void)
314 {
315         suspend_entry_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
316 }
317
318 static void tegra_get_suspend_time(void)
319 {
320         u64 suspend_end_time;
321         suspend_end_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
322
323         if (suspend_entry_time > suspend_end_time)
324                 suspend_end_time |= 1ull<<32;
325         suspend_time = suspend_end_time - suspend_entry_time;
326 }
327
328 unsigned long tegra_cpu_power_good_time(void)
329 {
330         if (WARN_ON_ONCE(!pdata))
331                 return 5000;
332
333         return pdata->cpu_timer;
334 }
335
336 unsigned long tegra_cpu_power_off_time(void)
337 {
338         if (WARN_ON_ONCE(!pdata))
339                 return 5000;
340
341         return pdata->cpu_off_timer;
342 }
343
344 unsigned long tegra_cpu_lp2_min_residency(void)
345 {
346         if (WARN_ON_ONCE(!pdata))
347                 return 2000;
348
349         return pdata->cpu_lp2_min_residency;
350 }
351
352 #define TEGRA_MIN_RESIDENCY_MCLK_STOP   20000
353
354 unsigned long tegra_mc_clk_stop_min_residency(void)
355 {
356         return pdata && pdata->min_residency_mclk_stop
357                         ? pdata->min_residency_mclk_stop
358                         : TEGRA_MIN_RESIDENCY_MCLK_STOP;
359 }
360
361 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
362 #define TEGRA_MIN_RESIDENCY_VMIN_FMIN   2000
363 #define TEGRA_MIN_RESIDENCY_NCPU_SLOW   2000
364 #define TEGRA_MIN_RESIDENCY_NCPU_FAST   13000
365 #define TEGRA_MIN_RESIDENCY_CRAIL       20000
366
367 unsigned long tegra_min_residency_vmin_fmin(void)
368 {
369         return pdata && pdata->min_residency_vmin_fmin
370                         ? pdata->min_residency_vmin_fmin
371                         : TEGRA_MIN_RESIDENCY_VMIN_FMIN;
372 }
373
374 unsigned long tegra_min_residency_ncpu()
375 {
376         if (is_lp_cluster()) {
377                 return pdata && pdata->min_residency_ncpu_slow
378                         ? pdata->min_residency_ncpu_slow
379                         : TEGRA_MIN_RESIDENCY_NCPU_SLOW;
380         } else
381                 return pdata && pdata->min_residency_ncpu_fast
382                         ? pdata->min_residency_ncpu_fast
383                         : TEGRA_MIN_RESIDENCY_NCPU_FAST;
384 }
385
386 unsigned long tegra_min_residency_crail(void)
387 {
388         return pdata && pdata->min_residency_crail
389                         ? pdata->min_residency_crail
390                         : TEGRA_MIN_RESIDENCY_CRAIL;
391 }
392
393 bool tegra_crail_can_start_early(void)
394 {
395         return pdata && pdata->crail_up_early;
396 }
397 #endif
398
399 static void suspend_cpu_dfll_mode(unsigned int flags)
400 {
401 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
402         /* If DFLL is used as CPU clock source go to open loop mode */
403         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
404                 if (!is_lp_cluster() && tegra_dfll &&
405                     tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
406                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
407         }
408
409         /* Suspend dfll bypass (safe rail down) on LP or if DFLL is Not used */
410         if (pdata && pdata->suspend_dfll_bypass &&
411             (!tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail) || is_lp_cluster()))
412                 pdata->suspend_dfll_bypass();
413 #endif
414 }
415
416 static void resume_cpu_dfll_mode(unsigned int flags)
417 {
418 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
419         /* If DFLL is Not used and resume on G restore bypass mode */
420         if (pdata && pdata->resume_dfll_bypass && !is_lp_cluster() &&
421             tegra_is_clk_initialized(tegra_dfll) &&
422             !tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
423                 pdata->resume_dfll_bypass();
424
425         /* If DFLL is used as CPU clock source restore closed loop mode */
426         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
427                 if (!is_lp_cluster() && tegra_dfll &&
428                     tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
429                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
430         }
431 #endif
432 }
433
434 /*
435  * create_suspend_pgtable
436  *
437  * Creates a page table with identity mappings of physical memory and IRAM
438  * for use when the MMU is off, in addition to all the regular kernel mappings.
439  */
440 static __init int create_suspend_pgtable(void)
441 {
442         tegra_pgd = pgd_alloc(&init_mm);
443         if (!tegra_pgd)
444                 return -ENOMEM;
445
446         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
447         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
448                 high_memory, 0);
449         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
450                 IO_IRAM_VIRT + SECTION_SIZE, 0);
451
452 #if defined(CONFIG_ARM_LPAE)
453         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK);
454 #else
455         /* inner/outer write-back/write-allocate, sharable */
456         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
457 #endif
458
459         return 0;
460 }
461
462 /* ensures that sufficient time is passed for a register write to
463  * serialize into the 32KHz domain */
464 static void pmc_32kwritel(u32 val, unsigned long offs)
465 {
466         writel(val, pmc + offs);
467         udelay(130);
468 }
469
470 #if !defined(CONFIG_OF) || !defined(CONFIG_COMMON_CLK)
471 static void set_power_timers(unsigned long us_on, unsigned long us_off,
472                              long rate)
473 {
474         static unsigned long last_us_off = 0;
475         unsigned long long ticks;
476         unsigned long long pclk;
477
478         if (WARN_ON_ONCE(rate <= 0))
479                 pclk = 100000000;
480         else
481                 pclk = rate;
482
483         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
484                 ticks = (us_on * pclk) + 999999ull;
485                 do_div(ticks, 1000000);
486                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
487
488                 ticks = (us_off * pclk) + 999999ull;
489                 do_div(ticks, 1000000);
490                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
491                 wmb();
492         }
493         tegra_last_pclk = pclk;
494         last_us_off = us_off;
495 }
496 #endif
497
498 void tegra_limit_cpu_power_timers(unsigned long us_on, unsigned long us_off)
499 {
500         /* make sure power timers would not exceed specified limits */
501         set_power_timers(us_on, us_off, clk_get_min_rate(tegra_pclk));
502 }
503
504 void (*tegra_tear_down_cpu)(void);
505
506 /*
507  * restore_cpu_complex
508  *
509  * restores cpu clock setting, clears flow controller
510  *
511  * Always called on CPU 0.
512  */
513 static void restore_cpu_complex(u32 mode)
514 {
515         int cpu = cpu_logical_map(smp_processor_id());
516         unsigned int reg;
517 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
518         unsigned int policy;
519 #endif
520
521 /*
522  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
523  * or restored by common clock suspend/resume procedures. Hence, we don't
524  * need it here.
525  */
526 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
527         /* Is CPU complex already running on PLLX? */
528         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
529         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
530
531         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
532                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
533         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
534                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
535         else
536                 BUG();
537
538         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
539                 /* restore PLLX settings if CPU is on different PLL */
540                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
541                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
542
543                 /* wait for PLL stabilization if PLLX was enabled */
544                 if (tegra_sctx.pllx_base & (1<<30)) {
545 #if USE_PLL_LOCK_BITS
546                         /* Enable lock detector */
547                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
548                         reg |= 1<<18;
549                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
550                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
551                                  (1<<27)))
552                                 cpu_relax();
553
554                         udelay(PLL_POST_LOCK_DELAY);
555 #else
556                         udelay(300);
557 #endif
558                 }
559         }
560
561         /* Restore original burst policy setting for calls resulting from CPU
562            LP2 in idle or system suspend; keep cluster switch prolog setting
563            intact. */
564         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
565                 writel(tegra_sctx.cclk_divider, clk_rst +
566                        CLK_RESET_CCLK_DIVIDER);
567                 writel(tegra_sctx.cpu_burst, clk_rst +
568                        CLK_RESET_CCLK_BURST);
569         }
570 #endif
571         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
572
573         /* Do not power-gate CPU 0 when flow controlled */
574         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
575         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
576         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
577         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
578         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
579         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
580         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
581
582         /* If an immedidate cluster switch is being perfomed, restore the
583            local timer registers. For calls resulting from CPU LP2 in
584            idle or system suspend, the local timer was shut down and
585            timekeeping switched over to the global system timer. In this
586            case keep local timer disabled, and restore only periodic load. */
587 #ifdef CONFIG_HAVE_ARM_TWD
588         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
589                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
590                 tegra_sctx.twd.twd_ctrl = 0;
591         }
592         tegra_twd_resume(&tegra_sctx.twd);
593 #endif
594 }
595
596 /*
597  * suspend_cpu_complex
598  *
599  * saves pll state for use by restart_plls, prepares flow controller for
600  * transition to suspend state
601  *
602  * Must always be called on cpu 0.
603  */
604 static void suspend_cpu_complex(u32 mode)
605 {
606         int cpu = cpu_logical_map(smp_processor_id());
607         unsigned int reg;
608         int i;
609
610         BUG_ON(cpu != 0);
611
612         /* switch coresite to clk_m, save off original source */
613         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
614         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
615
616         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
617         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
618         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
619         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
620         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
621         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
622         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
623         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
624
625 #ifdef CONFIG_HAVE_ARM_TWD
626         tegra_twd_suspend(&tegra_sctx.twd);
627 #endif
628
629         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
630         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
631         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
632         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
633         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
634 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
635         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
636 #else
637         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
638 #endif
639         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
640         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
641
642         for (i = 0; i < num_possible_cpus(); i++) {
643                 if (i == cpu)
644                         continue;
645                 reg = readl(FLOW_CTRL_CPU_CSR(i));
646                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
647                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
648                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
649         }
650
651         tegra_gic_cpu_disable(true);
652 }
653
654 void tegra_clear_cpu_in_pd(int cpu)
655 {
656         spin_lock(&tegra_lp2_lock);
657         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
658         cpumask_clear_cpu(cpu, &tegra_in_lp2);
659
660         /* Update the IRAM copy used by the reset handler. The IRAM copy
661            can't use used directly by cpumask_clear_cpu() because it uses
662            LDREX/STREX which requires the addressed location to be inner
663            cacheable and sharable which IRAM isn't. */
664         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
665         dsb();
666
667         spin_unlock(&tegra_lp2_lock);
668 }
669
670 bool tegra_set_cpu_in_pd(int cpu)
671 {
672         bool last_cpu = false;
673
674         spin_lock(&tegra_lp2_lock);
675         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
676         cpumask_set_cpu(cpu, &tegra_in_lp2);
677
678         /* Update the IRAM copy used by the reset handler. The IRAM copy
679            can't use used directly by cpumask_set_cpu() because it uses
680            LDREX/STREX which requires the addressed location to be inner
681            cacheable and sharable which IRAM isn't. */
682         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
683         dsb();
684
685         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
686                 last_cpu = true;
687 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
688         else if (cpu == 1)
689                 tegra2_cpu_set_resettable_soon();
690 #endif
691
692         spin_unlock(&tegra_lp2_lock);
693         return last_cpu;
694 }
695
696 static void tegra_sleep_core(enum tegra_suspend_mode mode,
697                              unsigned long v2p)
698 {
699 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
700         outer_flush_range(__pa(&tegra_resume_timestamps_start),
701                           __pa(&tegra_resume_timestamps_end));
702
703         if (mode == TEGRA_SUSPEND_LP0) {
704                 trace_smc_sleep_core(NVSEC_SMC_START);
705
706                 tegra_generic_smc(0x84000001, ((1 << 16) | (1 << 24) | 1),
707                                   virt_to_phys(tegra_resume));
708         } else {
709                 trace_smc_sleep_core(NVSEC_SMC_START);
710
711                 tegra_generic_smc(0x84000001, ((1 << 16) | 2),
712                                   (TEGRA_RESET_HANDLER_BASE +
713                                    tegra_cpu_reset_handler_offset));
714         }
715
716         trace_smc_sleep_core(NVSEC_SMC_DONE);
717 #endif
718         tegra_get_suspend_time();
719 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
720         cpu_suspend(v2p, tegra2_sleep_core_finish);
721 #else
722         cpu_suspend(v2p, tegra3_sleep_core_finish);
723 #endif
724 }
725
726 static inline void tegra_sleep_cpu(unsigned long v2p)
727 {
728         cpu_suspend(v2p, tegra_sleep_cpu_finish);
729 }
730
731 static inline void tegra_stop_mc_clk(unsigned long v2p)
732 {
733 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
734         outer_flush_range(__pa(&tegra_resume_timestamps_start),
735                           __pa(&tegra_resume_timestamps_end));
736         trace_smc_sleep_core(NVSEC_SMC_START);
737
738         tegra_generic_smc(0x84000001, ((1 << 16) | 3),
739                           (TEGRA_RESET_HANDLER_BASE +
740                            tegra_cpu_reset_handler_offset));
741
742         trace_smc_sleep_core(NVSEC_SMC_DONE);
743 #endif
744         cpu_suspend(v2p, tegra3_stop_mc_clk_finish);
745 }
746
747 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
748                                         unsigned int flags)
749 {
750         u32 reg;
751         unsigned int remain;
752
753         /* Only the last cpu down does the final suspend steps */
754         reg = readl(pmc + PMC_CTRL);
755         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
756         if (pdata->combined_req)
757                 reg &= ~TEGRA_POWER_PWRREQ_OE;
758         else
759                 reg |= TEGRA_POWER_PWRREQ_OE;
760
761         reg &= ~TEGRA_POWER_EFFECT_LP0;
762         writel(reg, pmc + PMC_CTRL);
763
764         /*
765          * We can use clk_get_rate_all_locked() here, because all other cpus
766          * are in LP2 state and irqs are disabled
767          */
768         suspend_cpu_dfll_mode(flags);
769         if (flags & TEGRA_POWER_CLUSTER_MASK) {
770 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
771                 set_power_timers(pdata->cpu_timer, 2);
772 #else
773                 set_power_timers(pdata->cpu_timer, 2,
774                         clk_get_rate_all_locked(tegra_pclk));
775 #endif
776                 if (flags & TEGRA_POWER_CLUSTER_G) {
777                         /*
778                          * To reduce the vdd_cpu up latency when LP->G
779                          * transition. Before the transition, enable
780                          * the vdd_cpu rail.
781                          */
782                         if (!tegra_crail_can_start_early() && is_lp_cluster()) {
783 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
784                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
785                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
786                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
787 #else
788                                 writel(UN_PWRGATE_CPU,
789                                        pmc + PMC_PWRGATE_TOGGLE);
790 #endif
791                         }
792                 }
793                 tegra_cluster_switch_prolog(flags);
794         } else {
795 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
796                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
797 #else
798                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
799                         clk_get_rate_all_locked(tegra_pclk));
800 #endif
801 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
802                 reg = readl(FLOW_CTRL_CPU_CSR(0));
803                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
804                 if (is_lp_cluster()) {
805                         /* for LP cluster, there is no option for rail gating */
806                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
807                                                 TEGRA_POWER_CLUSTER_PART_MASK)
808                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
809                         else if (flags)
810                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
811                 }
812                 else {
813                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
814                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
815                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
816                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
817                 }
818                 writel(reg, FLOW_CTRL_CPU_CSR(0));
819 #endif
820         }
821
822         if (sleep_time)
823                 tegra_pd_set_trigger(sleep_time);
824
825         cpu_cluster_pm_enter();
826         suspend_cpu_complex(flags);
827         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
828 #if defined(CONFIG_CACHE_L2X0)
829 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
830         flush_cache_all();
831         outer_disable();
832 #elif !defined(CONFIG_ARCH_TEGRA_14x_SOC)
833         tegra_resume_l2_init = 1;
834         __cpuc_flush_dcache_area(&tegra_resume_l2_init, sizeof(unsigned long));
835         outer_flush_range(__pa(&tegra_resume_l2_init),
836                           __pa(&tegra_resume_l2_init) + sizeof(unsigned long));
837 #endif
838 #endif
839
840         /* T148: Check for mem_req and mem_req_soon only if it is
841          * MC clock stop state.
842          */
843         if (flags & TEGRA_POWER_STOP_MC_CLK) {
844 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
845                 u32 val;
846
847                 /* Check if mem_req or mem_req_soon is asserted or if voice
848                  * call is active call, if yes then we skip SDRAM
849                  * self-refresh and just do CPU power-gating.
850                  */
851                 val = readl(pmc + PMC_IPC_STS);
852                 if ((val & (PMC_IPC_STS_MEM_REQ | PMC_IPC_STS_MEM_REQ_SOON)) ||
853                         tegra_is_voice_call_active()) {
854
855                         /* Reset LP1 and MC clock mask if we skipping SDRAM
856                          * self-refresh.
857                          */
858                         *iram_cpu_lp1_mask = 0;
859                         *iram_mc_clk_mask = 0;
860                         writel(0, pmc + PMC_SCRATCH41);
861
862                         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
863                 } else {
864                         /* Clear mem_sts since SDRAM will not be accessible
865                          * to BBC in this state.
866                          */
867                         val = PMC_IPC_CLR_MEM_STS;
868                         writel(val, pmc + PMC_IPC_CLR);
869
870                         tegra_stop_mc_clk(PHYS_OFFSET - PAGE_OFFSET);
871                 }
872 #else
873                 /* If it is not T148 then we do not have to
874                  * check mem_req and mem_req_soon.
875                  */
876                 tegra_stop_mc_clk(PHYS_OFFSET - PAGE_OFFSET);
877 #endif
878         } else {
879                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
880         }
881
882 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
883         tegra_init_cache(true);
884 #elif defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
885         tegra_init_cache(false);
886 #endif
887
888 #if defined(CONFIG_TRUSTED_FOUNDATIONS)
889 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
890         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
891         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
892 #endif
893 #endif
894
895         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
896         restore_cpu_complex(flags);
897         cpu_cluster_pm_exit();
898
899         remain = tegra_pd_timer_remain();
900         if (sleep_time)
901                 tegra_pd_set_trigger(0);
902
903         if (flags & TEGRA_POWER_CLUSTER_MASK) {
904                 tegra_cluster_switch_epilog(flags);
905         }
906         resume_cpu_dfll_mode(flags);
907         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
908
909 #if INSTRUMENT_CLUSTER_SWITCH
910         if (flags & TEGRA_POWER_CLUSTER_MASK) {
911                 pr_debug("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
912                         is_lp_cluster() ? "G=>LP" : "LP=>G",
913                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
914                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
915                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
916                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
917                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
918                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
919                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
920                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
921         }
922 #endif
923         return remain;
924 }
925
926 void tegra_mc_clk_prepare(void)
927 {
928         /* copy the reset vector and SDRAM shutdown code into IRAM */
929         memcpy(iram_save, iram_code, iram_save_size);
930         memcpy(iram_code, tegra_iram_start(), iram_save_size);
931
932         *iram_cpu_lp1_mask = 1;
933         *iram_mc_clk_mask = 1;
934
935         __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
936         wmb();
937 }
938
939 void tegra_mc_clk_finish(void)
940 {
941         /* restore IRAM */
942         memcpy(iram_code, iram_save, iram_save_size);
943         *iram_cpu_lp1_mask = 0;
944         *iram_mc_clk_mask = 0;
945         writel(0, pmc + PMC_SCRATCH41);
946 }
947
948 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
949 int tegra_is_lp1_suspend_mode(void)
950 {
951         return (current_suspend_mode == TEGRA_SUSPEND_LP1);
952 }
953 #endif
954
955 static int tegra_common_suspend(void)
956 {
957         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
958
959         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
960         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
961         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
962
963 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
964         if (pdata && pdata->lp1_lowvolt_support) {
965                 u32 lp1_core_lowvolt =
966                         (tegra_is_voice_call_active() ||
967                         tegra_dvfs_rail_get_thermal_floor(tegra_core_rail)) ?
968                         pdata->lp1_core_volt_low_cold << 8 :
969                         pdata->lp1_core_volt_low << 8;
970
971                 lp1_core_lowvolt |= pdata->core_reg_addr;
972                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
973         }
974 #endif
975
976         /* copy the reset vector and SDRAM shutdown code into IRAM */
977         memcpy(iram_save, iram_code, iram_save_size);
978         memcpy(iram_code, tegra_iram_start(), iram_save_size);
979
980         return 0;
981 }
982
983 static void tegra_common_resume(void)
984 {
985         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
986 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
987         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
988 #endif
989
990 #if defined(CONFIG_ARCH_TEGRA_14x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
991         /* Clear DPD Enable */
992         writel(0x0, pmc + PMC_DPD_ENABLE);
993 #endif
994
995         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
996         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
997         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
998 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
999         /* trigger emc mode write */
1000         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
1001         /* clear scratch registers shared by suspend and the reset pen */
1002         writel(0x0, pmc + PMC_SCRATCH39);
1003 #endif
1004         writel(0x0, pmc + PMC_SCRATCH41);
1005
1006         /* restore IRAM */
1007         memcpy(iram_code, iram_save, iram_save_size);
1008 }
1009
1010 static int tegra_suspend_prepare_late(void)
1011 {
1012 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1013         disable_irq(INT_SYS_STATS_MON);
1014 #endif
1015         return 0;
1016 }
1017
1018 static void tegra_suspend_wake(void)
1019 {
1020 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1021         enable_irq(INT_SYS_STATS_MON);
1022 #endif
1023 }
1024
1025 static void tegra_pm_set(enum tegra_suspend_mode mode)
1026 {
1027         u32 reg, boot_flag;
1028         unsigned long rate = 32768;
1029
1030         reg = readl(pmc + PMC_CTRL);
1031         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1032         if (pdata->combined_req)
1033                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1034         else
1035                 reg |= TEGRA_POWER_PWRREQ_OE;
1036         reg &= ~TEGRA_POWER_EFFECT_LP0;
1037
1038         switch (mode) {
1039         case TEGRA_SUSPEND_LP0:
1040 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
1041                 rate = clk_get_rate_all_locked(tegra_pclk);
1042 #endif
1043                 if (pdata->combined_req) {
1044                         reg |= TEGRA_POWER_PWRREQ_OE;
1045                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
1046                 }
1047
1048 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1049                 /*
1050                  * LP0 boots through the AVP, which then resumes the AVP to
1051                  * the address in scratch 39, and the cpu to the address in
1052                  * scratch 41 to tegra_resume
1053                  */
1054                 writel(0x0, pmc + PMC_SCRATCH39);
1055 #endif
1056
1057                 /* Enable DPD sample to trigger sampling pads data and direction
1058                  * in which pad will be driven during lp0 mode*/
1059                 writel(0x1, pmc + PMC_DPD_SAMPLE);
1060 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
1061 #if defined(CONFIG_ARCH_TEGRA_11x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
1062                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
1063                 tegra_is_dpd_mode = true;
1064 #else
1065                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ);
1066 #endif
1067                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
1068 #endif
1069
1070 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1071                 /* this is needed only for T11x, not for other chips */
1072                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
1073 #endif
1074
1075                 /* Set warmboot flag */
1076                 boot_flag = readl(pmc + PMC_SCRATCH0);
1077                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
1078
1079                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
1080
1081                 reg |= TEGRA_POWER_EFFECT_LP0;
1082                 /* No break here. LP0 code falls through to write SCRATCH41 */
1083         case TEGRA_SUSPEND_LP1:
1084                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
1085                 wmb();
1086                 rate = clk_get_rate(tegra_clk_m);
1087                 break;
1088         case TEGRA_SUSPEND_LP2:
1089                 rate = clk_get_rate(tegra_pclk);
1090                 break;
1091         case TEGRA_SUSPEND_NONE:
1092                 return;
1093         default:
1094                 BUG();
1095         }
1096
1097 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
1098         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
1099 #else
1100         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
1101 #endif
1102
1103         pmc_32kwritel(reg, PMC_CTRL);
1104 }
1105
1106 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
1107         [TEGRA_SUSPEND_NONE] = "none",
1108         [TEGRA_SUSPEND_LP2] = "LP2",
1109         [TEGRA_SUSPEND_LP1] = "LP1",
1110         [TEGRA_SUSPEND_LP0] = "LP0",
1111 };
1112
1113 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
1114 static int save_se_context(void)
1115 {
1116         struct device *smmu_dev, *se_dev;
1117         int ret = 0;
1118
1119         smmu_dev = get_smmu_device();
1120         if (!smmu_dev) {
1121                 pr_info("Failed to get smmu device\n");
1122                 goto save_fail;
1123         }
1124
1125         se_dev = get_se_device();
1126         if (!se_dev) {
1127                 pr_info("Failed to get SE device \n");
1128                 goto save_fail;
1129         }
1130
1131         /* smmu resume needs to be called
1132          * for se_suspend() operation */
1133         ret = tegra_smmu_resume(smmu_dev);
1134         if (ret) {
1135                 pr_info("Failed to resume smmu device\n");
1136                 goto save_fail;
1137         }
1138
1139         ret = se_suspend(se_dev, true);
1140         if (ret) {
1141                 pr_info("Failed to suspend SE device\n");
1142                 goto save_fail;
1143         }
1144
1145         ret = tegra_smmu_suspend(smmu_dev);
1146         if (ret) {
1147                 pr_info("Failed to suspend smmu device\n");
1148                 goto save_fail;
1149         }
1150
1151 save_fail:
1152         return ret;
1153 }
1154 #endif
1155
1156 static int tegra_suspend_enter(suspend_state_t state)
1157 {
1158         int ret = 0;
1159         ktime_t delta;
1160         struct timespec ts_entry, ts_exit;
1161
1162         if (pdata && pdata->board_suspend)
1163                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
1164
1165         read_persistent_clock(&ts_entry);
1166
1167         ret = tegra_suspend_dram(current_suspend_mode, 0);
1168         if (ret) {
1169                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
1170                 goto abort_suspend;
1171         }
1172
1173 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
1174         ret = save_se_context();
1175         if (ret) {
1176                 pr_info("Failed to save SE context\n");
1177                 goto abort_suspend;
1178         }
1179 #endif
1180
1181         read_persistent_clock(&ts_exit);
1182
1183         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
1184                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
1185
1186                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
1187                 tegra_dvfs_rail_pause(tegra_gpu_rail, delta, false);
1188                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1189                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
1190                 else
1191                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
1192         }
1193
1194 abort_suspend:
1195         if (pdata && pdata->board_resume)
1196                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
1197
1198         return ret;
1199 }
1200
1201 static void tegra_suspend_check_pwr_stats(void)
1202 {
1203         /* cpus and l2 are powered off later */
1204         unsigned long pwrgate_partid_mask =
1205 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
1206                 (1 << TEGRA_POWERGATE_HEG)      |
1207                 (1 << TEGRA_POWERGATE_SATA)     |
1208                 (1 << TEGRA_POWERGATE_3D1)      |
1209 #endif
1210                 (1 << TEGRA_POWERGATE_3D)       |
1211                 (1 << TEGRA_POWERGATE_VENC)     |
1212                 (1 << TEGRA_POWERGATE_PCIE)     |
1213                 (1 << TEGRA_POWERGATE_VDEC)     |
1214                 (1 << TEGRA_POWERGATE_MPE);
1215
1216         int partid;
1217
1218         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
1219                 if ((1 << partid) & pwrgate_partid_mask)
1220                         if (tegra_powergate_is_powered(partid))
1221                                 pr_debug("partition %s is left on before suspend\n",
1222                                         tegra_powergate_get_name(partid));
1223
1224         return;
1225 }
1226
1227 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1228 /* This is the opposite of the LP1BB related PMC setup that occurs
1229  * during suspend.
1230  */
1231 static void tegra_disable_lp1bb_interrupt(void)
1232 {
1233         unsigned reg;
1234         /* mem_req = 0 was set as an interrupt during LP1BB entry.
1235          * It has to be disabled now
1236          */
1237         reg = readl(pmc + PMC_CTRL2);
1238         reg &= ~(PMC_CTRL2_WAKE_DET_EN);
1239         pmc_32kwritel(reg, PMC_CTRL2);
1240
1241         /* Program mem_req NOT to be a wake event */
1242         reg = readl(pmc + PMC_WAKE2_MASK);
1243         reg &= ~(PMC_WAKE2_BB_MEM_REQ);
1244         pmc_32kwritel(reg, PMC_WAKE2_MASK);
1245
1246         reg = PMC_WAKE2_BB_MEM_REQ;
1247         pmc_32kwritel(reg, PMC_WAKE2_STATUS);
1248
1249         /* Set up the LIC to NOT accept pmc_wake events as interrupts */
1250         reg = TRI_ICTLR_PMC_WAKE_INT;
1251         writel(reg, tert_ictlr + TRI_ICTLR_CPU_IER_CLR);
1252 }
1253 #endif
1254
1255 static void tegra_suspend_powergate_control(int partid, bool turn_off)
1256 {
1257         if (turn_off)
1258                 tegra_powergate_partition(partid);
1259         else
1260                 tegra_unpowergate_partition(partid);
1261 }
1262
1263 #ifdef CONFIG_TEGRA_LP0_IN_IDLE
1264 int tegra_enter_lp0(unsigned long sleep_time)
1265 {
1266         int err = 0;
1267
1268         /* This state is managed by power domains, hence no voice call expected if
1269          * we are entering this state */
1270
1271         tegra_rtc_set_trigger(sleep_time);
1272
1273         tegra_actmon_save();
1274
1275         tegra_dma_save();
1276
1277         tegra_smmu_save();
1278
1279         err = syscore_save();
1280         if (err) {
1281                 tegra_smmu_restore();
1282                 tegra_dma_restore();
1283                 tegra_rtc_set_trigger(0);
1284                 return err;
1285         }
1286
1287         tegra_suspend_dram(TEGRA_SUSPEND_LP0, 0);
1288
1289         syscore_restore();
1290
1291         tegra_smmu_restore();
1292
1293         tegra_dma_restore();
1294
1295         tegra_actmon_restore();
1296
1297         tegra_rtc_set_trigger(0);
1298
1299         return 0;
1300 }
1301 #endif
1302
1303 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
1304 {
1305         int err = 0;
1306         u32 scratch37 = 0xDEADBEEF;
1307         u32 reg;
1308
1309 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1310         u32 enter_state = 0;
1311 #endif
1312         bool tegra_suspend_vde_powergated = false;
1313
1314         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
1315                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
1316                 err = -ENXIO;
1317                 goto fail;
1318         }
1319
1320 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1321         update_pmc_registers(tegra_lp1bb_emc_min_rate_get());
1322 #endif
1323
1324         if (tegra_is_voice_call_active()) {
1325                 /* backup the current value of scratch37 */
1326                 scratch37 = readl(pmc + PMC_SCRATCH37);
1327
1328                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
1329                 reg = TEGRA_POWER_LP1_AUDIO;
1330                 pmc_32kwritel(reg, PMC_SCRATCH37);
1331         }
1332
1333         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
1334                 pr_info("LP0 not used due to unsupported wakeup events\n");
1335                 mode = TEGRA_SUSPEND_LP1;
1336         }
1337
1338         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
1339                 tegra_suspend_check_pwr_stats();
1340
1341         /* turn off VDE partition in LP1 */
1342         if (mode == TEGRA_SUSPEND_LP1 &&
1343                 tegra_powergate_is_powered(TEGRA_POWERGATE_VDEC)) {
1344                 pr_info("turning off partition %s in LP1\n",
1345                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
1346                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, true);
1347                 tegra_suspend_vde_powergated = true;
1348         }
1349
1350         tegra_common_suspend();
1351
1352         tegra_pm_set(mode);
1353
1354         if (pdata && pdata->board_suspend)
1355                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
1356
1357         local_fiq_disable();
1358
1359 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1360         tegra_smp_save_power_mask();
1361 #endif
1362
1363         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
1364
1365         if (mode == TEGRA_SUSPEND_LP0) {
1366 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
1367                 reg = readl(pmc + PMC_SCRATCH4);
1368                 if (is_lp_cluster())
1369                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
1370                 else
1371                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
1372                 pmc_32kwritel(reg, PMC_SCRATCH4);
1373 #endif
1374                 tegra_tsc_suspend();
1375                 tegra_lp0_suspend_mc();
1376                 tegra_cpu_reset_handler_save();
1377                 tegra_tsc_wait_for_suspend();
1378                 if (!tegra_cpu_is_asim())
1379                         tegra_smp_clear_power_mask();
1380         }
1381
1382 #if !defined(CONFIG_ARCH_TEGRA_14x_SOC)
1383         if (mode == TEGRA_SUSPEND_LP1)
1384 #endif
1385                 *iram_cpu_lp1_mask = 1;
1386
1387         suspend_cpu_complex(flags);
1388
1389 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
1390         /* In case of LP0/1, program external power gating accordinly */
1391         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
1392                 reg = readl(FLOW_CTRL_CPU_CSR(0));
1393                 if (is_lp_cluster())
1394                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
1395                 else
1396                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
1397                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
1398         }
1399 #endif
1400
1401         flush_cache_all();
1402         outer_disable();
1403
1404         if (mode == TEGRA_SUSPEND_LP2)
1405                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
1406         else
1407                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
1408
1409         resume_entry_time = 0;
1410         if (mode != TEGRA_SUSPEND_LP0)
1411                 resume_entry_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
1412
1413         tegra_init_cache(true);
1414
1415 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1416         reg = readl(pmc + PMC_LP_STATE_SCRATCH_REG);
1417         enter_state = (reg >> PMC_LP_STATE_BIT_OFFSET) & PMC_LP_STATE_BIT_MASK;
1418         /* If we actually had entered in either LP1 or LP1BB,
1419          * restore power mask and disable mem_req interrupt PMC
1420          */
1421         if (enter_state) {
1422                 pr_debug("Exited state is LP1/LP1BB\n");
1423                 tegra_disable_lp1bb_interrupt();
1424                 tegra_smp_restore_power_mask();
1425         }
1426 #endif
1427
1428 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
1429 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
1430         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
1431         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
1432 #endif
1433
1434         if (mode == TEGRA_SUSPEND_LP0) {
1435                 trace_secureos_init(tegra_resume_entry_time,
1436                         NVSEC_SUSPEND_EXIT_DONE);
1437         }
1438 #endif
1439
1440         if (mode == TEGRA_SUSPEND_LP0) {
1441
1442                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
1443                 * Otherwise it is creating issue in cluster switch after LP0 *
1444 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1445                 reg = readl(pmc+PMC_CTRL);
1446                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
1447                 pmc_32kwritel(reg, PMC_CTRL);
1448 #endif
1449                 */
1450
1451                 tegra_tsc_resume();
1452                 tegra_cpu_reset_handler_restore();
1453                 tegra_lp0_resume_mc();
1454                 tegra_tsc_wait_for_resume();
1455         }
1456
1457 #if !defined(CONFIG_ARCH_TEGRA_14x_SOC)
1458         if (mode == TEGRA_SUSPEND_LP1)
1459 #endif
1460                 *iram_cpu_lp1_mask = 0;
1461
1462         /* if scratch37 was clobbered during LP1, restore it */
1463         if (scratch37 != 0xDEADBEEF)
1464                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1465
1466         restore_cpu_complex(flags);
1467
1468         /* for platforms where the core & CPU power requests are
1469          * combined as a single request to the PMU, transition out
1470          * of LP0 state by temporarily enabling both requests
1471          */
1472         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1473                 reg = readl(pmc + PMC_CTRL);
1474                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1475                 pmc_32kwritel(reg, PMC_CTRL);
1476                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1477                 pmc_32kwritel(reg, PMC_CTRL);
1478         }
1479
1480         if (pdata && pdata->board_resume)
1481                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1482
1483         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1484
1485         local_fiq_enable();
1486
1487         tegra_common_resume();
1488
1489         /* turn on VDE partition in LP1 */
1490         if (mode == TEGRA_SUSPEND_LP1 && tegra_suspend_vde_powergated) {
1491                 pr_info("turning on partition %s in LP1\n",
1492                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
1493                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, false);
1494         }
1495
1496 fail:
1497         return err;
1498 }
1499
1500 /*
1501  * Function pointers to optional board specific function
1502  */
1503 void (*tegra_deep_sleep)(int);
1504 EXPORT_SYMBOL(tegra_deep_sleep);
1505
1506 static int tegra_suspend_prepare(void)
1507 {
1508         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1509                 tegra_deep_sleep(1);
1510         return 0;
1511 }
1512
1513 static void tegra_suspend_finish(void)
1514 {
1515         if (pdata && pdata->cpu_resume_boost) {
1516                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1517                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1518                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1519         }
1520
1521         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1522                 tegra_deep_sleep(0);
1523 }
1524
1525 static const struct platform_suspend_ops tegra_suspend_ops = {
1526         .valid          = suspend_valid_only_mem,
1527         .prepare        = tegra_suspend_prepare,
1528         .finish         = tegra_suspend_finish,
1529         .prepare_late   = tegra_suspend_prepare_late,
1530         .wake           = tegra_suspend_wake,
1531         .enter          = tegra_suspend_enter,
1532 };
1533
1534 static ssize_t suspend_mode_show(struct kobject *kobj,
1535                                         struct kobj_attribute *attr, char *buf)
1536 {
1537         char *start = buf;
1538         char *end = buf + PAGE_SIZE;
1539
1540         start += scnprintf(start, end - start, "%s ", \
1541                                 tegra_suspend_name[current_suspend_mode]);
1542         start += scnprintf(start, end - start, "\n");
1543
1544         return start - buf;
1545 }
1546
1547 static ssize_t suspend_mode_store(struct kobject *kobj,
1548                                         struct kobj_attribute *attr,
1549                                         const char *buf, size_t n)
1550 {
1551         int len;
1552         const char *name_ptr;
1553         enum tegra_suspend_mode new_mode;
1554
1555         name_ptr = buf;
1556         while (*name_ptr && !isspace(*name_ptr))
1557                 name_ptr++;
1558         len = name_ptr - buf;
1559         if (!len)
1560                 goto bad_name;
1561         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1562         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1563                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1564                 pr_info("Illegal tegra suspend state: %s\n", buf);
1565                 goto bad_name;
1566         }
1567
1568         for (new_mode = TEGRA_SUSPEND_NONE; \
1569                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1570                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1571                         current_suspend_mode = new_mode;
1572                         break;
1573                 }
1574         }
1575
1576 bad_name:
1577         return n;
1578 }
1579
1580 static struct kobj_attribute suspend_mode_attribute =
1581         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1582
1583 static ssize_t suspend_resume_time_show(struct kobject *kobj,
1584                                         struct kobj_attribute *attr,
1585                                         char *buf)
1586 {
1587         return sprintf(buf, "%ums\n", ((u32)resume_time / 1000));
1588 }
1589
1590 static struct kobj_attribute suspend_resume_time_attribute =
1591         __ATTR(resume_time, 0444, suspend_resume_time_show, 0);
1592
1593 static ssize_t suspend_time_show(struct kobject *kobj,
1594                                         struct kobj_attribute *attr,
1595                                         char *buf)
1596 {
1597         return sprintf(buf, "%ums\n", ((u32)suspend_time / 1000));
1598 }
1599
1600 static struct kobj_attribute suspend_time_attribute =
1601         __ATTR(suspend_time, 0444, suspend_time_show, 0);
1602
1603 static struct kobject *suspend_kobj;
1604
1605 static int tegra_pm_enter_suspend(void)
1606 {
1607         suspend_cpu_dfll_mode(0);
1608         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1609                 tegra_lp0_cpu_mode(true);
1610         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1611         return 0;
1612 }
1613
1614 static void tegra_pm_enter_resume(void)
1615 {
1616         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1617         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1618                 tegra_lp0_cpu_mode(false);
1619         resume_cpu_dfll_mode(0);
1620 }
1621
1622 static void tegra_pm_enter_shutdown(void)
1623 {
1624         suspend_cpu_dfll_mode(0);
1625         pr_info("Shutting down tegra ...\n");
1626 }
1627
1628 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1629         .suspend = tegra_pm_enter_suspend,
1630         .resume = tegra_pm_enter_resume,
1631         .save = tegra_pm_enter_suspend,
1632         .restore = tegra_pm_enter_resume,
1633         .shutdown = tegra_pm_enter_shutdown,
1634 };
1635
1636 static __init int tegra_pm_enter_syscore_init(void)
1637 {
1638         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1639         return 0;
1640 }
1641 subsys_initcall(tegra_pm_enter_syscore_init);
1642 #endif
1643
1644 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1645 {
1646         u32 reg;
1647         u32 mode;
1648         struct pmc_pm_data *pm_dat;
1649         bool is_board_pdata = true;
1650
1651 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1652         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1653         BUG_ON(IS_ERR(tegra_dfll));
1654 #endif
1655         tegra_pclk = clk_get_sys(NULL, "pclk");
1656         BUG_ON(IS_ERR(tegra_pclk));
1657         tegra_clk_m = clk_get_sys(NULL, "clk_m");
1658         BUG_ON(IS_ERR(tegra_clk_m));
1659
1660         /* create the pdata from DT information */
1661         pm_dat = tegra_get_pm_data();
1662         if (pm_dat) {
1663                 pr_debug("PMC dt information non-NULL %s\n", __func__);
1664                 is_board_pdata = false;
1665                 pdata = kzalloc(sizeof(struct tegra_suspend_platform_data),
1666                         GFP_KERNEL);
1667                 if (pm_dat->combined_req != plat->combined_req) {
1668                         pr_err("PMC DT attribute combined_req=%d, board value=%d\n",
1669                                 pm_dat->combined_req, plat->combined_req);
1670                         pdata->combined_req = plat->combined_req;
1671                 } else {
1672                         pdata->combined_req = pm_dat->combined_req;
1673                 }
1674                 if (pm_dat->sysclkreq_high != plat->sysclkreq_high) {
1675                         pr_err("PMC DT attribute sysclkreq_high=%d, board value=%d\n",
1676                                 pm_dat->sysclkreq_high, plat->sysclkreq_high);
1677                         pdata->sysclkreq_high = plat->sysclkreq_high;
1678                 } else {
1679                         pdata->sysclkreq_high = pm_dat->sysclkreq_high;
1680                 }
1681                 if (pm_dat->corereq_high != plat->corereq_high) {
1682                         pr_err("PMC DT attribute corereq_high=%d, board value=%d\n",
1683                                 pm_dat->corereq_high, plat->corereq_high);
1684                         pdata->corereq_high = plat->corereq_high;
1685                 } else {
1686                         pdata->corereq_high = pm_dat->corereq_high;
1687                 }
1688                 if (pm_dat->cpu_off_time != plat->cpu_off_timer) {
1689                         pr_err("PMC DT attribute cpu_off_timer=%d, board value=%ld\n",
1690                                 pm_dat->cpu_off_time, plat->cpu_off_timer);
1691                         pdata->cpu_off_timer = plat->cpu_off_timer;
1692                 } else {
1693                         pdata->cpu_off_timer = pm_dat->cpu_off_time;
1694                 }
1695                 if (pm_dat->cpu_good_time != plat->cpu_timer) {
1696                         pr_err("PMC DT attribute cpu_timer=%d, board value=%ld\n",
1697                                 pm_dat->cpu_good_time, plat->cpu_timer);
1698                         pdata->cpu_timer = plat->cpu_timer;
1699                 } else {
1700                         pdata->cpu_timer = pm_dat->cpu_good_time;
1701                 }
1702                 if (pm_dat->suspend_mode != plat->suspend_mode) {
1703                         pr_err("PMC DT attribute suspend_mode=%d, board value=%d\n",
1704                                 pm_dat->suspend_mode, plat->suspend_mode);
1705                         pdata->suspend_mode = plat->suspend_mode;
1706                 } else {
1707                         pdata->suspend_mode = pm_dat->suspend_mode;
1708                 }
1709                 /* FIXME: pmc_pm_data fields to be reused
1710                  *      core_osc_time, core_pmu_time, core_off_time
1711                  *      units of above fields is uSec while
1712                  *      platform data values are in ticks
1713                  */
1714                 /* FIXME: pmc_pm_data unused by downstream code
1715                  *      cpu_pwr_good_en, lp0_vec_size, lp0_vec_phy_addr
1716                  */
1717                 /* FIXME: add missing DT bindings taken from platform data */
1718                 pdata->core_timer = plat->core_timer;
1719                 pdata->core_off_timer = plat->core_off_timer;
1720                 pdata->board_suspend = plat->board_suspend;
1721                 pdata->board_resume = plat->board_resume;
1722                 pdata->sysclkreq_gpio = plat->sysclkreq_gpio;
1723                 pdata->cpu_lp2_min_residency = plat->cpu_lp2_min_residency;
1724                 pdata->cpu_resume_boost = plat->cpu_resume_boost;
1725 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
1726                 pdata->lp1_lowvolt_support = plat->lp1_lowvolt_support;
1727                 pdata->i2c_base_addr = plat->i2c_base_addr;
1728                 pdata->pmuslave_addr = plat->pmuslave_addr;
1729                 pdata->core_reg_addr = plat->core_reg_addr;
1730                 pdata->lp1_core_volt_low_cold = plat->lp1_core_volt_low_cold;
1731                 pdata->lp1_core_volt_low = plat->lp1_core_volt_low;
1732                 pdata->lp1_core_volt_high = plat->lp1_core_volt_high;
1733 #endif
1734 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
1735                 pdata->min_residency_vmin_fmin = plat->min_residency_vmin_fmin;
1736                 pdata->min_residency_ncpu_slow = plat->min_residency_ncpu_slow;
1737                 pdata->min_residency_ncpu_fast = plat->min_residency_ncpu_fast;
1738                 pdata->min_residency_crail = plat->min_residency_crail;
1739                 pdata->crail_up_early = plat->crail_up_early;
1740 #endif
1741                 pdata->min_residency_mclk_stop = plat->min_residency_mclk_stop;
1742                 pdata->usb_vbus_internal_wake = plat->usb_vbus_internal_wake;
1743                 pdata->usb_id_internal_wake = plat->usb_id_internal_wake;
1744                 pdata->suspend_dfll_bypass = plat->suspend_dfll_bypass;
1745                 pdata->resume_dfll_bypass = plat->resume_dfll_bypass;
1746         } else {
1747                 pr_err("PMC board data used in %s\n", __func__);
1748                 pdata = plat;
1749         }
1750         (void)reg;
1751         (void)mode;
1752
1753         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1754                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1755
1756 #ifndef CONFIG_PM_SLEEP
1757         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1758                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1759                            "disabling suspend\n", __func__);
1760                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1761         }
1762 #else
1763         if (create_suspend_pgtable() < 0) {
1764                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1765                                 __func__);
1766                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1767                 goto fail;
1768         }
1769
1770         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1771             (tegra_revision == TEGRA_REVISION_A01) &&
1772             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1773                 /* Tegra 3 A01 supports only LP1 */
1774                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1775                            "-- disabling LP0\n", __func__);
1776                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1777         }
1778         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1779                 tegra_lp0_vec_relocate) {
1780                 unsigned char *reloc_lp0;
1781                 unsigned long tmp;
1782                 void __iomem *orig;
1783                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1784                                         GFP_KERNEL);
1785                 WARN_ON(!reloc_lp0);
1786                 if (!reloc_lp0) {
1787                         pr_err("%s: Failed to allocate reloc_lp0\n",
1788                                 __func__);
1789                         goto out;
1790                 }
1791
1792                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1793                 WARN_ON(!orig);
1794                 if (!orig) {
1795                         pr_err("%s: Failed to map tegra_lp0_vec_start %08x\n",
1796                                 __func__, (unsigned int) tegra_lp0_vec_start);
1797                         kfree(reloc_lp0);
1798                         goto out;
1799                 }
1800
1801                 tmp = (unsigned long) reloc_lp0;
1802                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1803                 reloc_lp0 = (unsigned char *)tmp;
1804                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1805                 iounmap(orig);
1806                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1807         }
1808
1809 out:
1810         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1811                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1812                            "provided by bootlader -- disabling LP0\n",
1813                            __func__);
1814                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1815         }
1816
1817         iram_save_size = tegra_iram_end() - tegra_iram_start();
1818
1819         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1820         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1821                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1822                        "-- LP0/LP1 unavailable\n", __func__);
1823                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1824         }
1825
1826 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
1827         if (pdata->lp1_lowvolt_support) {
1828                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1829                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1830                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1831
1832                 lp1_core_lowvolt = 0;
1833                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1834                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1835
1836                 lp1_core_highvolt = 0;
1837                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1838                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1839         }
1840 #endif
1841         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1842         /* Initialize scratch registers used for CPU LP2 synchronization */
1843         writel(0, pmc + PMC_SCRATCH37);
1844         writel(0, pmc + PMC_SCRATCH38);
1845 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1846         writel(0, pmc + PMC_SCRATCH39);
1847 #endif
1848         writel(0, pmc + PMC_SCRATCH41);
1849
1850         /* Always enable CPU power request; just normal polarity is supported */
1851         reg = readl(pmc + PMC_CTRL);
1852         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1853         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1854         pmc_32kwritel(reg, PMC_CTRL);
1855
1856         /* Configure core power request and system clock control if LP0
1857            is supported */
1858         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1859         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1860
1861         reg = readl(pmc + PMC_CTRL);
1862
1863         if (!pdata->sysclkreq_high)
1864                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1865         else
1866                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1867
1868         if (!pdata->corereq_high)
1869                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1870         else
1871                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1872
1873         /* configure output inverters while the request is tristated */
1874         pmc_32kwritel(reg, PMC_CTRL);
1875
1876         /* now enable requests */
1877         reg |= TEGRA_POWER_SYSCLK_OE;
1878         if (!pdata->combined_req)
1879                 reg |= TEGRA_POWER_PWRREQ_OE;
1880         pmc_32kwritel(reg, PMC_CTRL);
1881
1882         if (pdata->sysclkreq_gpio) {
1883                 reg = readl(pmc + PMC_DPAD_ORIDE);
1884                 reg &= ~TEGRA_DPAD_ORIDE_SYS_CLK_REQ;
1885                 pmc_32kwritel(reg, PMC_DPAD_ORIDE);
1886         }
1887
1888         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1889                 tegra_lp0_suspend_init();
1890
1891         suspend_set_ops(&tegra_suspend_ops);
1892
1893         /* Create /sys/power/suspend/type */
1894         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1895         if (suspend_kobj) {
1896                 if (sysfs_create_file(suspend_kobj, \
1897                                                 &suspend_mode_attribute.attr))
1898                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1899                                                                 __func__);
1900                 if (sysfs_create_file(suspend_kobj, \
1901                                         &suspend_resume_time_attribute.attr))
1902                         pr_err("%s: sysfs_create_file resume_time failed!\n",
1903                                                                 __func__);
1904                 if (sysfs_create_file(suspend_kobj, \
1905                                         &suspend_time_attribute.attr))
1906                         pr_err("%s: sysfs_create_file suspend_time failed!\n",
1907                                                                 __func__);
1908         }
1909
1910         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1911         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1912         iram_mc_clk_mask = tegra_mc_clk_mask;
1913
1914         /* clear io dpd settings before kernel */
1915         tegra_bl_io_dpd_cleanup();
1916
1917 fail:
1918 #endif
1919         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1920                 tegra_pd_in_idle(false);
1921
1922         current_suspend_mode = plat->suspend_mode;
1923 }
1924
1925 bool tegra_dvfs_is_dfll_bypass(void)
1926 {
1927 #ifdef CONFIG_REGULATOR_TEGRA_DFLL_BYPASS
1928         return pdata && (pdata->suspend_dfll_bypass ||
1929                          pdata->resume_dfll_bypass);
1930 #else
1931         return false;
1932 #endif
1933 }
1934
1935 void tegra_lp1bb_suspend_emc_rate(unsigned long emc_min, unsigned long emc_max)
1936 {
1937         pdata->lp1bb_emc_rate_min = emc_min;
1938         pdata->lp1bb_emc_rate_max = emc_max;
1939 }
1940
1941 void tegra_lp1bb_suspend_mv_set(int mv)
1942 {
1943         if (WARN_ON_ONCE(!pdata))
1944                 return;
1945
1946         pdata->lp1bb_core_volt_min = mv;
1947 }
1948
1949 unsigned long tegra_lp1bb_emc_min_rate_get(void)
1950 {
1951         if (WARN_ON_ONCE(!pdata) || !pdata->lp1bb_emc_rate_min)
1952                 return 204000000;
1953
1954         return pdata->lp1bb_emc_rate_min;
1955 }
1956
1957 unsigned long debug_uart_port_base = 0;
1958 EXPORT_SYMBOL(debug_uart_port_base);
1959
1960 static int tegra_debug_uart_suspend(void)
1961 {
1962         void __iomem *uart;
1963         u32 lcr;
1964
1965         if (!debug_uart_port_base)
1966                 return 0;
1967
1968         uart = IO_ADDRESS(debug_uart_port_base);
1969
1970         lcr = readb(uart + UART_LCR * 4);
1971
1972         tegra_sctx.uart[0] = lcr;
1973         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1974
1975         /* DLAB = 0 */
1976         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1977
1978         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1979
1980         /* DLAB = 1 */
1981         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1982
1983         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1984         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1985
1986         writeb(lcr, uart + UART_LCR * 4);
1987
1988         return 0;
1989 }
1990
1991 static void tegra_debug_uart_resume(void)
1992 {
1993         void __iomem *uart;
1994         u32 lcr;
1995
1996         if (!debug_uart_port_base)
1997                 return;
1998
1999         uart = IO_ADDRESS(debug_uart_port_base);
2000
2001         lcr = tegra_sctx.uart[0];
2002
2003         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
2004
2005         /* DLAB = 0 */
2006         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
2007
2008         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
2009                         uart + UART_FCR * 4);
2010
2011         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
2012
2013         /* DLAB = 1 */
2014         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
2015
2016         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
2017         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
2018
2019         writeb(lcr, uart + UART_LCR * 4);
2020 }
2021
2022 static struct syscore_ops tegra_debug_uart_syscore_ops = {
2023         .suspend = tegra_debug_uart_suspend,
2024         .resume = tegra_debug_uart_resume,
2025         .save = tegra_debug_uart_suspend,
2026         .restore = tegra_debug_uart_resume,
2027 };
2028
2029 struct clk *debug_uart_clk = NULL;
2030 EXPORT_SYMBOL(debug_uart_clk);
2031
2032 void tegra_console_uart_suspend(void)
2033 {
2034         if (console_suspend_enabled && debug_uart_clk)
2035                 tegra_clk_disable_unprepare(debug_uart_clk);
2036 }
2037
2038 void tegra_console_uart_resume(void)
2039 {
2040         if (console_suspend_enabled && debug_uart_clk)
2041                 tegra_clk_prepare_enable(debug_uart_clk);
2042 }
2043
2044 static int tegra_debug_uart_syscore_init(void)
2045 {
2046         register_syscore_ops(&tegra_debug_uart_syscore_ops);
2047         return 0;
2048 }
2049 arch_initcall(tegra_debug_uart_syscore_init);
2050
2051 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
2052 static inline bool pmc_write_check(int index, int bit_position)
2053 {
2054         if (pmc_write_bitmap[index] & (1 << bit_position))
2055                 return true;
2056         else
2057                 return false;
2058 }
2059
2060 static void update_pmc_registers(unsigned long rate)
2061 {
2062         u32 i, j;
2063         int instance = 1;
2064
2065         /* FIXME: convert rate to instance */
2066
2067         /* Based on index, we select that block of scratches */
2068         u32 base2 = (tegra_wb0_params_address + (instance - 1) *
2069                 tegra_wb0_params_block_size);
2070         void __iomem *base = ioremap(base2, tegra_wb0_params_block_size);
2071
2072 #define copy_dram_to_pmc(index, bit)    \
2073         pmc_32kwritel(readl(base + PMC_REGISTER_OFFSET(index, bit)), \
2074                 PMC_REGISTER_OFFSET(index, bit) + PMC_SCRATCH0)
2075
2076
2077         /* Iterate through the bitmap, and copy those registers
2078          * which are marked in the bitmap
2079          */
2080         for (i = 0, j = 0; j < ARRAY_SIZE(pmc_write_bitmap);) {
2081                 if (pmc_write_bitmap[j] == 0) {
2082                         j++;
2083                         i = 0;
2084                         continue;
2085                 }
2086
2087                 if (pmc_write_check(j, i))
2088                         copy_dram_to_pmc(j, i);
2089
2090                 if (++i > (sizeof(pmc_write_bitmap[0]) * 8)) {
2091                         i = 0;
2092                         j++;
2093                 }
2094         }
2095
2096 #undef copy_dram_to_pmc
2097         iounmap(base);
2098 }
2099 #endif
2100
2101 #ifdef CONFIG_ARM_ARCH_TIMER
2102
2103 static u32 tsc_suspend_start;
2104 static u32 tsc_resume_start;
2105
2106 #define pmc_writel(value, reg) \
2107                 writel(value, pmc + (reg))
2108 #define pmc_readl(reg) \
2109                 readl(pmc + (reg))
2110
2111 #define PMC_DPD_ENABLE                  0x24
2112 #define PMC_DPD_ENABLE_TSC_MULT_ENABLE  (1 << 1)
2113 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
2114 #define PMC_DPD_ENABLE_ON               (1 << 0)
2115 #endif
2116
2117 #define PMC_TSC_MULT                    0x2b4
2118 #define PMC_TSC_MULT_FREQ_STS           (1 << 16)
2119
2120 #define TSC_TIMEOUT_US                  32
2121
2122 void tegra_tsc_suspend(void)
2123 {
2124         if (arch_timer_initialized) {
2125                 u32 reg = pmc_readl(PMC_DPD_ENABLE);
2126                 BUG_ON(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE);
2127                 reg |= PMC_DPD_ENABLE_TSC_MULT_ENABLE;
2128                 pmc_writel(reg, PMC_DPD_ENABLE);
2129                 tsc_suspend_start = timer_readl(TIMERUS_CNTR_1US);
2130         }
2131 }
2132
2133 void tegra_tsc_resume(void)
2134 {
2135         if (arch_timer_initialized) {
2136                 u32 reg = pmc_readl(PMC_DPD_ENABLE);
2137                 BUG_ON(!(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE));
2138                 reg &= ~PMC_DPD_ENABLE_TSC_MULT_ENABLE;
2139 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
2140                 /*
2141                  * FIXME: T12x SW WAR -
2142                  * Resume ensures DPD_ENABLE is 0 when writing
2143                  * TSC_MULT_ENABLE, else PMC wake status gets reset
2144                  */
2145                 reg &= ~PMC_DPD_ENABLE_ON;
2146 #endif
2147                 pmc_writel(reg, PMC_DPD_ENABLE);
2148                 tsc_resume_start = timer_readl(TIMERUS_CNTR_1US);
2149         }
2150 }
2151
2152 void tegra_tsc_wait_for_suspend(void)
2153 {
2154         if (arch_timer_initialized) {
2155                 while ((timer_readl(TIMERUS_CNTR_1US) - tsc_suspend_start) <
2156                         TSC_TIMEOUT_US) {
2157                         if (pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS)
2158                                 break;
2159                         cpu_relax();
2160                 }
2161         }
2162 }
2163
2164 void tegra_tsc_wait_for_resume(void)
2165 {
2166         if (arch_timer_initialized) {
2167                 while ((timer_readl(TIMERUS_CNTR_1US) - tsc_resume_start) <
2168                         TSC_TIMEOUT_US) {
2169                         if (!(pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS))
2170                                 break;
2171                         cpu_relax();
2172                 }
2173         }
2174 }
2175 #endif
2176
2177 #if defined(CONFIG_DEBUG_FS) && INSTRUMENT_CLUSTER_SWITCH
2178
2179 static void cluster_switch_stats_show(
2180         struct seq_file *s, struct tegra_cluster_switch_time_stats *stats)
2181 {
2182         seq_printf(s, "%u-samples average:           %lu\n",
2183                    CLUSTER_SWITCH_AVG_SAMPLES,
2184                    stats->avg >> CLUSTER_SWITCH_TIME_AVG_SHIFT);
2185         seq_printf(s, "exponential average:          %lu\n",
2186                    stats->exp_avg >> CLUSTER_SWITCH_TIME_AVG_SHIFT);
2187         seq_printf(s, "maximum since boot:           %lu\n\n", stats->max);
2188 }
2189
2190
2191 static int tegra_cluster_switch_stats_show(struct seq_file *s, void *data)
2192 {
2193         seq_printf(s, "G=>LP cluster switch timing:  (us)\n");
2194         cluster_switch_stats_show(s, &g2lp_stats);
2195         seq_printf(s, "LP=>G cluster switch timing:  (us)\n");
2196         cluster_switch_stats_show(s, &lp2g_stats);
2197         return 0;
2198 }
2199
2200 static int tegra_cluster_switch_stats_open(
2201         struct inode *inode, struct file *file)
2202 {
2203         return single_open(file, tegra_cluster_switch_stats_show,
2204                            inode->i_private);
2205 }
2206
2207 static const struct file_operations tegra_cluster_switch_stats_ops = {
2208         .open           = tegra_cluster_switch_stats_open,
2209         .read           = seq_read,
2210         .llseek         = seq_lseek,
2211         .release        = single_release,
2212 };
2213
2214 static int __init tegra_pm_core_debug_init(void)
2215 {
2216         struct dentry *dir, *d;
2217
2218         dir = debugfs_create_dir("tegra_pm_core", NULL);
2219         if (!dir)
2220                 return -ENOMEM;
2221
2222         d = debugfs_create_file("cluster_switch_stats", S_IRUGO, dir, NULL,
2223                 &tegra_cluster_switch_stats_ops);
2224         if (!d)
2225                 return -ENOMEM;
2226
2227         return 0;
2228 }
2229
2230 late_initcall(tegra_pm_core_debug_init);
2231 #endif
2232
2233 #ifdef CONFIG_DEBUG_RODATA
2234 void set_platform_text_rw(void)
2235 {
2236 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
2237         set_memory_rw((unsigned long)tegra_generic_smc, 1);
2238 #endif
2239 }
2240 #endif