arm64: tegra: pm: extended state ID format for power states
[linux-3.10.git] / arch / arm64 / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2013-2015, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/cpumask.h>
32 #include <linux/delay.h>
33 #include <linux/cpu_pm.h>
34 #include <linux/err.h>
35 #include <linux/debugfs.h>
36 #include <linux/delay.h>
37 #include <linux/suspend.h>
38 #include <linux/serial_reg.h>
39 #include <linux/seq_file.h>
40 #include <linux/uaccess.h>
41 #include <linux/syscore_ops.h>
42 #include <linux/cpu_pm.h>
43 #include <linux/clk/tegra.h>
44 #include <linux/export.h>
45 #include <linux/vmalloc.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48 #include <linux/clk/tegra.h>
49 #include <linux/tegra-powergate.h>
50 #include <linux/tegra-soc.h>
51 #include <linux/tegra-timer.h>
52 #include <linux/tegra-cpuidle.h>
53 #include <linux/irqchip/tegra.h>
54 #include <linux/tegra-pm.h>
55 #include <linux/tegra_pm_domains.h>
56 #include <linux/tegra_smmu.h>
57 #include <soc/tegra/tegra_bpmp.h>
58 #include <linux/kmemleak.h>
59
60 #include <trace/events/power.h>
61 #include <trace/events/nvsecurity.h>
62 #include <linux/tegra-pmc.h>
63
64 #include <asm/cacheflush.h>
65 #include <asm/idmap.h>
66 #include <asm/pgalloc.h>
67 #include <asm/pgtable.h>
68 #include <asm/tlbflush.h>
69 #include <asm/suspend.h>
70 #include <asm/smp_plat.h>
71 #include <asm/psci.h>
72
73 #include <mach/irqs.h>
74 #include <mach/dc.h>
75
76 #include "board.h"
77 #include <linux/platform/tegra/clock.h>
78 #include <linux/platform/tegra/common.h>
79 #include "iomap.h"
80 #include "pm.h"
81 #include "pm-soc.h"
82 #include <linux/platform/tegra/reset.h>
83 #include "sleep.h"
84 #include <linux/platform/tegra/dvfs.h>
85 #include <linux/platform/tegra/cpu-tegra.h>
86 #include <linux/platform/tegra/flowctrl.h>
87
88 #include "pm-tegra132.h"
89
90 /* core power request enable */
91 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)
92 /* enter LP0 when CPU pwr gated */
93 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)
94 /* CPU pwr req enable */
95 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)
96
97 struct suspend_context {
98         /*
99          * The next 7 values are referenced by offset in __restart_plls
100          * in headsmp-t2.S, and should not be moved
101          */
102         u32 pllx_misc;
103         u32 pllx_base;
104         u32 pllp_misc;
105         u32 pllp_base;
106         u32 pllp_outa;
107         u32 pllp_outb;
108         u32 pll_timeout;
109
110         u32 cpu_burst;
111         u32 clk_csite_src;
112         u32 cclk_divider;
113
114         u32 mc[3];
115
116         struct tegra_twd_context twd;
117 };
118 #define PMC_CTRL                0x0
119
120 #ifdef CONFIG_PM_SLEEP
121 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
122 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
123 #ifndef CONFIG_ARCH_TEGRA_21x_SOC
124 static int tegra_last_pclk;
125 #endif
126 static u64 resume_time;
127 static u64 resume_entry_time;
128 static u64 suspend_time;
129 static u64 suspend_entry_time;
130 #endif
131
132 static struct suspend_context tegra_sctx;
133
134 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
135 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
136 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
137 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
138 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
139 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
140 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
141 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
142 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
143
144 #define TEGRA_DPAD_ORIDE_SYS_CLK_REQ    (1 << 21)
145
146 #define PMC_CTRL                0x0
147 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
148 #define PMC_WAKE_MASK           0xc
149 #define PMC_WAKE_LEVEL          0x10
150 #define PMC_DPAD_ORIDE          0x1C
151 #define PMC_WAKE_DELAY          0xe0
152 #define PMC_DPD_SAMPLE          0x20
153 #define PMC_DPD_ENABLE          0x24
154 #define PMC_IO_DPD_REQ          0x1B8
155 #define PMC_IO_DPD2_REQ         0x1C0
156
157 #define PMC_CTRL2               0x440
158 #define PMC_ALLOW_PULSE_WAKE    BIT(14)
159
160 #define PMC_WAKE_STATUS         0x14
161 #define PMC_SW_WAKE_STATUS      0x18
162 #define PMC_COREPWRGOOD_TIMER   0x3c
163 #define PMC_CPUPWRGOOD_TIMER    0xc8
164 #define PMC_CPUPWROFF_TIMER     0xcc
165 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
166
167 #define PMC_PWRGATE_TOGGLE      0x30
168 #define PWRGATE_TOGGLE_START    (1 << 8)
169 #define UN_PWRGATE_CPU          \
170         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
171
172 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
173
174 #define CLK_RESET_CCLK_BURST    0x20
175 #define CLK_RESET_CCLK_DIVIDER  0x24
176 #define CLK_RESET_PLLC_BASE     0x80
177 #define CLK_RESET_PLLM_BASE     0x90
178 #define CLK_RESET_PLLX_BASE     0xe0
179 #define CLK_RESET_PLLX_MISC     0xe4
180 #define CLK_RESET_PLLP_BASE     0xa0
181 #define CLK_RESET_PLLP_OUTA     0xa4
182 #define CLK_RESET_PLLP_OUTB     0xa8
183 #define CLK_RESET_PLLP_MISC     0xac
184
185 #define CLK_RESET_SOURCE_CSITE  0x1d4
186
187 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
188 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
189 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
190 #define CLK_RESET_CCLK_IDLE_POLICY         1
191 #define CLK_RESET_CCLK_RUN_POLICY          2
192 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
193 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
194
195 #define MC_SECURITY_START       0x6c
196 #define MC_SECURITY_SIZE        0x70
197 #define MC_SECURITY_CFG2        0x7c
198
199 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
200 static struct clk *tegra_dfll;
201 #endif
202 static struct clk *tegra_pclk;
203 static struct clk *tegra_clk_m;
204 static struct tegra_suspend_platform_data *pdata;
205 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
206
207 bool tegra_is_dpd_mode = false;
208
209 bool tegra_dvfs_is_dfll_bypass(void)
210 {
211 #ifdef CONFIG_REGULATOR_TEGRA_DFLL_BYPASS
212         if (tegra_cpu_rail->dt_reg_pwm)
213                 return true;
214 #endif
215         return false;
216 }
217
218 static inline unsigned int is_slow_cluster(void)
219 {
220         int reg;
221
222         asm("mrs        %0, mpidr_el1\n"
223             "ubfx       %0, %0, #8, #4"
224             : "=r" (reg)
225             :
226             : "cc", "memory");
227
228         return reg & 1;
229 }
230
231 #ifdef CONFIG_PM_SLEEP
232
233 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
234         [TEGRA_SUSPEND_NONE]    = "none",
235         [TEGRA_SUSPEND_LP2]     = "lp2",
236         [TEGRA_SUSPEND_LP1]     = "lp1",
237         [TEGRA_SUSPEND_LP0]     = "lp0",
238 };
239
240 int tegra_is_lp0_suspend_mode(void)
241 {
242         return (current_suspend_mode == TEGRA_SUSPEND_LP0);
243 }
244
245 void tegra_log_resume_time(void)
246 {
247         u64 resume_end_time = tegra_read_usec_raw();
248
249         if (resume_entry_time > resume_end_time)
250                 resume_end_time |= 1ull<<32;
251         resume_time = resume_end_time - resume_entry_time;
252 }
253
254 void tegra_log_suspend_time(void)
255 {
256         suspend_entry_time = tegra_read_usec_raw();
257 }
258
259 static void tegra_get_suspend_time(void)
260 {
261         u64 suspend_end_time;
262         suspend_end_time = tegra_read_usec_raw();
263
264         if (suspend_entry_time > suspend_end_time)
265                 suspend_end_time |= 1ull<<32;
266         suspend_time = suspend_end_time - suspend_entry_time;
267 }
268
269 unsigned long tegra_cpu_power_good_time(void)
270 {
271         if (WARN_ON_ONCE(!pdata))
272                 return 5000;
273
274         return pdata->cpu_timer;
275 }
276
277 unsigned long tegra_cpu_power_off_time(void)
278 {
279         if (WARN_ON_ONCE(!pdata))
280                 return 5000;
281
282         return pdata->cpu_off_timer;
283 }
284
285 unsigned int tegra_cpu_suspend_freq(void)
286 {
287         if (WARN_ON_ONCE(!pdata))
288                 return 0;
289
290         return pdata->cpu_suspend_freq;
291 }
292
293 static void suspend_cpu_dfll_mode(unsigned int flags)
294 {
295 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
296         /* If DFLL is used as CPU clock source go to open loop mode */
297         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
298                 if (tegra_dfll && tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
299                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
300         }
301
302         /* Suspend dfll bypass (safe rail down) on LP or if DFLL is Not used */
303         if (pdata && pdata->suspend_dfll_bypass &&
304             (!tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail)))
305                 pdata->suspend_dfll_bypass();
306 #endif
307 }
308
309 static void resume_cpu_dfll_mode(unsigned int flags)
310 {
311 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
312         /* If DFLL is Not used and resume on G restore bypass mode */
313         if (pdata && pdata->resume_dfll_bypass &&
314             tegra_is_clk_initialized(tegra_dfll) &&
315             !tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
316                 pdata->resume_dfll_bypass();
317
318         /* If DFLL is used as CPU clock source restore closed loop mode */
319         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
320                 if (tegra_dfll && tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
321                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
322         }
323 #endif
324 }
325
326 /* ensures that sufficient time is passed for a register write to
327  * serialize into the 32KHz domain */
328 static void pmc_32kwritel(u32 val, unsigned long offs)
329 {
330         writel(val, pmc + offs);
331         udelay(130);
332 }
333
334 #if !defined(CONFIG_OF) || !defined(CONFIG_COMMON_CLK)
335 #if defined(CONFIG_ARCH_TEGRA_21x_SOC)
336 static void set_power_timers(unsigned long us_on, unsigned long us_off,
337                              long rate)
338 {
339         writel(0, pmc + PMC_CPUPWRGOOD_TIMER);
340         writel(0, pmc + PMC_CPUPWROFF_TIMER);
341         wmb();
342 }
343 #else
344 static void set_power_timers(unsigned long us_on, unsigned long us_off,
345                              long rate)
346 {
347         static unsigned long last_us_off;
348         unsigned long long ticks;
349         unsigned long long pclk;
350
351         if (WARN_ON_ONCE(rate <= 0))
352                 pclk = 100000000;
353         else
354                 pclk = rate;
355
356         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
357                 ticks = (us_on * pclk) + 999999ull;
358                 do_div(ticks, 1000000);
359                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
360
361                 ticks = (us_off * pclk) + 999999ull;
362                 do_div(ticks, 1000000);
363                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
364                 wmb();
365         }
366         tegra_last_pclk = pclk;
367         last_us_off = us_off;
368 }
369 #endif
370 #endif
371
372 /*
373  * restore_csite_clock
374  */
375 static void restore_csite_clock(void)
376 {
377         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
378 }
379
380 /*
381  * Clear flow controller registers
382  */
383 static void clear_flow_controller(void)
384 {
385         int cpu = cpu_logical_map(smp_processor_id());
386         unsigned int reg;
387
388         /* Do not power-gate CPU 0 when flow controlled */
389         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
390         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
391         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
392         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
393         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
394         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
395         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
396 }
397
398 /*
399  * saves pll state for use by restart_plls
400  */
401 static void save_pll_state(void)
402 {
403         /* switch coresite to clk_m, save off original source */
404         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
405         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
406
407         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
408         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
409         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
410         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
411         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
412         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
413         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
414         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
415 }
416
417 static int tegra_psci_suspend_cpu(long unsigned int val)
418 {
419         u32 state = (PSCI_POWER_STATE_TYPE_POWER_DOWN << 30) |
420                     TEGRA_ID_CPU_SUSPEND_LP0;
421         if (tegra_cpu_is_secure()) {
422                 if (psci_ops.cpu_suspend)
423                         psci_ops.cpu_suspend(state, __pa(cpu_resume));
424         }
425
426         return 0;
427 }
428
429 static void tegra_sleep_core(enum tegra_suspend_mode mode,
430                              unsigned long v2p)
431 {
432         if (tegra_cpu_is_secure()) {
433                 __flush_dcache_area(&tegra_resume_timestamps_start,
434                                         (&tegra_resume_timestamps_end -
435                                          &tegra_resume_timestamps_start));
436
437                 BUG_ON(mode != TEGRA_SUSPEND_LP0);
438
439                 trace_smc_sleep_core(NVSEC_SMC_START);
440                 cpu_suspend(v2p, tegra_psci_suspend_cpu);
441                 trace_smc_sleep_core(NVSEC_SMC_DONE);
442         }
443
444         tegra_get_suspend_time();
445 }
446
447 static int tegra_common_suspend(void)
448 {
449         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
450         u32 reg;
451
452         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
453         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
454         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
455
456         reg = readl(pmc + PMC_SCRATCH4);
457         if (is_slow_cluster())
458                 reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
459         else
460                 reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
461         writel(reg, pmc + PMC_SCRATCH4);
462
463         return 0;
464 }
465
466 static void tegra_common_resume(void)
467 {
468         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
469
470         /* Clear DPD Enable */
471         writel(0x0, pmc + PMC_DPD_ENABLE);
472
473         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
474         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
475         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
476         writel(0x0, pmc + PMC_SCRATCH41);
477 }
478
479 #if defined(CONFIG_ARCH_TEGRA_21x_SOC)
480 static void tegra_pm_sc7_set(void)
481 {
482         u32 reg;
483         unsigned long rate;
484
485         rate = clk_get_rate(tegra_pclk);
486
487         reg = readl(pmc + PMC_CTRL);
488         reg |= TEGRA_POWER_PWRREQ_OE;
489         if (pdata->combined_req)
490                 reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
491         else
492                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
493
494         reg &= ~TEGRA_POWER_EFFECT_LP0;
495
496         pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
497
498 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
499         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
500 #else
501         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
502 #endif
503
504         pmc_32kwritel(reg, PMC_CTRL);
505
506         /*
507          * This is to ensure the wake status is correctly captured and held
508          * correctly for all the wake.  This is necessitated due to a bug in
509          * the logic which had made the   E_INPUT control for specific pin
510          * i.e. GPIO_PA6 to Logic0 because of which wake event status didn't
511          * get latched in some corner cases.  Details are available in the
512          * bug 1495809
513          */
514         tegra_pmc_enable_wake_det(true);
515 }
516
517 int tegra_pm_prepare_sc7(void)
518 {
519         if (!tegra_pm_irq_lp0_allowed())
520                 return -EINVAL;
521
522         tegra_common_suspend();
523
524         tegra_pm_sc7_set();
525
526         tegra_tsc_suspend();
527
528         tegra_lp0_suspend_mc();
529
530         tegra_tsc_wait_for_suspend();
531
532         save_pll_state();
533
534         return 0;
535 }
536
537 int tegra_pm_post_sc7(void)
538 {
539         u32 reg;
540
541         tegra_tsc_resume();
542
543         tegra_lp0_resume_mc();
544
545         tegra_tsc_wait_for_resume();
546
547         restore_csite_clock();
548
549         /* for platforms where the core & CPU power requests are
550          * combined as a single request to the PMU, transition out
551          * of LP0 state by temporarily enabling both requests
552          */
553         if (pdata->combined_req) {
554                 reg = readl(pmc + PMC_CTRL);
555                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
556                 pmc_32kwritel(reg, PMC_CTRL);
557                 reg &= ~TEGRA_POWER_PWRREQ_OE;
558                 pmc_32kwritel(reg, PMC_CTRL);
559         }
560
561         tegra_common_resume();
562
563         return 0;
564 }
565 #endif /* CONFIG_ARCH_TEGRA_21x_SOC */
566
567 static void tegra_pm_set(enum tegra_suspend_mode mode)
568 {
569         u32 reg, boot_flag;
570         unsigned long rate = 32768;
571
572         reg = readl(pmc + PMC_CTRL);
573         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
574         if (pdata->combined_req)
575                 reg &= ~TEGRA_POWER_PWRREQ_OE;
576         else
577                 reg |= TEGRA_POWER_PWRREQ_OE;
578         reg &= ~TEGRA_POWER_EFFECT_LP0;
579
580         switch (mode) {
581         case TEGRA_SUSPEND_LP0:
582                 if (pdata->combined_req) {
583                         reg |= TEGRA_POWER_PWRREQ_OE;
584                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
585                 }
586
587                 /* Enable DPD sample to trigger sampling pads data and direction
588                  * in which pad will be driven during lp0 mode*/
589                 writel(0x1, pmc + PMC_DPD_SAMPLE);
590                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
591                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
592                 tegra_is_dpd_mode = true;
593
594                 /* Set warmboot flag */
595                 boot_flag = readl(pmc + PMC_SCRATCH0);
596                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
597
598                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
599
600                 reg |= TEGRA_POWER_EFFECT_LP0;
601
602                 /* No break here. LP0 code falls through to write SCRATCH41 */
603         case TEGRA_SUSPEND_LP1:
604                 __raw_writel(virt_to_phys(cpu_resume), pmc + PMC_SCRATCH41);
605                 wmb();
606                 rate = clk_get_rate(tegra_clk_m);
607                 break;
608         case TEGRA_SUSPEND_LP2:
609                 rate = clk_get_rate(tegra_pclk);
610                 break;
611         case TEGRA_SUSPEND_NONE:
612                 return;
613         default:
614                 BUG();
615         }
616
617 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
618         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
619 #else
620         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
621 #endif
622
623         pmc_32kwritel(reg, PMC_CTRL);
624 }
625
626 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
627         [TEGRA_SUSPEND_NONE] = "none",
628         [TEGRA_SUSPEND_LP2] = "LP2",
629         [TEGRA_SUSPEND_LP1] = "LP1",
630         [TEGRA_SUSPEND_LP0] = "LP0",
631 };
632
633 #if defined(CONFIG_ARCH_TEGRA_21x_SOC)
634
635 static void tegra_bpmp_enable_suspend(int mode, int flags)
636 {
637         int32_t mb[] = { cpu_to_le32(mode), cpu_to_le32(flags) };
638         int r = tegra_bpmp_send(MRQ_ENABLE_SUSPEND, &mb, sizeof(mb));
639         WARN_ON(r);
640 }
641
642 static int tegra210_suspend_dram(enum tegra_suspend_mode mode)
643 {
644         int err = 0;
645         u32 power_state = (PSCI_POWER_STATE_TYPE_POWER_DOWN << 30);
646
647         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
648                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
649                 err = -ENXIO;
650                 goto fail;
651         }
652
653         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
654                 err = -ENXIO;
655                 goto fail;
656         }
657
658         if (mode == TEGRA_SUSPEND_LP1) {
659                 tegra_bpmp_enable_suspend(TEGRA_PM_SC4, 0);
660
661                 trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
662                 tegra_get_suspend_time();
663
664                 power_state |= TEGRA210_CPUIDLE_CC7;
665                 cpu_suspend(power_state, NULL);
666
667                 resume_entry_time = tegra_read_usec_raw();
668                 trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
669
670                 return err;
671         }
672
673         tegra_bpmp_enable_suspend(TEGRA_PM_SC7, 0);
674         tegra_pm_prepare_sc7();
675         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
676         tegra_get_suspend_time();
677
678         power_state |= TEGRA210_CPUIDLE_SC7;
679         cpu_suspend(power_state, NULL);
680
681         resume_entry_time = tegra_read_usec_raw();
682         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
683         tegra_pm_post_sc7();
684
685 fail:
686         return err;
687 }
688 #endif /* CONFIG_ARCH_TEGRA_21x_SOC */
689
690 static int tegra_suspend_enter(suspend_state_t state)
691 {
692         int ret = 0;
693         ktime_t delta;
694         struct timespec ts_entry, ts_exit;
695
696         if (pdata && pdata->board_suspend)
697                 pdata->board_suspend(current_suspend_mode,
698                         TEGRA_SUSPEND_BEFORE_PERIPHERAL);
699
700         read_persistent_clock(&ts_entry);
701
702 #ifdef CONFIG_ARCH_TEGRA_21x_SOC
703         ret = tegra210_suspend_dram(current_suspend_mode);
704 #else
705         ret = tegra_suspend_dram(current_suspend_mode, 0);
706 #endif
707         if (ret) {
708                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
709                 goto abort_suspend;
710         }
711
712         read_persistent_clock(&ts_exit);
713
714         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
715                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
716
717                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
718                 tegra_dvfs_rail_pause(tegra_gpu_rail, delta, false);
719                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
720                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
721                 else
722                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
723         }
724
725 abort_suspend:
726         if (pdata && pdata->board_resume)
727                 pdata->board_resume(current_suspend_mode,
728                         TEGRA_RESUME_AFTER_PERIPHERAL);
729
730         return ret;
731 }
732
733 static void tegra_suspend_check_pwr_stats(void)
734 {
735         /* cpus and l2 are powered off later */
736         unsigned long pwrgate_partid_mask =
737                 (1 << TEGRA_POWERGATE_HEG)      |
738                 (1 << TEGRA_POWERGATE_SATA)     |
739                 (1 << TEGRA_POWERGATE_3D1)      |
740                 (1 << TEGRA_POWERGATE_3D)       |
741                 (1 << TEGRA_POWERGATE_VENC)     |
742                 (1 << TEGRA_POWERGATE_PCIE)     |
743                 (1 << TEGRA_POWERGATE_VDEC)     |
744                 (1 << TEGRA_POWERGATE_MPE);
745
746         int partid;
747
748         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
749                 if ((1 << partid) & pwrgate_partid_mask)
750                         if (tegra_powergate_is_powered(partid))
751                                 pr_debug("partition %s is left on before suspend\n",
752                                         tegra_powergate_get_name(partid));
753
754         return;
755 }
756
757 static void tegra_suspend_powergate_control(int partid, bool turn_off)
758 {
759         if (turn_off)
760                 tegra_powergate_partition(partid);
761         else
762                 tegra_unpowergate_partition(partid);
763 }
764
765 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
766 {
767         int err = 0;
768         u32 scratch37 = 0xDEADBEEF;
769         u32 reg;
770
771         bool tegra_suspend_vde_powergated = false;
772
773         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
774                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
775                 err = -ENXIO;
776                 goto fail;
777         }
778
779         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
780                 pr_info("LP0 not used due to unsupported wakeup events\n");
781                 mode = TEGRA_SUSPEND_LP1;
782         }
783
784         /* turn off VDE partition in LP1 */
785         if (mode == TEGRA_SUSPEND_LP1 &&
786                 tegra_powergate_is_powered(TEGRA_POWERGATE_VDEC)) {
787                 pr_info("turning off partition %s in LP1\n",
788                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
789                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, true);
790                 tegra_suspend_vde_powergated = true;
791         }
792
793         tegra_common_suspend();
794
795         tegra_pm_set(mode);
796
797         if (pdata && pdata->board_suspend)
798                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
799
800         local_fiq_disable();
801
802         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
803
804         if (mode == TEGRA_SUSPEND_LP0) {
805                 tegra_tsc_suspend();
806                 tegra_lp0_suspend_mc();
807                 tegra_tsc_wait_for_suspend();
808         }
809
810         cpu_cluster_pm_enter();
811         save_pll_state();
812
813         if (tegra_get_chipid() != TEGRA_CHIPID_TEGRA13)
814                 flush_cache_all();
815
816         tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
817
818         resume_entry_time = 0;
819         if (mode != TEGRA_SUSPEND_LP0)
820                 resume_entry_time = tegra_read_usec_raw();
821
822         if (mode == TEGRA_SUSPEND_LP0) {
823                 tegra_tsc_resume();
824                 tegra_lp0_resume_mc();
825                 tegra_tsc_wait_for_resume();
826         }
827
828         /* if scratch37 was clobbered during LP1, restore it */
829         if (scratch37 != 0xDEADBEEF)
830                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
831
832         restore_csite_clock();
833         clear_flow_controller();
834         cpu_cluster_pm_exit();
835
836         /* for platforms where the core & CPU power requests are
837          * combined as a single request to the PMU, transition out
838          * of LP0 state by temporarily enabling both requests
839          */
840         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
841                 reg = readl(pmc + PMC_CTRL);
842                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
843                 pmc_32kwritel(reg, PMC_CTRL);
844                 reg &= ~TEGRA_POWER_PWRREQ_OE;
845                 pmc_32kwritel(reg, PMC_CTRL);
846         }
847
848         if (pdata && pdata->board_resume)
849                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
850
851         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
852
853         local_fiq_enable();
854
855         tegra_common_resume();
856
857         /* turn on VDE partition in LP1 */
858         if (mode == TEGRA_SUSPEND_LP1 && tegra_suspend_vde_powergated) {
859                 pr_info("turning on partition %s in LP1\n",
860                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
861                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, false);
862         }
863
864 fail:
865         return err;
866 }
867
868 static int tegra_suspend_valid(suspend_state_t state)
869 {
870         int valid = 1;
871
872         /* LP0 is not supported on T132 A01 silicon */
873         if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA13 && \
874                 tegra_revision == TEGRA_REVISION_A01)
875                 valid = 0;
876
877         return valid;
878 }
879
880 static int tegra_suspend_prepare_late(void)
881 {
882         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) ||
883                         (current_suspend_mode == TEGRA_SUSPEND_LP1))
884                 tegra_suspend_check_pwr_stats();
885
886         return 0;
887 }
888
889 static void tegra_suspend_finish(void)
890 {
891         if (pdata && pdata->cpu_resume_boost) {
892                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
893                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
894                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
895         }
896 }
897
898 static const struct platform_suspend_ops tegra_suspend_ops = {
899         .prepare_late   = tegra_suspend_prepare_late,
900         .valid          = tegra_suspend_valid,
901         .finish         = tegra_suspend_finish,
902         .enter          = tegra_suspend_enter,
903 };
904
905 static ssize_t suspend_mode_show(struct kobject *kobj,
906                                         struct kobj_attribute *attr, char *buf)
907 {
908         char *start = buf;
909         char *end = buf + PAGE_SIZE;
910
911         start += scnprintf(start, end - start, "%s ",
912                                 tegra_suspend_name[current_suspend_mode]);
913         start += scnprintf(start, end - start, "\n");
914
915         return start - buf;
916 }
917
918 static ssize_t suspend_mode_store(struct kobject *kobj,
919                                         struct kobj_attribute *attr,
920                                         const char *buf, size_t n)
921 {
922         int len;
923         const char *name_ptr;
924         enum tegra_suspend_mode new_mode;
925
926         name_ptr = buf;
927         while (*name_ptr && !isspace(*name_ptr))
928                 name_ptr++;
929         len = name_ptr - buf;
930         if (!len)
931                 goto bad_name;
932         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
933         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
934                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
935                 pr_info("Illegal tegra suspend state: %s\n", buf);
936                 goto bad_name;
937         }
938
939         for (new_mode = TEGRA_SUSPEND_NONE;
940                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
941                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
942                         current_suspend_mode = new_mode;
943                         break;
944                 }
945         }
946
947 bad_name:
948         return n;
949 }
950
951 static struct kobj_attribute suspend_mode_attribute =
952         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
953
954 static ssize_t suspend_resume_time_show(struct kobject *kobj,
955                                         struct kobj_attribute *attr,
956                                         char *buf)
957 {
958         return sprintf(buf, "%ums\n", ((u32)resume_time / 1000));
959 }
960
961 static struct kobj_attribute suspend_resume_time_attribute =
962         __ATTR(resume_time, 0444, suspend_resume_time_show, NULL);
963
964 static ssize_t suspend_time_show(struct kobject *kobj,
965                                         struct kobj_attribute *attr,
966                                         char *buf)
967 {
968         return sprintf(buf, "%ums\n", ((u32)suspend_time / 1000));
969 }
970
971 static struct kobj_attribute suspend_time_attribute =
972         __ATTR(suspend_time, 0444, suspend_time_show, NULL);
973
974 static struct kobject *suspend_kobj;
975
976 static int tegra_pm_enter_suspend(void)
977 {
978         suspend_cpu_dfll_mode(0);
979         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
980         return 0;
981 }
982
983 static void tegra_pm_enter_resume(void)
984 {
985         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
986         resume_cpu_dfll_mode(0);
987 }
988
989 static void tegra_pm_enter_shutdown(void)
990 {
991         suspend_cpu_dfll_mode(0);
992         pr_info("Shutting down tegra ...\n");
993 }
994
995 static struct syscore_ops tegra_pm_enter_syscore_ops = {
996         .suspend = tegra_pm_enter_suspend,
997         .resume = tegra_pm_enter_resume,
998         .save = tegra_pm_enter_suspend,
999         .restore = tegra_pm_enter_resume,
1000         .shutdown = tegra_pm_enter_shutdown,
1001 };
1002
1003 static __init int tegra_pm_enter_syscore_init(void)
1004 {
1005         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1006         return 0;
1007 }
1008 subsys_initcall(tegra_pm_enter_syscore_init);
1009 #endif
1010
1011 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1012 {
1013         u32 reg;
1014         u32 mode;
1015         struct pmc_pm_data *pm_dat;
1016         bool is_board_pdata = true;
1017
1018 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1019         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1020         BUG_ON(IS_ERR(tegra_dfll));
1021 #endif
1022         tegra_pclk = clk_get_sys(NULL, "pclk");
1023         BUG_ON(IS_ERR(tegra_pclk));
1024         tegra_clk_m = clk_get_sys(NULL, "clk_m");
1025         BUG_ON(IS_ERR(tegra_clk_m));
1026
1027         /* create the pdata from DT information */
1028         pm_dat = tegra_get_pm_data();
1029         if (pm_dat) {
1030                 pr_err("PMC dt information non-NULL %s\n", __func__);
1031                 is_board_pdata = false;
1032                 pdata = kzalloc(sizeof(struct tegra_suspend_platform_data),
1033                         GFP_KERNEL);
1034                 if (pm_dat->combined_req != plat->combined_req) {
1035                         pr_debug("PMC DT attribute combined_req=%d, board value=%d\n",
1036                                 pm_dat->combined_req, plat->combined_req);
1037                         pdata->combined_req = plat->combined_req;
1038                 } else {
1039                         pdata->combined_req = pm_dat->combined_req;
1040                 }
1041                 if (pm_dat->sysclkreq_high != plat->sysclkreq_high) {
1042                         pr_debug("PMC DT attribute sysclkreq_high=%d, board value=%d\n",
1043                                 pm_dat->sysclkreq_high, plat->sysclkreq_high);
1044                         pdata->sysclkreq_high = plat->sysclkreq_high;
1045                 } else {
1046                         pdata->sysclkreq_high = pm_dat->sysclkreq_high;
1047                 }
1048                 if (pm_dat->corereq_high != plat->corereq_high) {
1049                         pr_debug("PMC DT attribute corereq_high=%d, board value=%d\n",
1050                                 pm_dat->corereq_high, plat->corereq_high);
1051                         pdata->corereq_high = plat->corereq_high;
1052                 } else {
1053                         pdata->corereq_high = pm_dat->corereq_high;
1054                 }
1055                 if (pm_dat->cpu_off_time != plat->cpu_off_timer) {
1056                         pr_debug("PMC DT attribute cpu_off_timer=%d, board value=%ld\n",
1057                                 pm_dat->cpu_off_time, plat->cpu_off_timer);
1058                         pdata->cpu_off_timer = plat->cpu_off_timer;
1059                 } else {
1060                         pdata->cpu_off_timer = pm_dat->cpu_off_time;
1061                 }
1062                 if (pm_dat->cpu_good_time != plat->cpu_timer) {
1063                         pr_debug("PMC DT attribute cpu_timer=%d, board value=%ld\n",
1064                                 pm_dat->cpu_good_time, plat->cpu_timer);
1065                         pdata->cpu_timer = plat->cpu_timer;
1066                 } else {
1067                         pdata->cpu_timer = pm_dat->cpu_good_time;
1068                 }
1069                 if (pm_dat->suspend_mode != plat->suspend_mode) {
1070                         pr_debug("PMC DT attribute suspend_mode=%d, board value=%d\n",
1071                                 pm_dat->suspend_mode, plat->suspend_mode);
1072                         pdata->suspend_mode = plat->suspend_mode;
1073                 } else {
1074                         pdata->suspend_mode = pm_dat->suspend_mode;
1075                 }
1076                 if (pm_dat->cpu_suspend_freq != plat->cpu_suspend_freq) {
1077                         pr_debug("PMC DT attribute cpu_suspend_freq=%d, cpu_suspend_freq=%d\n",
1078                                 pm_dat->cpu_suspend_freq,
1079                                 plat->cpu_suspend_freq);
1080                         pdata->cpu_suspend_freq = plat->cpu_suspend_freq;
1081                 } else {
1082                         pdata->cpu_suspend_freq = pm_dat->cpu_suspend_freq;
1083                 }
1084
1085                 /* FIXME: pmc_pm_data fields to be reused
1086                  *      core_osc_time, core_pmu_time, core_off_time
1087                  *      units of above fields is uSec while
1088                  *      platform data values are in ticks
1089                  */
1090                 /* FIXME: pmc_pm_data unused by downstream code
1091                  *      cpu_pwr_good_en, lp0_vec_size, lp0_vec_phy_addr
1092                  */
1093                 /* FIXME: add missing DT bindings taken from platform data */
1094                 pdata->core_timer = plat->core_timer;
1095                 pdata->core_off_timer = plat->core_off_timer;
1096                 pdata->board_suspend = plat->board_suspend;
1097                 pdata->board_resume = plat->board_resume;
1098                 pdata->sysclkreq_gpio = plat->sysclkreq_gpio;
1099                 pdata->cpu_lp2_min_residency = plat->cpu_lp2_min_residency;
1100                 pdata->cpu_resume_boost = plat->cpu_resume_boost;
1101 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
1102                 pdata->min_residency_vmin_fmin = plat->min_residency_vmin_fmin;
1103                 pdata->min_residency_ncpu_slow = plat->min_residency_ncpu_slow;
1104                 pdata->min_residency_ncpu_fast = plat->min_residency_ncpu_fast;
1105                 pdata->min_residency_crail = plat->min_residency_crail;
1106                 pdata->crail_up_early = plat->crail_up_early;
1107 #endif
1108                 pdata->min_residency_mclk_stop = plat->min_residency_mclk_stop;
1109                 pdata->usb_vbus_internal_wake = plat->usb_vbus_internal_wake;
1110                 pdata->usb_id_internal_wake = plat->usb_id_internal_wake;
1111                 pdata->suspend_dfll_bypass = plat->suspend_dfll_bypass;
1112                 pdata->resume_dfll_bypass = plat->resume_dfll_bypass;
1113         } else {
1114                 pr_err("PMC board data used in %s\n", __func__);
1115                 pdata = plat;
1116         }
1117         (void)reg;
1118         (void)mode;
1119
1120         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1121                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1122
1123 #ifndef CONFIG_PM_SLEEP
1124         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1125                 pr_warn("%s: Suspend requires CONFIG_PM_SLEEP -- "
1126                            "disabling suspend\n", __func__);
1127                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1128         }
1129 #else
1130
1131         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1132                 tegra_lp0_vec_relocate) {
1133                 unsigned char *reloc_lp0;
1134                 unsigned long tmp;
1135                 void __iomem *orig;
1136                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1137                                         GFP_KERNEL);
1138                 WARN_ON(!reloc_lp0);
1139                 if (!reloc_lp0) {
1140                         pr_err("%s: Failed to allocate reloc_lp0\n",
1141                                 __func__);
1142                         goto out;
1143                 }
1144
1145                 /* Avoid a kmemleak false positive. The allocated memory
1146                  * block is later referenced by a physical address (i.e.
1147                  * tegra_lp0_vec_start) which kmemleak can't detect.
1148                  */
1149                 kmemleak_not_leak(reloc_lp0);
1150
1151                 orig = ioremap_wc(tegra_lp0_vec_start, tegra_lp0_vec_size);
1152                 WARN_ON(!orig);
1153                 if (!orig) {
1154                         pr_err("%s: Failed to map tegra_lp0_vec_start %llx\n",
1155                                 __func__, tegra_lp0_vec_start);
1156                         kfree(reloc_lp0);
1157                         goto out;
1158                 }
1159
1160                 tmp = (unsigned long) reloc_lp0;
1161                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1162                 reloc_lp0 = (unsigned char *)tmp;
1163                 memcpy(reloc_lp0, (const void *)(uintptr_t)orig,
1164                         tegra_lp0_vec_size);
1165                 wmb();
1166                 iounmap(orig);
1167                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1168         }
1169
1170 out:
1171         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1172                 pr_warn("%s: Suspend mode LP0 requested, no lp0_vec "
1173                            "provided by bootlader -- disabling LP0\n",
1174                            __func__);
1175                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1176         }
1177
1178         /* Initialize SOC-specific data */
1179 #ifdef CONFIG_ARCH_TEGRA_13x_SOC
1180         tegra_soc_suspend_init();
1181
1182         current_suspend_mode = TEGRA_SUSPEND_LP0;
1183 #endif
1184
1185 #ifdef CONFIG_ARCH_TEGRA_21x_SOC
1186         reg = readl(pmc + PMC_CTRL2);
1187         reg |= PMC_ALLOW_PULSE_WAKE;
1188         pmc_32kwritel(reg, PMC_CTRL2);
1189 #endif
1190
1191         /* Always enable CPU power request; just normal polarity is supported */
1192         reg = readl(pmc + PMC_CTRL);
1193         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1194         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1195         pmc_32kwritel(reg, PMC_CTRL);
1196
1197         /* Configure core power request and system clock control if LP0
1198            is supported */
1199         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1200         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1201
1202         reg = readl(pmc + PMC_CTRL);
1203
1204         if (!pdata->sysclkreq_high)
1205                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1206         else
1207                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1208
1209         if (!pdata->corereq_high)
1210                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1211         else
1212                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1213
1214         /* configure output inverters while the request is tristated */
1215         pmc_32kwritel(reg, PMC_CTRL);
1216
1217         /* now enable requests */
1218         reg |= TEGRA_POWER_SYSCLK_OE;
1219         if (!pdata->combined_req)
1220                 reg |= TEGRA_POWER_PWRREQ_OE;
1221         pmc_32kwritel(reg, PMC_CTRL);
1222
1223         if (pdata->sysclkreq_gpio) {
1224                 reg = readl(pmc + PMC_DPAD_ORIDE);
1225                 reg &= ~TEGRA_DPAD_ORIDE_SYS_CLK_REQ;
1226                 pmc_32kwritel(reg, PMC_DPAD_ORIDE);
1227         }
1228
1229         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1230                 tegra_lp0_suspend_init();
1231
1232         suspend_set_ops(&tegra_suspend_ops);
1233
1234         /* Create /sys/power/suspend/type */
1235         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1236         if (suspend_kobj) {
1237                 if (sysfs_create_file(suspend_kobj,
1238                                                 &suspend_mode_attribute.attr))
1239                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1240                                                                 __func__);
1241                 if (sysfs_create_file(suspend_kobj,
1242                                         &suspend_resume_time_attribute.attr))
1243                         pr_err("%s: sysfs_create_file resume_time failed!\n",
1244                                                                 __func__);
1245                 if (sysfs_create_file(suspend_kobj,
1246                                         &suspend_time_attribute.attr))
1247                         pr_err("%s: sysfs_create_file suspend_time failed!\n",
1248                                                                 __func__);
1249         }
1250
1251 #endif
1252         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1253                 tegra_pd_in_idle(false);
1254
1255         current_suspend_mode = plat->suspend_mode;
1256 }
1257
1258 #ifdef CONFIG_ARM_ARCH_TIMER
1259
1260 static u32 tsc_suspend_start;
1261 static u32 tsc_resume_start;
1262
1263 #define pmc_writel(value, reg) \
1264                 writel(value, pmc + (reg))
1265 #define pmc_readl(reg) \
1266                 readl(pmc + (reg))
1267
1268 #define PMC_DPD_ENABLE_TSC_MULT_ENABLE  (1 << 1)
1269 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
1270 #define PMC_DPD_ENABLE_ON               (1 << 0)
1271 #endif
1272
1273 #define PMC_TSC_MULT                    0x2b4
1274 #define PMC_TSC_MULT_FREQ_STS           (1 << 16)
1275
1276 #define TSC_TIMEOUT_US                  32
1277
1278 void tegra_tsc_suspend(void)
1279 {
1280         u32 reg = pmc_readl(PMC_DPD_ENABLE);
1281         BUG_ON(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE);
1282         reg |= PMC_DPD_ENABLE_TSC_MULT_ENABLE;
1283         pmc_writel(reg, PMC_DPD_ENABLE);
1284         tsc_suspend_start = tegra_read_usec_raw();
1285 }
1286
1287 void tegra_tsc_resume(void)
1288 {
1289         u32 reg = pmc_readl(PMC_DPD_ENABLE);
1290         BUG_ON(!(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE));
1291         reg &= ~PMC_DPD_ENABLE_TSC_MULT_ENABLE;
1292 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
1293         /*
1294          * FIXME: T12x SW WAR -
1295          * Resume ensures DPD_ENABLE is 0 when writing
1296          * TSC_MULT_ENABLE, else PMC wake status gets reset
1297          */
1298         reg &= ~PMC_DPD_ENABLE_ON;
1299 #endif
1300         pmc_writel(reg, PMC_DPD_ENABLE);
1301         tsc_resume_start = tegra_read_usec_raw();
1302 }
1303
1304 void tegra_tsc_wait_for_suspend(void)
1305 {
1306         while ((tegra_read_usec_raw() - tsc_suspend_start) <
1307                 TSC_TIMEOUT_US) {
1308                 if (pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS)
1309                         break;
1310                 cpu_relax();
1311         }
1312 }
1313
1314 void tegra_tsc_wait_for_resume(void)
1315 {
1316         while ((tegra_read_usec_raw() - tsc_resume_start) <
1317                 TSC_TIMEOUT_US) {
1318                 if (!(pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS))
1319                         break;
1320                 cpu_relax();
1321         }
1322 }
1323 #endif