787dfd54723dfa920c4dcf5adbf24682e62a8583
[linux-3.10.git] / arch / arm64 / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2013-2015, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/cpumask.h>
32 #include <linux/delay.h>
33 #include <linux/cpu_pm.h>
34 #include <linux/err.h>
35 #include <linux/debugfs.h>
36 #include <linux/delay.h>
37 #include <linux/suspend.h>
38 #include <linux/serial_reg.h>
39 #include <linux/seq_file.h>
40 #include <linux/uaccess.h>
41 #include <linux/syscore_ops.h>
42 #include <linux/cpu_pm.h>
43 #include <linux/clk/tegra.h>
44 #include <linux/export.h>
45 #include <linux/vmalloc.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48 #include <linux/clk/tegra.h>
49 #include <linux/tegra-powergate.h>
50 #include <linux/tegra-soc.h>
51 #include <linux/tegra-timer.h>
52 #include <linux/tegra-cpuidle.h>
53 #include <linux/irqchip/tegra.h>
54 #include <linux/tegra-pm.h>
55 #include <linux/tegra_pm_domains.h>
56 #include <linux/tegra_smmu.h>
57 #include <soc/tegra/tegra_bpmp.h>
58 #include <linux/kmemleak.h>
59
60 #include <trace/events/power.h>
61 #include <trace/events/nvsecurity.h>
62 #include <linux/tegra-pmc.h>
63
64 #include <asm/cacheflush.h>
65 #include <asm/idmap.h>
66 #include <asm/pgalloc.h>
67 #include <asm/pgtable.h>
68 #include <asm/tlbflush.h>
69 #include <asm/suspend.h>
70 #include <asm/smp_plat.h>
71 #include <asm/psci.h>
72
73 #include <mach/irqs.h>
74 #include <mach/dc.h>
75
76 #include "board.h"
77 #include <linux/platform/tegra/clock.h>
78 #include <linux/platform/tegra/common.h>
79 #include "iomap.h"
80 #include "pm.h"
81 #include "pm-soc.h"
82 #include <linux/platform/tegra/reset.h>
83 #include "sleep.h"
84 #include <linux/platform/tegra/dvfs.h>
85 #include <linux/platform/tegra/cpu-tegra.h>
86 #include <linux/platform/tegra/flowctrl.h>
87
88 #include "pm-tegra132.h"
89
90 /* core power request enable */
91 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)
92 /* enter LP0 when CPU pwr gated */
93 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)
94 /* CPU pwr req enable */
95 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)
96
97 struct suspend_context {
98         /*
99          * The next 7 values are referenced by offset in __restart_plls
100          * in headsmp-t2.S, and should not be moved
101          */
102         u32 pllx_misc;
103         u32 pllx_base;
104         u32 pllp_misc;
105         u32 pllp_base;
106         u32 pllp_outa;
107         u32 pllp_outb;
108         u32 pll_timeout;
109
110         u32 cpu_burst;
111         u32 clk_csite_src;
112         u32 cclk_divider;
113
114         u32 mc[3];
115
116         struct tegra_twd_context twd;
117 };
118 #define PMC_CTRL                0x0
119
120 #ifdef CONFIG_PM_SLEEP
121 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
122 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
123 #ifndef CONFIG_ARCH_TEGRA_21x_SOC
124 static int tegra_last_pclk;
125 #endif
126 static u64 resume_time;
127 static u64 resume_entry_time;
128 static u64 suspend_time;
129 static u64 suspend_entry_time;
130 #endif
131
132 static struct suspend_context tegra_sctx;
133
134 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
135 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
136 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
137 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
138 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
139 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
140 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
141 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
142 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
143
144 #define TEGRA_DPAD_ORIDE_SYS_CLK_REQ    (1 << 21)
145
146 #define PMC_CTRL                0x0
147 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
148 #define PMC_WAKE_MASK           0xc
149 #define PMC_WAKE_LEVEL          0x10
150 #define PMC_DPAD_ORIDE          0x1C
151 #define PMC_WAKE_DELAY          0xe0
152 #define PMC_DPD_SAMPLE          0x20
153 #define PMC_DPD_ENABLE          0x24
154 #define PMC_IO_DPD_REQ          0x1B8
155 #define PMC_IO_DPD2_REQ         0x1C0
156
157 #define PMC_CTRL2               0x440
158 #define PMC_ALLOW_PULSE_WAKE    BIT(14)
159
160 #define PMC_WAKE_STATUS         0x14
161 #define PMC_SW_WAKE_STATUS      0x18
162 #define PMC_COREPWRGOOD_TIMER   0x3c
163 #define PMC_CPUPWRGOOD_TIMER    0xc8
164 #define PMC_CPUPWROFF_TIMER     0xcc
165 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
166
167 #define PMC_PWRGATE_TOGGLE      0x30
168 #define PWRGATE_TOGGLE_START    (1 << 8)
169 #define UN_PWRGATE_CPU          \
170         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
171
172 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
173
174 #define CLK_RESET_CCLK_BURST    0x20
175 #define CLK_RESET_CCLK_DIVIDER  0x24
176 #define CLK_RESET_PLLC_BASE     0x80
177 #define CLK_RESET_PLLM_BASE     0x90
178 #define CLK_RESET_PLLX_BASE     0xe0
179 #define CLK_RESET_PLLX_MISC     0xe4
180 #define CLK_RESET_PLLP_BASE     0xa0
181 #define CLK_RESET_PLLP_OUTA     0xa4
182 #define CLK_RESET_PLLP_OUTB     0xa8
183 #define CLK_RESET_PLLP_MISC     0xac
184
185 #define CLK_RESET_SOURCE_CSITE  0x1d4
186
187 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
188 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
189 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
190 #define CLK_RESET_CCLK_IDLE_POLICY         1
191 #define CLK_RESET_CCLK_RUN_POLICY          2
192 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
193 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
194
195 #define MC_SECURITY_START       0x6c
196 #define MC_SECURITY_SIZE        0x70
197 #define MC_SECURITY_CFG2        0x7c
198
199 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
200 static struct clk *tegra_dfll;
201 #endif
202 static struct clk *tegra_pclk;
203 static struct clk *tegra_clk_m;
204 static struct tegra_suspend_platform_data *pdata;
205 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
206
207 bool tegra_is_dpd_mode = false;
208
209 bool tegra_dvfs_is_dfll_bypass(void)
210 {
211 #ifdef CONFIG_REGULATOR_TEGRA_DFLL_BYPASS
212         if (tegra_cpu_rail->dt_reg_pwm)
213                 return true;
214 #endif
215         return false;
216 }
217
218 static inline unsigned int is_slow_cluster(void)
219 {
220         int reg;
221
222         asm("mrs        %0, mpidr_el1\n"
223             "ubfx       %0, %0, #8, #4"
224             : "=r" (reg)
225             :
226             : "cc", "memory");
227
228         return reg & 1;
229 }
230
231 #ifdef CONFIG_PM_SLEEP
232
233 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
234         [TEGRA_SUSPEND_NONE]    = "none",
235         [TEGRA_SUSPEND_LP2]     = "lp2",
236         [TEGRA_SUSPEND_LP1]     = "lp1",
237         [TEGRA_SUSPEND_LP0]     = "lp0",
238 };
239
240 int tegra_is_lp0_suspend_mode(void)
241 {
242         return (current_suspend_mode == TEGRA_SUSPEND_LP0);
243 }
244
245 void tegra_log_resume_time(void)
246 {
247         u64 resume_end_time = tegra_read_usec_raw();
248
249         if (resume_entry_time > resume_end_time)
250                 resume_end_time |= 1ull<<32;
251         resume_time = resume_end_time - resume_entry_time;
252 }
253
254 void tegra_log_suspend_time(void)
255 {
256         suspend_entry_time = tegra_read_usec_raw();
257 }
258
259 static void tegra_get_suspend_time(void)
260 {
261         u64 suspend_end_time;
262         suspend_end_time = tegra_read_usec_raw();
263
264         if (suspend_entry_time > suspend_end_time)
265                 suspend_end_time |= 1ull<<32;
266         suspend_time = suspend_end_time - suspend_entry_time;
267 }
268
269 unsigned long tegra_cpu_power_good_time(void)
270 {
271         if (WARN_ON_ONCE(!pdata))
272                 return 5000;
273
274         return pdata->cpu_timer;
275 }
276
277 unsigned long tegra_cpu_power_off_time(void)
278 {
279         if (WARN_ON_ONCE(!pdata))
280                 return 5000;
281
282         return pdata->cpu_off_timer;
283 }
284
285 unsigned int tegra_cpu_suspend_freq(void)
286 {
287         if (WARN_ON_ONCE(!pdata))
288                 return 0;
289
290         return pdata->cpu_suspend_freq;
291 }
292
293 static void suspend_cpu_dfll_mode(unsigned int flags)
294 {
295 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
296         /* If DFLL is used as CPU clock source go to open loop mode */
297         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
298                 if (tegra_dfll && tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
299                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
300         }
301
302         /* Suspend dfll bypass (safe rail down) on LP or if DFLL is Not used */
303         if (pdata && pdata->suspend_dfll_bypass &&
304             (!tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail)))
305                 pdata->suspend_dfll_bypass();
306 #endif
307 }
308
309 static void resume_cpu_dfll_mode(unsigned int flags)
310 {
311 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
312         /* If DFLL is Not used and resume on G restore bypass mode */
313         if (pdata && pdata->resume_dfll_bypass &&
314             tegra_is_clk_initialized(tegra_dfll) &&
315             !tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
316                 pdata->resume_dfll_bypass();
317
318         /* If DFLL is used as CPU clock source restore closed loop mode */
319         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
320                 if (tegra_dfll && tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
321                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
322         }
323 #endif
324 }
325
326 /* ensures that sufficient time is passed for a register write to
327  * serialize into the 32KHz domain */
328 static void pmc_32kwritel(u32 val, unsigned long offs)
329 {
330         writel(val, pmc + offs);
331         udelay(130);
332 }
333
334 #if !defined(CONFIG_OF) || !defined(CONFIG_COMMON_CLK)
335 #if defined(CONFIG_ARCH_TEGRA_21x_SOC)
336 static void set_power_timers(unsigned long us_on, unsigned long us_off,
337                              long rate)
338 {
339         writel(0, pmc + PMC_CPUPWRGOOD_TIMER);
340         writel(0, pmc + PMC_CPUPWROFF_TIMER);
341         wmb();
342 }
343 #else
344 static void set_power_timers(unsigned long us_on, unsigned long us_off,
345                              long rate)
346 {
347         static unsigned long last_us_off;
348         unsigned long long ticks;
349         unsigned long long pclk;
350
351         if (WARN_ON_ONCE(rate <= 0))
352                 pclk = 100000000;
353         else
354                 pclk = rate;
355
356         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
357                 ticks = (us_on * pclk) + 999999ull;
358                 do_div(ticks, 1000000);
359                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
360
361                 ticks = (us_off * pclk) + 999999ull;
362                 do_div(ticks, 1000000);
363                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
364                 wmb();
365         }
366         tegra_last_pclk = pclk;
367         last_us_off = us_off;
368 }
369 #endif
370 #endif
371
372 /*
373  * restore_csite_clock
374  */
375 static void restore_csite_clock(void)
376 {
377         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
378 }
379
380 /*
381  * Clear flow controller registers
382  */
383 static void clear_flow_controller(void)
384 {
385         int cpu = cpu_logical_map(smp_processor_id());
386         unsigned int reg;
387
388         /* Do not power-gate CPU 0 when flow controlled */
389         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
390         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
391         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
392         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
393         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
394         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
395         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
396 }
397
398 /*
399  * saves pll state for use by restart_plls
400  */
401 static void save_pll_state(void)
402 {
403         /* switch coresite to clk_m, save off original source */
404         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
405         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
406
407         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
408         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
409         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
410         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
411         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
412         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
413         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
414         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
415 }
416
417 static int tegra_psci_suspend_cpu(long unsigned int val)
418 {
419         struct psci_power_state pps;
420
421         if (tegra_cpu_is_secure()) {
422                 if (psci_ops.cpu_suspend) {
423                         pps.id = TEGRA_ID_CPU_SUSPEND_LP0;
424                         pps.type = PSCI_POWER_STATE_TYPE_POWER_DOWN;
425                         pps.affinity_level = TEGRA_PWR_DN_AFFINITY_SYSTEM;
426
427                         psci_ops.cpu_suspend(pps, __pa(cpu_resume));
428                 }
429         }
430
431         return 0;
432 }
433
434 static void tegra_sleep_core(enum tegra_suspend_mode mode,
435                              unsigned long v2p)
436 {
437         if (tegra_cpu_is_secure()) {
438                 __flush_dcache_area(&tegra_resume_timestamps_start,
439                                         (&tegra_resume_timestamps_end -
440                                          &tegra_resume_timestamps_start));
441
442                 BUG_ON(mode != TEGRA_SUSPEND_LP0);
443
444                 trace_smc_sleep_core(NVSEC_SMC_START);
445                 cpu_suspend(v2p, tegra_psci_suspend_cpu);
446                 trace_smc_sleep_core(NVSEC_SMC_DONE);
447         }
448
449         tegra_get_suspend_time();
450 }
451
452 static int tegra_common_suspend(void)
453 {
454         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
455         u32 reg;
456
457         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
458         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
459         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
460
461         reg = readl(pmc + PMC_SCRATCH4);
462         if (is_slow_cluster())
463                 reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
464         else
465                 reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
466         writel(reg, pmc + PMC_SCRATCH4);
467
468         return 0;
469 }
470
471 static void tegra_common_resume(void)
472 {
473         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
474
475         /* Clear DPD Enable */
476         writel(0x0, pmc + PMC_DPD_ENABLE);
477
478         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
479         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
480         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
481         writel(0x0, pmc + PMC_SCRATCH41);
482 }
483
484 #if defined(CONFIG_ARCH_TEGRA_21x_SOC)
485 static void tegra_pm_sc7_set(void)
486 {
487         u32 reg;
488         unsigned long rate;
489
490         rate = clk_get_rate(tegra_pclk);
491
492         reg = readl(pmc + PMC_CTRL);
493         reg |= TEGRA_POWER_PWRREQ_OE;
494         if (pdata->combined_req)
495                 reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
496         else
497                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
498
499         reg &= ~TEGRA_POWER_EFFECT_LP0;
500
501         pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
502
503 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
504         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
505 #else
506         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
507 #endif
508
509         pmc_32kwritel(reg, PMC_CTRL);
510
511         /*
512          * This is to ensure the wake status is correctly captured and held
513          * correctly for all the wake.  This is necessitated due to a bug in
514          * the logic which had made the   E_INPUT control for specific pin
515          * i.e. GPIO_PA6 to Logic0 because of which wake event status didn't
516          * get latched in some corner cases.  Details are available in the
517          * bug 1495809
518          */
519         tegra_pmc_enable_wake_det(true);
520 }
521
522 int tegra_pm_prepare_sc7(void)
523 {
524         if (!tegra_pm_irq_lp0_allowed())
525                 return -EINVAL;
526
527         tegra_common_suspend();
528
529         tegra_pm_sc7_set();
530
531         tegra_tsc_suspend();
532
533         tegra_lp0_suspend_mc();
534
535         tegra_tsc_wait_for_suspend();
536
537         save_pll_state();
538
539         return 0;
540 }
541
542 int tegra_pm_post_sc7(void)
543 {
544         u32 reg;
545
546         tegra_tsc_resume();
547
548         tegra_lp0_resume_mc();
549
550         tegra_tsc_wait_for_resume();
551
552         restore_csite_clock();
553
554         /* for platforms where the core & CPU power requests are
555          * combined as a single request to the PMU, transition out
556          * of LP0 state by temporarily enabling both requests
557          */
558         if (pdata->combined_req) {
559                 reg = readl(pmc + PMC_CTRL);
560                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
561                 pmc_32kwritel(reg, PMC_CTRL);
562                 reg &= ~TEGRA_POWER_PWRREQ_OE;
563                 pmc_32kwritel(reg, PMC_CTRL);
564         }
565
566         tegra_common_resume();
567
568         return 0;
569 }
570 #endif /* CONFIG_ARCH_TEGRA_21x_SOC */
571
572 static void tegra_pm_set(enum tegra_suspend_mode mode)
573 {
574         u32 reg, boot_flag;
575         unsigned long rate = 32768;
576
577         reg = readl(pmc + PMC_CTRL);
578         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
579         if (pdata->combined_req)
580                 reg &= ~TEGRA_POWER_PWRREQ_OE;
581         else
582                 reg |= TEGRA_POWER_PWRREQ_OE;
583         reg &= ~TEGRA_POWER_EFFECT_LP0;
584
585         switch (mode) {
586         case TEGRA_SUSPEND_LP0:
587                 if (pdata->combined_req) {
588                         reg |= TEGRA_POWER_PWRREQ_OE;
589                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
590                 }
591
592                 /* Enable DPD sample to trigger sampling pads data and direction
593                  * in which pad will be driven during lp0 mode*/
594                 writel(0x1, pmc + PMC_DPD_SAMPLE);
595                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
596                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
597                 tegra_is_dpd_mode = true;
598
599                 /* Set warmboot flag */
600                 boot_flag = readl(pmc + PMC_SCRATCH0);
601                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
602
603                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
604
605                 reg |= TEGRA_POWER_EFFECT_LP0;
606
607                 /* No break here. LP0 code falls through to write SCRATCH41 */
608         case TEGRA_SUSPEND_LP1:
609                 __raw_writel(virt_to_phys(cpu_resume), pmc + PMC_SCRATCH41);
610                 wmb();
611                 rate = clk_get_rate(tegra_clk_m);
612                 break;
613         case TEGRA_SUSPEND_LP2:
614                 rate = clk_get_rate(tegra_pclk);
615                 break;
616         case TEGRA_SUSPEND_NONE:
617                 return;
618         default:
619                 BUG();
620         }
621
622 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
623         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
624 #else
625         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
626 #endif
627
628         pmc_32kwritel(reg, PMC_CTRL);
629 }
630
631 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
632         [TEGRA_SUSPEND_NONE] = "none",
633         [TEGRA_SUSPEND_LP2] = "LP2",
634         [TEGRA_SUSPEND_LP1] = "LP1",
635         [TEGRA_SUSPEND_LP0] = "LP0",
636 };
637
638 #if defined(CONFIG_ARCH_TEGRA_21x_SOC)
639
640 static void tegra_bpmp_enable_suspend(int mode, int flags)
641 {
642         int32_t mb[] = { cpu_to_le32(mode), cpu_to_le32(flags) };
643         int r = tegra_bpmp_send(MRQ_ENABLE_SUSPEND, &mb, sizeof(mb));
644         WARN_ON(r);
645 }
646
647 static int tegra210_suspend_dram(enum tegra_suspend_mode mode)
648 {
649         int err = 0;
650         unsigned long arg;
651         struct psci_power_state ps = {
652                 .id = TEGRA210_CPUIDLE_SC7,
653                 .type = PSCI_POWER_STATE_TYPE_POWER_DOWN,
654                 .affinity_level = 2,
655         };
656
657         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
658                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
659                 err = -ENXIO;
660                 goto fail;
661         }
662
663         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
664                 err = -ENXIO;
665                 goto fail;
666         }
667
668         if (mode == TEGRA_SUSPEND_LP1) {
669                 tegra_bpmp_enable_suspend(TEGRA_PM_SC4, 0);
670                 ps.id = TEGRA210_CPUIDLE_CC7;
671                 ps.affinity_level = 1;
672
673                 trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
674                 tegra_get_suspend_time();
675
676                 arg = psci_power_state_pack(ps);
677                 cpu_suspend(arg, NULL);
678
679                 resume_entry_time = tegra_read_usec_raw();
680                 trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
681
682                 return err;
683         }
684
685         tegra_bpmp_enable_suspend(TEGRA_PM_SC7, 0);
686         tegra_pm_prepare_sc7();
687         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
688         tegra_get_suspend_time();
689
690         arg = psci_power_state_pack(ps);
691         cpu_suspend(arg, NULL);
692
693         resume_entry_time = tegra_read_usec_raw();
694         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
695         tegra_pm_post_sc7();
696
697 fail:
698         return err;
699 }
700 #endif /* CONFIG_ARCH_TEGRA_21x_SOC */
701
702 static int tegra_suspend_enter(suspend_state_t state)
703 {
704         int ret = 0;
705         ktime_t delta;
706         struct timespec ts_entry, ts_exit;
707
708         if (pdata && pdata->board_suspend)
709                 pdata->board_suspend(current_suspend_mode,
710                         TEGRA_SUSPEND_BEFORE_PERIPHERAL);
711
712         read_persistent_clock(&ts_entry);
713
714 #ifdef CONFIG_ARCH_TEGRA_21x_SOC
715         ret = tegra210_suspend_dram(current_suspend_mode);
716 #else
717         ret = tegra_suspend_dram(current_suspend_mode, 0);
718 #endif
719         if (ret) {
720                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
721                 goto abort_suspend;
722         }
723
724         read_persistent_clock(&ts_exit);
725
726         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
727                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
728
729                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
730                 tegra_dvfs_rail_pause(tegra_gpu_rail, delta, false);
731                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
732                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
733                 else
734                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
735         }
736
737 abort_suspend:
738         if (pdata && pdata->board_resume)
739                 pdata->board_resume(current_suspend_mode,
740                         TEGRA_RESUME_AFTER_PERIPHERAL);
741
742         return ret;
743 }
744
745 static void tegra_suspend_check_pwr_stats(void)
746 {
747         /* cpus and l2 are powered off later */
748         unsigned long pwrgate_partid_mask =
749                 (1 << TEGRA_POWERGATE_HEG)      |
750                 (1 << TEGRA_POWERGATE_SATA)     |
751                 (1 << TEGRA_POWERGATE_3D1)      |
752                 (1 << TEGRA_POWERGATE_3D)       |
753                 (1 << TEGRA_POWERGATE_VENC)     |
754                 (1 << TEGRA_POWERGATE_PCIE)     |
755                 (1 << TEGRA_POWERGATE_VDEC)     |
756                 (1 << TEGRA_POWERGATE_MPE);
757
758         int partid;
759
760         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
761                 if ((1 << partid) & pwrgate_partid_mask)
762                         if (tegra_powergate_is_powered(partid))
763                                 pr_debug("partition %s is left on before suspend\n",
764                                         tegra_powergate_get_name(partid));
765
766         return;
767 }
768
769 static void tegra_suspend_powergate_control(int partid, bool turn_off)
770 {
771         if (turn_off)
772                 tegra_powergate_partition(partid);
773         else
774                 tegra_unpowergate_partition(partid);
775 }
776
777 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
778 {
779         int err = 0;
780         u32 scratch37 = 0xDEADBEEF;
781         u32 reg;
782
783         bool tegra_suspend_vde_powergated = false;
784
785         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
786                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
787                 err = -ENXIO;
788                 goto fail;
789         }
790
791         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
792                 pr_info("LP0 not used due to unsupported wakeup events\n");
793                 mode = TEGRA_SUSPEND_LP1;
794         }
795
796         /* turn off VDE partition in LP1 */
797         if (mode == TEGRA_SUSPEND_LP1 &&
798                 tegra_powergate_is_powered(TEGRA_POWERGATE_VDEC)) {
799                 pr_info("turning off partition %s in LP1\n",
800                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
801                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, true);
802                 tegra_suspend_vde_powergated = true;
803         }
804
805         tegra_common_suspend();
806
807         tegra_pm_set(mode);
808
809         if (pdata && pdata->board_suspend)
810                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
811
812         local_fiq_disable();
813
814         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
815
816         if (mode == TEGRA_SUSPEND_LP0) {
817                 tegra_tsc_suspend();
818                 tegra_lp0_suspend_mc();
819                 tegra_tsc_wait_for_suspend();
820         }
821
822         cpu_cluster_pm_enter();
823         save_pll_state();
824
825         if (tegra_get_chipid() != TEGRA_CHIPID_TEGRA13)
826                 flush_cache_all();
827
828         tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
829
830         resume_entry_time = 0;
831         if (mode != TEGRA_SUSPEND_LP0)
832                 resume_entry_time = tegra_read_usec_raw();
833
834         if (mode == TEGRA_SUSPEND_LP0) {
835                 tegra_tsc_resume();
836                 tegra_lp0_resume_mc();
837                 tegra_tsc_wait_for_resume();
838         }
839
840         /* if scratch37 was clobbered during LP1, restore it */
841         if (scratch37 != 0xDEADBEEF)
842                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
843
844         restore_csite_clock();
845         clear_flow_controller();
846         cpu_cluster_pm_exit();
847
848         /* for platforms where the core & CPU power requests are
849          * combined as a single request to the PMU, transition out
850          * of LP0 state by temporarily enabling both requests
851          */
852         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
853                 reg = readl(pmc + PMC_CTRL);
854                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
855                 pmc_32kwritel(reg, PMC_CTRL);
856                 reg &= ~TEGRA_POWER_PWRREQ_OE;
857                 pmc_32kwritel(reg, PMC_CTRL);
858         }
859
860         if (pdata && pdata->board_resume)
861                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
862
863         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
864
865         local_fiq_enable();
866
867         tegra_common_resume();
868
869         /* turn on VDE partition in LP1 */
870         if (mode == TEGRA_SUSPEND_LP1 && tegra_suspend_vde_powergated) {
871                 pr_info("turning on partition %s in LP1\n",
872                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
873                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, false);
874         }
875
876 fail:
877         return err;
878 }
879
880 static int tegra_suspend_valid(suspend_state_t state)
881 {
882         int valid = 1;
883
884         /* LP0 is not supported on T132 A01 silicon */
885         if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA13 && \
886                 tegra_revision == TEGRA_REVISION_A01)
887                 valid = 0;
888
889         return valid;
890 }
891
892 static int tegra_suspend_prepare_late(void)
893 {
894         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) ||
895                         (current_suspend_mode == TEGRA_SUSPEND_LP1))
896                 tegra_suspend_check_pwr_stats();
897
898         return 0;
899 }
900
901 static void tegra_suspend_finish(void)
902 {
903         if (pdata && pdata->cpu_resume_boost) {
904                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
905                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
906                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
907         }
908 }
909
910 static const struct platform_suspend_ops tegra_suspend_ops = {
911         .prepare_late   = tegra_suspend_prepare_late,
912         .valid          = tegra_suspend_valid,
913         .finish         = tegra_suspend_finish,
914         .enter          = tegra_suspend_enter,
915 };
916
917 static ssize_t suspend_mode_show(struct kobject *kobj,
918                                         struct kobj_attribute *attr, char *buf)
919 {
920         char *start = buf;
921         char *end = buf + PAGE_SIZE;
922
923         start += scnprintf(start, end - start, "%s ",
924                                 tegra_suspend_name[current_suspend_mode]);
925         start += scnprintf(start, end - start, "\n");
926
927         return start - buf;
928 }
929
930 static ssize_t suspend_mode_store(struct kobject *kobj,
931                                         struct kobj_attribute *attr,
932                                         const char *buf, size_t n)
933 {
934         int len;
935         const char *name_ptr;
936         enum tegra_suspend_mode new_mode;
937
938         name_ptr = buf;
939         while (*name_ptr && !isspace(*name_ptr))
940                 name_ptr++;
941         len = name_ptr - buf;
942         if (!len)
943                 goto bad_name;
944         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
945         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
946                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
947                 pr_info("Illegal tegra suspend state: %s\n", buf);
948                 goto bad_name;
949         }
950
951         for (new_mode = TEGRA_SUSPEND_NONE;
952                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
953                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
954                         current_suspend_mode = new_mode;
955                         break;
956                 }
957         }
958
959 bad_name:
960         return n;
961 }
962
963 static struct kobj_attribute suspend_mode_attribute =
964         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
965
966 static ssize_t suspend_resume_time_show(struct kobject *kobj,
967                                         struct kobj_attribute *attr,
968                                         char *buf)
969 {
970         return sprintf(buf, "%ums\n", ((u32)resume_time / 1000));
971 }
972
973 static struct kobj_attribute suspend_resume_time_attribute =
974         __ATTR(resume_time, 0444, suspend_resume_time_show, NULL);
975
976 static ssize_t suspend_time_show(struct kobject *kobj,
977                                         struct kobj_attribute *attr,
978                                         char *buf)
979 {
980         return sprintf(buf, "%ums\n", ((u32)suspend_time / 1000));
981 }
982
983 static struct kobj_attribute suspend_time_attribute =
984         __ATTR(suspend_time, 0444, suspend_time_show, NULL);
985
986 static struct kobject *suspend_kobj;
987
988 static int tegra_pm_enter_suspend(void)
989 {
990         suspend_cpu_dfll_mode(0);
991         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
992         return 0;
993 }
994
995 static void tegra_pm_enter_resume(void)
996 {
997         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
998         resume_cpu_dfll_mode(0);
999 }
1000
1001 static void tegra_pm_enter_shutdown(void)
1002 {
1003         suspend_cpu_dfll_mode(0);
1004         pr_info("Shutting down tegra ...\n");
1005 }
1006
1007 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1008         .suspend = tegra_pm_enter_suspend,
1009         .resume = tegra_pm_enter_resume,
1010         .save = tegra_pm_enter_suspend,
1011         .restore = tegra_pm_enter_resume,
1012         .shutdown = tegra_pm_enter_shutdown,
1013 };
1014
1015 static __init int tegra_pm_enter_syscore_init(void)
1016 {
1017         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1018         return 0;
1019 }
1020 subsys_initcall(tegra_pm_enter_syscore_init);
1021 #endif
1022
1023 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1024 {
1025         u32 reg;
1026         u32 mode;
1027         struct pmc_pm_data *pm_dat;
1028         bool is_board_pdata = true;
1029
1030 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1031         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1032         BUG_ON(IS_ERR(tegra_dfll));
1033 #endif
1034         tegra_pclk = clk_get_sys(NULL, "pclk");
1035         BUG_ON(IS_ERR(tegra_pclk));
1036         tegra_clk_m = clk_get_sys(NULL, "clk_m");
1037         BUG_ON(IS_ERR(tegra_clk_m));
1038
1039         /* create the pdata from DT information */
1040         pm_dat = tegra_get_pm_data();
1041         if (pm_dat) {
1042                 pr_err("PMC dt information non-NULL %s\n", __func__);
1043                 is_board_pdata = false;
1044                 pdata = kzalloc(sizeof(struct tegra_suspend_platform_data),
1045                         GFP_KERNEL);
1046                 if (pm_dat->combined_req != plat->combined_req) {
1047                         pr_debug("PMC DT attribute combined_req=%d, board value=%d\n",
1048                                 pm_dat->combined_req, plat->combined_req);
1049                         pdata->combined_req = plat->combined_req;
1050                 } else {
1051                         pdata->combined_req = pm_dat->combined_req;
1052                 }
1053                 if (pm_dat->sysclkreq_high != plat->sysclkreq_high) {
1054                         pr_debug("PMC DT attribute sysclkreq_high=%d, board value=%d\n",
1055                                 pm_dat->sysclkreq_high, plat->sysclkreq_high);
1056                         pdata->sysclkreq_high = plat->sysclkreq_high;
1057                 } else {
1058                         pdata->sysclkreq_high = pm_dat->sysclkreq_high;
1059                 }
1060                 if (pm_dat->corereq_high != plat->corereq_high) {
1061                         pr_debug("PMC DT attribute corereq_high=%d, board value=%d\n",
1062                                 pm_dat->corereq_high, plat->corereq_high);
1063                         pdata->corereq_high = plat->corereq_high;
1064                 } else {
1065                         pdata->corereq_high = pm_dat->corereq_high;
1066                 }
1067                 if (pm_dat->cpu_off_time != plat->cpu_off_timer) {
1068                         pr_debug("PMC DT attribute cpu_off_timer=%d, board value=%ld\n",
1069                                 pm_dat->cpu_off_time, plat->cpu_off_timer);
1070                         pdata->cpu_off_timer = plat->cpu_off_timer;
1071                 } else {
1072                         pdata->cpu_off_timer = pm_dat->cpu_off_time;
1073                 }
1074                 if (pm_dat->cpu_good_time != plat->cpu_timer) {
1075                         pr_debug("PMC DT attribute cpu_timer=%d, board value=%ld\n",
1076                                 pm_dat->cpu_good_time, plat->cpu_timer);
1077                         pdata->cpu_timer = plat->cpu_timer;
1078                 } else {
1079                         pdata->cpu_timer = pm_dat->cpu_good_time;
1080                 }
1081                 if (pm_dat->suspend_mode != plat->suspend_mode) {
1082                         pr_debug("PMC DT attribute suspend_mode=%d, board value=%d\n",
1083                                 pm_dat->suspend_mode, plat->suspend_mode);
1084                         pdata->suspend_mode = plat->suspend_mode;
1085                 } else {
1086                         pdata->suspend_mode = pm_dat->suspend_mode;
1087                 }
1088                 if (pm_dat->cpu_suspend_freq != plat->cpu_suspend_freq) {
1089                         pr_debug("PMC DT attribute cpu_suspend_freq=%d, cpu_suspend_freq=%d\n",
1090                                 pm_dat->cpu_suspend_freq,
1091                                 plat->cpu_suspend_freq);
1092                         pdata->cpu_suspend_freq = plat->cpu_suspend_freq;
1093                 } else {
1094                         pdata->cpu_suspend_freq = pm_dat->cpu_suspend_freq;
1095                 }
1096
1097                 /* FIXME: pmc_pm_data fields to be reused
1098                  *      core_osc_time, core_pmu_time, core_off_time
1099                  *      units of above fields is uSec while
1100                  *      platform data values are in ticks
1101                  */
1102                 /* FIXME: pmc_pm_data unused by downstream code
1103                  *      cpu_pwr_good_en, lp0_vec_size, lp0_vec_phy_addr
1104                  */
1105                 /* FIXME: add missing DT bindings taken from platform data */
1106                 pdata->core_timer = plat->core_timer;
1107                 pdata->core_off_timer = plat->core_off_timer;
1108                 pdata->board_suspend = plat->board_suspend;
1109                 pdata->board_resume = plat->board_resume;
1110                 pdata->sysclkreq_gpio = plat->sysclkreq_gpio;
1111                 pdata->cpu_lp2_min_residency = plat->cpu_lp2_min_residency;
1112                 pdata->cpu_resume_boost = plat->cpu_resume_boost;
1113 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
1114                 pdata->min_residency_vmin_fmin = plat->min_residency_vmin_fmin;
1115                 pdata->min_residency_ncpu_slow = plat->min_residency_ncpu_slow;
1116                 pdata->min_residency_ncpu_fast = plat->min_residency_ncpu_fast;
1117                 pdata->min_residency_crail = plat->min_residency_crail;
1118                 pdata->crail_up_early = plat->crail_up_early;
1119 #endif
1120                 pdata->min_residency_mclk_stop = plat->min_residency_mclk_stop;
1121                 pdata->usb_vbus_internal_wake = plat->usb_vbus_internal_wake;
1122                 pdata->usb_id_internal_wake = plat->usb_id_internal_wake;
1123                 pdata->suspend_dfll_bypass = plat->suspend_dfll_bypass;
1124                 pdata->resume_dfll_bypass = plat->resume_dfll_bypass;
1125         } else {
1126                 pr_err("PMC board data used in %s\n", __func__);
1127                 pdata = plat;
1128         }
1129         (void)reg;
1130         (void)mode;
1131
1132         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1133                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1134
1135 #ifndef CONFIG_PM_SLEEP
1136         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1137                 pr_warn("%s: Suspend requires CONFIG_PM_SLEEP -- "
1138                            "disabling suspend\n", __func__);
1139                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1140         }
1141 #else
1142
1143         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1144                 tegra_lp0_vec_relocate) {
1145                 unsigned char *reloc_lp0;
1146                 unsigned long tmp;
1147                 void __iomem *orig;
1148                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1149                                         GFP_KERNEL);
1150                 WARN_ON(!reloc_lp0);
1151                 if (!reloc_lp0) {
1152                         pr_err("%s: Failed to allocate reloc_lp0\n",
1153                                 __func__);
1154                         goto out;
1155                 }
1156
1157                 /* Avoid a kmemleak false positive. The allocated memory
1158                  * block is later referenced by a physical address (i.e.
1159                  * tegra_lp0_vec_start) which kmemleak can't detect.
1160                  */
1161                 kmemleak_not_leak(reloc_lp0);
1162
1163                 orig = ioremap_wc(tegra_lp0_vec_start, tegra_lp0_vec_size);
1164                 WARN_ON(!orig);
1165                 if (!orig) {
1166                         pr_err("%s: Failed to map tegra_lp0_vec_start %llx\n",
1167                                 __func__, tegra_lp0_vec_start);
1168                         kfree(reloc_lp0);
1169                         goto out;
1170                 }
1171
1172                 tmp = (unsigned long) reloc_lp0;
1173                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1174                 reloc_lp0 = (unsigned char *)tmp;
1175                 memcpy(reloc_lp0, (const void *)(uintptr_t)orig,
1176                         tegra_lp0_vec_size);
1177                 wmb();
1178                 iounmap(orig);
1179                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1180         }
1181
1182 out:
1183         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1184                 pr_warn("%s: Suspend mode LP0 requested, no lp0_vec "
1185                            "provided by bootlader -- disabling LP0\n",
1186                            __func__);
1187                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1188         }
1189
1190         /* Initialize SOC-specific data */
1191 #ifdef CONFIG_ARCH_TEGRA_13x_SOC
1192         tegra_soc_suspend_init();
1193
1194         current_suspend_mode = TEGRA_SUSPEND_LP0;
1195 #endif
1196
1197 #ifdef CONFIG_ARCH_TEGRA_21x_SOC
1198         reg = readl(pmc + PMC_CTRL2);
1199         reg |= PMC_ALLOW_PULSE_WAKE;
1200         pmc_32kwritel(reg, PMC_CTRL2);
1201 #endif
1202
1203         /* Always enable CPU power request; just normal polarity is supported */
1204         reg = readl(pmc + PMC_CTRL);
1205         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1206         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1207         pmc_32kwritel(reg, PMC_CTRL);
1208
1209         /* Configure core power request and system clock control if LP0
1210            is supported */
1211         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1212         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1213
1214         reg = readl(pmc + PMC_CTRL);
1215
1216         if (!pdata->sysclkreq_high)
1217                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1218         else
1219                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1220
1221         if (!pdata->corereq_high)
1222                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1223         else
1224                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1225
1226         /* configure output inverters while the request is tristated */
1227         pmc_32kwritel(reg, PMC_CTRL);
1228
1229         /* now enable requests */
1230         reg |= TEGRA_POWER_SYSCLK_OE;
1231         if (!pdata->combined_req)
1232                 reg |= TEGRA_POWER_PWRREQ_OE;
1233         pmc_32kwritel(reg, PMC_CTRL);
1234
1235         if (pdata->sysclkreq_gpio) {
1236                 reg = readl(pmc + PMC_DPAD_ORIDE);
1237                 reg &= ~TEGRA_DPAD_ORIDE_SYS_CLK_REQ;
1238                 pmc_32kwritel(reg, PMC_DPAD_ORIDE);
1239         }
1240
1241         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1242                 tegra_lp0_suspend_init();
1243
1244         suspend_set_ops(&tegra_suspend_ops);
1245
1246         /* Create /sys/power/suspend/type */
1247         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1248         if (suspend_kobj) {
1249                 if (sysfs_create_file(suspend_kobj,
1250                                                 &suspend_mode_attribute.attr))
1251                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1252                                                                 __func__);
1253                 if (sysfs_create_file(suspend_kobj,
1254                                         &suspend_resume_time_attribute.attr))
1255                         pr_err("%s: sysfs_create_file resume_time failed!\n",
1256                                                                 __func__);
1257                 if (sysfs_create_file(suspend_kobj,
1258                                         &suspend_time_attribute.attr))
1259                         pr_err("%s: sysfs_create_file suspend_time failed!\n",
1260                                                                 __func__);
1261         }
1262
1263 #endif
1264         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1265                 tegra_pd_in_idle(false);
1266
1267         current_suspend_mode = plat->suspend_mode;
1268 }
1269
1270 #ifdef CONFIG_ARM_ARCH_TIMER
1271
1272 static u32 tsc_suspend_start;
1273 static u32 tsc_resume_start;
1274
1275 #define pmc_writel(value, reg) \
1276                 writel(value, pmc + (reg))
1277 #define pmc_readl(reg) \
1278                 readl(pmc + (reg))
1279
1280 #define PMC_DPD_ENABLE_TSC_MULT_ENABLE  (1 << 1)
1281 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
1282 #define PMC_DPD_ENABLE_ON               (1 << 0)
1283 #endif
1284
1285 #define PMC_TSC_MULT                    0x2b4
1286 #define PMC_TSC_MULT_FREQ_STS           (1 << 16)
1287
1288 #define TSC_TIMEOUT_US                  32
1289
1290 void tegra_tsc_suspend(void)
1291 {
1292         u32 reg = pmc_readl(PMC_DPD_ENABLE);
1293         BUG_ON(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE);
1294         reg |= PMC_DPD_ENABLE_TSC_MULT_ENABLE;
1295         pmc_writel(reg, PMC_DPD_ENABLE);
1296         tsc_suspend_start = tegra_read_usec_raw();
1297 }
1298
1299 void tegra_tsc_resume(void)
1300 {
1301         u32 reg = pmc_readl(PMC_DPD_ENABLE);
1302         BUG_ON(!(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE));
1303         reg &= ~PMC_DPD_ENABLE_TSC_MULT_ENABLE;
1304 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
1305         /*
1306          * FIXME: T12x SW WAR -
1307          * Resume ensures DPD_ENABLE is 0 when writing
1308          * TSC_MULT_ENABLE, else PMC wake status gets reset
1309          */
1310         reg &= ~PMC_DPD_ENABLE_ON;
1311 #endif
1312         pmc_writel(reg, PMC_DPD_ENABLE);
1313         tsc_resume_start = tegra_read_usec_raw();
1314 }
1315
1316 void tegra_tsc_wait_for_suspend(void)
1317 {
1318         while ((tegra_read_usec_raw() - tsc_suspend_start) <
1319                 TSC_TIMEOUT_US) {
1320                 if (pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS)
1321                         break;
1322                 cpu_relax();
1323         }
1324 }
1325
1326 void tegra_tsc_wait_for_resume(void)
1327 {
1328         while ((tegra_read_usec_raw() - tsc_resume_start) <
1329                 TSC_TIMEOUT_US) {
1330                 if (!(pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS))
1331                         break;
1332                 cpu_relax();
1333         }
1334 }
1335 #endif