arm: tegra: Move driver files to drivers/platform
[linux-3.10.git] / arch / arm64 / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2013-2014, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/cpumask.h>
32 #include <linux/delay.h>
33 #include <linux/cpu_pm.h>
34 #include <linux/err.h>
35 #include <linux/debugfs.h>
36 #include <linux/delay.h>
37 #include <linux/suspend.h>
38 #include <linux/serial_reg.h>
39 #include <linux/seq_file.h>
40 #include <linux/uaccess.h>
41 #include <linux/syscore_ops.h>
42 #include <linux/cpu_pm.h>
43 #include <linux/clk/tegra.h>
44 #include <linux/export.h>
45 #include <linux/vmalloc.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48 #include <linux/clk/tegra.h>
49 #include <linux/tegra-powergate.h>
50 #include <linux/tegra-soc.h>
51 #include <linux/tegra-timer.h>
52 #include <linux/tegra-cpuidle.h>
53 #include <linux/irqchip/tegra.h>
54 #include <linux/tegra-pm.h>
55 #include <linux/tegra_pm_domains.h>
56 #include <linux/tegra_smmu.h>
57 #include <linux/platform_data/tegra_bpmp.h>
58 #include <linux/kmemleak.h>
59
60 #include <trace/events/power.h>
61 #include <trace/events/nvsecurity.h>
62 #include <linux/tegra-pmc.h>
63
64 #include <asm/cacheflush.h>
65 #include <asm/idmap.h>
66 #include <asm/pgalloc.h>
67 #include <asm/pgtable.h>
68 #include <asm/tlbflush.h>
69 #include <asm/suspend.h>
70 #include <asm/smp_plat.h>
71 #include <asm/psci.h>
72
73 #include <mach/irqs.h>
74
75 #include "board.h"
76 #include <linux/platform/tegra/clock.h>
77 #include <linux/platform/tegra/common.h>
78 #include "iomap.h"
79 #include "pm.h"
80 #include "pm-soc.h"
81 #include <linux/platform/tegra/reset.h>
82 #include "sleep.h"
83 #include <linux/platform/tegra/dvfs.h>
84 #include <linux/platform/tegra/cpu-tegra.h>
85 #include <linux/platform/tegra/flowctrl.h>
86 #include "board.h"
87
88 #include "pm-tegra132.h"
89
90 /* core power request enable */
91 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)
92 /* enter LP0 when CPU pwr gated */
93 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)
94 /* CPU pwr req enable */
95 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)
96
97 struct suspend_context {
98         /*
99          * The next 7 values are referenced by offset in __restart_plls
100          * in headsmp-t2.S, and should not be moved
101          */
102         u32 pllx_misc;
103         u32 pllx_base;
104         u32 pllp_misc;
105         u32 pllp_base;
106         u32 pllp_outa;
107         u32 pllp_outb;
108         u32 pll_timeout;
109
110         u32 cpu_burst;
111         u32 clk_csite_src;
112         u32 cclk_divider;
113
114         u32 mc[3];
115
116         struct tegra_twd_context twd;
117 };
118 #define PMC_CTRL                0x0
119
120 #ifdef CONFIG_PM_SLEEP
121 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
122 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
123 #ifndef CONFIG_ARCH_TEGRA_21x_SOC
124 static int tegra_last_pclk;
125 #endif
126 static u64 resume_time;
127 static u64 resume_entry_time;
128 static u64 suspend_time;
129 static u64 suspend_entry_time;
130 #endif
131
132 struct suspend_context tegra_sctx;
133
134 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
135 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
136 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
137 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
138 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
139 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
140 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
141 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
142 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
143
144 #define TEGRA_DPAD_ORIDE_SYS_CLK_REQ    (1 << 21)
145
146 #define PMC_CTRL                0x0
147 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
148 #define PMC_WAKE_MASK           0xc
149 #define PMC_WAKE_LEVEL          0x10
150 #define PMC_DPAD_ORIDE          0x1C
151 #define PMC_WAKE_DELAY          0xe0
152 #define PMC_DPD_SAMPLE          0x20
153 #define PMC_DPD_ENABLE          0x24
154 #define PMC_IO_DPD_REQ          0x1B8
155 #define PMC_IO_DPD2_REQ         0x1C0
156
157 #define PMC_WAKE_STATUS         0x14
158 #define PMC_SW_WAKE_STATUS      0x18
159 #define PMC_COREPWRGOOD_TIMER   0x3c
160 #define PMC_CPUPWRGOOD_TIMER    0xc8
161 #define PMC_CPUPWROFF_TIMER     0xcc
162 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
163
164 #define PMC_PWRGATE_TOGGLE      0x30
165 #define PWRGATE_TOGGLE_START    (1 << 8)
166 #define UN_PWRGATE_CPU          \
167         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
168
169 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
170
171 #define CLK_RESET_CCLK_BURST    0x20
172 #define CLK_RESET_CCLK_DIVIDER  0x24
173 #define CLK_RESET_PLLC_BASE     0x80
174 #define CLK_RESET_PLLM_BASE     0x90
175 #define CLK_RESET_PLLX_BASE     0xe0
176 #define CLK_RESET_PLLX_MISC     0xe4
177 #define CLK_RESET_PLLP_BASE     0xa0
178 #define CLK_RESET_PLLP_OUTA     0xa4
179 #define CLK_RESET_PLLP_OUTB     0xa8
180 #define CLK_RESET_PLLP_MISC     0xac
181
182 #define CLK_RESET_SOURCE_CSITE  0x1d4
183
184 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
185 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
186 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
187 #define CLK_RESET_CCLK_IDLE_POLICY         1
188 #define CLK_RESET_CCLK_RUN_POLICY          2
189 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
190 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
191
192 #define MC_SECURITY_START       0x6c
193 #define MC_SECURITY_SIZE        0x70
194 #define MC_SECURITY_CFG2        0x7c
195
196 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
197 static struct clk *tegra_dfll;
198 #endif
199 static struct clk *tegra_pclk;
200 static struct clk *tegra_clk_m;
201 static struct tegra_suspend_platform_data *pdata;
202 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
203
204 void (*tegra_tear_down_cpu)(void);
205 int (*tegra_sleep_core_finish)(unsigned long v2p);
206
207 bool tegra_is_dpd_mode = false;
208
209 bool tegra_dvfs_is_dfll_bypass(void)
210 {
211 #ifdef CONFIG_REGULATOR_TEGRA_DFLL_BYPASS
212         if (tegra_cpu_rail->dt_reg_pwm)
213                 return true;
214 #endif
215         return false;
216 }
217
218 static inline unsigned int is_slow_cluster(void)
219 {
220         int reg;
221
222         asm("mrs        %0, mpidr_el1\n"
223             "ubfx       %0, %0, #8, #4"
224             : "=r" (reg)
225             :
226             : "cc", "memory");
227
228         return reg & 1;
229 }
230
231 #ifdef CONFIG_PM_SLEEP
232
233 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
234         [TEGRA_SUSPEND_NONE]    = "none",
235         [TEGRA_SUSPEND_LP2]     = "lp2",
236         [TEGRA_SUSPEND_LP1]     = "lp1",
237         [TEGRA_SUSPEND_LP0]     = "lp0",
238 };
239
240 int tegra_is_lp0_suspend_mode(void)
241 {
242         return (current_suspend_mode == TEGRA_SUSPEND_LP0);
243 }
244
245 void tegra_log_resume_time(void)
246 {
247         u64 resume_end_time = tegra_read_usec_raw();
248
249         if (resume_entry_time > resume_end_time)
250                 resume_end_time |= 1ull<<32;
251         resume_time = resume_end_time - resume_entry_time;
252 }
253
254 void tegra_log_suspend_time(void)
255 {
256         suspend_entry_time = tegra_read_usec_raw();
257 }
258
259 static void tegra_get_suspend_time(void)
260 {
261         u64 suspend_end_time;
262         suspend_end_time = tegra_read_usec_raw();
263
264         if (suspend_entry_time > suspend_end_time)
265                 suspend_end_time |= 1ull<<32;
266         suspend_time = suspend_end_time - suspend_entry_time;
267 }
268
269 unsigned long tegra_cpu_power_good_time(void)
270 {
271         if (WARN_ON_ONCE(!pdata))
272                 return 5000;
273
274         return pdata->cpu_timer;
275 }
276
277 unsigned long tegra_cpu_power_off_time(void)
278 {
279         if (WARN_ON_ONCE(!pdata))
280                 return 5000;
281
282         return pdata->cpu_off_timer;
283 }
284
285 unsigned int tegra_cpu_suspend_freq(void)
286 {
287         if (WARN_ON_ONCE(!pdata))
288                 return 0;
289
290         return pdata->cpu_suspend_freq;
291 }
292
293 static void suspend_cpu_dfll_mode(unsigned int flags)
294 {
295 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
296         /* If DFLL is used as CPU clock source go to open loop mode */
297         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
298                 if (tegra_dfll && tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
299                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
300         }
301
302         /* Suspend dfll bypass (safe rail down) on LP or if DFLL is Not used */
303         if (pdata && pdata->suspend_dfll_bypass &&
304             (!tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail)))
305                 pdata->suspend_dfll_bypass();
306 #endif
307 }
308
309 static void resume_cpu_dfll_mode(unsigned int flags)
310 {
311 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
312         /* If DFLL is Not used and resume on G restore bypass mode */
313         if (pdata && pdata->resume_dfll_bypass &&
314             tegra_is_clk_initialized(tegra_dfll) &&
315             !tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
316                 pdata->resume_dfll_bypass();
317
318         /* If DFLL is used as CPU clock source restore closed loop mode */
319         if (!(flags & TEGRA_POWER_CLUSTER_MASK)) {
320                 if (tegra_dfll && tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
321                         tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
322         }
323 #endif
324 }
325
326 /* ensures that sufficient time is passed for a register write to
327  * serialize into the 32KHz domain */
328 static void pmc_32kwritel(u32 val, unsigned long offs)
329 {
330         writel(val, pmc + offs);
331         udelay(130);
332 }
333
334 #if !defined(CONFIG_OF) || !defined(CONFIG_COMMON_CLK)
335 #if defined(CONFIG_ARCH_TEGRA_21x_SOC)
336 static void set_power_timers(unsigned long us_on, unsigned long us_off,
337                              long rate)
338 {
339         writel(0, pmc + PMC_CPUPWRGOOD_TIMER);
340         writel(0, pmc + PMC_CPUPWROFF_TIMER);
341         wmb();
342 }
343 #else
344 static void set_power_timers(unsigned long us_on, unsigned long us_off,
345                              long rate)
346 {
347         static unsigned long last_us_off;
348         unsigned long long ticks;
349         unsigned long long pclk;
350
351         if (WARN_ON_ONCE(rate <= 0))
352                 pclk = 100000000;
353         else
354                 pclk = rate;
355
356         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
357                 ticks = (us_on * pclk) + 999999ull;
358                 do_div(ticks, 1000000);
359                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
360
361                 ticks = (us_off * pclk) + 999999ull;
362                 do_div(ticks, 1000000);
363                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
364                 wmb();
365         }
366         tegra_last_pclk = pclk;
367         last_us_off = us_off;
368 }
369 #endif
370 #endif
371
372 /*
373  * restore_csite_clock
374  */
375 static void restore_csite_clock(void)
376 {
377         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
378 }
379
380 /*
381  * Clear flow controller registers
382  */
383 static void clear_flow_controller(void)
384 {
385         int cpu = cpu_logical_map(smp_processor_id());
386         unsigned int reg;
387
388         /* Do not power-gate CPU 0 when flow controlled */
389         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
390         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
391         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
392         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
393         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
394         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
395         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
396 }
397
398 /*
399  * saves pll state for use by restart_plls
400  */
401 static void save_pll_state(void)
402 {
403         /* switch coresite to clk_m, save off original source */
404         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
405         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
406
407         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
408         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
409         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
410         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
411         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
412         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
413         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
414         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
415 }
416
417 /*
418  * Prepares flow controller for transition to suspend state
419  *
420  * Must always be called on cpu 0.
421  */
422 static void prepare_flow_controller(void)
423 {
424         int cpu = cpu_logical_map(smp_processor_id());
425         unsigned int reg;
426         int i;
427
428         BUG_ON(cpu != 0);
429         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
430         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
431         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
432         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
433         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
434         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
435         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
436         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
437
438         for (i = 0; i < num_possible_cpus(); i++) {
439                 if (i == cpu)
440                         continue;
441                 reg = readl(FLOW_CTRL_CPU_CSR(i));
442                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
443                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
444                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
445         }
446
447         tegra_gic_cpu_disable(true);
448 }
449
450 void tegra_psci_suspend_cpu(void *entry_point)
451 {
452         struct psci_power_state pps;
453
454         if (tegra_cpu_is_secure()) {
455                 if (psci_ops.cpu_suspend) {
456                         pps.id = TEGRA_ID_CPU_SUSPEND_LP0;
457                         pps.type = PSCI_POWER_STATE_TYPE_POWER_DOWN;
458                         pps.affinity_level = TEGRA_PWR_DN_AFFINITY_CLUSTER;
459
460                         psci_ops.cpu_suspend(pps, virt_to_phys(entry_point));
461                 }
462         }
463 }
464
465 static void tegra_sleep_core(enum tegra_suspend_mode mode,
466                              unsigned long v2p)
467 {
468         if (tegra_cpu_is_secure()) {
469                 __flush_dcache_area(&tegra_resume_timestamps_start,
470                                         (&tegra_resume_timestamps_end -
471                                          &tegra_resume_timestamps_start));
472
473                 BUG_ON(mode != TEGRA_SUSPEND_LP0);
474
475                 trace_smc_sleep_core(NVSEC_SMC_START);
476                 tegra_psci_suspend_cpu(tegra_resume);
477                 trace_smc_sleep_core(NVSEC_SMC_DONE);
478         }
479
480         tegra_get_suspend_time();
481         cpu_suspend(v2p, tegra_sleep_core_finish);
482 }
483
484 static inline void tegra_sleep_cpu(unsigned long v2p)
485 {
486         cpu_suspend(v2p, tegra_sleep_cpu_finish);
487 }
488
489 static int tegra_common_suspend(void)
490 {
491         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
492         u32 reg;
493
494         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
495         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
496         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
497
498         reg = readl(pmc + PMC_SCRATCH4);
499         if (is_slow_cluster())
500                 reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
501         else
502                 reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
503         writel(reg, pmc + PMC_SCRATCH4);
504
505         return 0;
506 }
507
508 static void tegra_common_resume(void)
509 {
510         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
511
512         /* Clear DPD Enable */
513         writel(0x0, pmc + PMC_DPD_ENABLE);
514
515         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
516         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
517         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
518         writel(0x0, pmc + PMC_SCRATCH41);
519 }
520
521 #if defined(CONFIG_ARCH_TEGRA_21x_SOC)
522 static void tegra_pm_sc7_set(void)
523 {
524         u32 reg, boot_flag;
525         unsigned long rate = 32768;
526
527         reg = readl(pmc + PMC_CTRL);
528         reg |= TEGRA_POWER_PWRREQ_OE;
529         if (pdata->combined_req)
530                 reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
531         else
532                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
533
534         reg &= ~TEGRA_POWER_EFFECT_LP0;
535
536         /* Enable DPD sample to trigger sampling pads data and direction
537          * in which pad will be driven during lp0 mode*/
538         writel(0x1, pmc + PMC_DPD_SAMPLE);
539
540         /* Set warmboot flag */
541         boot_flag = readl(pmc + PMC_SCRATCH0);
542         pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
543
544         pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
545
546 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
547         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
548 #else
549         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
550 #endif
551
552         pmc_32kwritel(reg, PMC_CTRL);
553
554         /*
555          * This is to ensure the wake status is correctly captured and held
556          * correctly for all the wake.  This is necessitated due to a bug in
557          * the logic which had made the   E_INPUT control for specific pin
558          * i.e. GPIO_PA6 to Logic0 because of which wake event status didn't
559          * get latched in some corner cases.  Details are available in the
560          * bug 1495809
561          */
562         tegra_pmc_enable_wake_det(true);
563 }
564
565 int tegra_pm_prepare_sc7(void)
566 {
567         if (!tegra_pm_irq_lp0_allowed())
568                 return -EINVAL;
569
570         tegra_common_suspend();
571
572         tegra_pm_sc7_set();
573
574         tegra_tsc_suspend();
575
576         tegra_lp0_suspend_mc();
577
578         tegra_tsc_wait_for_suspend();
579
580         save_pll_state();
581
582         return 0;
583 }
584
585 int tegra_pm_post_sc7(void)
586 {
587         u32 reg;
588
589         tegra_tsc_resume();
590
591         tegra_lp0_resume_mc();
592
593         tegra_tsc_wait_for_resume();
594
595         restore_csite_clock();
596
597         /* for platforms where the core & CPU power requests are
598          * combined as a single request to the PMU, transition out
599          * of LP0 state by temporarily enabling both requests
600          */
601         if (pdata->combined_req) {
602                 reg = readl(pmc + PMC_CTRL);
603                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
604                 pmc_32kwritel(reg, PMC_CTRL);
605                 reg &= ~TEGRA_POWER_PWRREQ_OE;
606                 pmc_32kwritel(reg, PMC_CTRL);
607         }
608
609         tegra_common_resume();
610
611         return 0;
612 }
613 #endif
614
615 static void tegra_pm_set(enum tegra_suspend_mode mode)
616 {
617         u32 reg, boot_flag;
618         unsigned long rate = 32768;
619
620         reg = readl(pmc + PMC_CTRL);
621         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
622         if (pdata->combined_req)
623                 reg &= ~TEGRA_POWER_PWRREQ_OE;
624         else
625                 reg |= TEGRA_POWER_PWRREQ_OE;
626         reg &= ~TEGRA_POWER_EFFECT_LP0;
627
628         switch (mode) {
629         case TEGRA_SUSPEND_LP0:
630                 if (pdata->combined_req) {
631                         reg |= TEGRA_POWER_PWRREQ_OE;
632                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
633                 }
634
635                 /* Enable DPD sample to trigger sampling pads data and direction
636                  * in which pad will be driven during lp0 mode*/
637                 writel(0x1, pmc + PMC_DPD_SAMPLE);
638                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
639                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
640                 tegra_is_dpd_mode = true;
641
642                 /* Set warmboot flag */
643                 boot_flag = readl(pmc + PMC_SCRATCH0);
644                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
645
646                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
647
648                 reg |= TEGRA_POWER_EFFECT_LP0;
649
650                 /* No break here. LP0 code falls through to write SCRATCH41 */
651         case TEGRA_SUSPEND_LP1:
652                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
653                 wmb();
654                 rate = clk_get_rate(tegra_clk_m);
655                 break;
656         case TEGRA_SUSPEND_LP2:
657                 rate = clk_get_rate(tegra_pclk);
658                 break;
659         case TEGRA_SUSPEND_NONE:
660                 return;
661         default:
662                 BUG();
663         }
664
665 #if defined(CONFIG_OF) && defined(CONFIG_COMMON_CLK)
666         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer);
667 #else
668         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
669 #endif
670
671         pmc_32kwritel(reg, PMC_CTRL);
672 }
673
674 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
675         [TEGRA_SUSPEND_NONE] = "none",
676         [TEGRA_SUSPEND_LP2] = "LP2",
677         [TEGRA_SUSPEND_LP1] = "LP1",
678         [TEGRA_SUSPEND_LP0] = "LP0",
679 };
680
681 int tegra210_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
682 {
683         int err = 0;
684         unsigned long arg;
685         int cpu = smp_processor_id();
686         struct psci_power_state ps = {
687                 .id = TEGRA210_CPUIDLE_SC7,
688                 .type = PSCI_POWER_STATE_TYPE_POWER_DOWN,
689                 .affinity_level = 2,
690         };
691
692         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
693                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
694                 err = -ENXIO;
695                 goto fail;
696         }
697
698         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
699                 err = -ENXIO;
700                 goto fail;
701         }
702
703         if (mode == TEGRA_SUSPEND_LP1) {
704                 if (tegra_bpmp_do_idle(cpu, TEGRA_PM_CC7,
705                                         TEGRA_PM_SC4) != 0)
706                         return -ENXIO;
707
708                 ps.id = TEGRA210_CPUIDLE_CC7;
709                 ps.affinity_level = 1;
710
711                 arg = psci_power_state_pack(ps);
712                 cpu_suspend(arg, NULL);
713
714                 return err;
715         }
716
717         if (tegra_bpmp_do_idle(cpu, TEGRA_PM_CC7,
718                                 TEGRA_PM_SC7) != 0) {
719                 err = -ENXIO;
720                 goto fail;
721         }
722
723         tegra_pm_prepare_sc7();
724
725         arg = psci_power_state_pack(ps);
726         cpu_suspend(arg, NULL);
727
728         tegra_pm_post_sc7();
729 fail:
730         return err;
731 }
732
733 static int tegra_suspend_enter(suspend_state_t state)
734 {
735         int ret = 0;
736         ktime_t delta;
737         struct timespec ts_entry, ts_exit;
738
739         if (pdata && pdata->board_suspend)
740                 pdata->board_suspend(current_suspend_mode,
741                         TEGRA_SUSPEND_BEFORE_PERIPHERAL);
742
743         read_persistent_clock(&ts_entry);
744
745 #ifdef CONFIG_ARCH_TEGRA_21x_SOC
746         ret = tegra210_suspend_dram(current_suspend_mode, 0);
747 #else
748         ret = tegra_suspend_dram(current_suspend_mode, 0);
749 #endif
750         if (ret) {
751                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
752                 goto abort_suspend;
753         }
754
755         read_persistent_clock(&ts_exit);
756
757         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
758                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
759
760                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
761                 tegra_dvfs_rail_pause(tegra_gpu_rail, delta, false);
762                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
763                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
764                 else
765                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
766         }
767
768 abort_suspend:
769         if (pdata && pdata->board_resume)
770                 pdata->board_resume(current_suspend_mode,
771                         TEGRA_RESUME_AFTER_PERIPHERAL);
772
773         return ret;
774 }
775
776 static void tegra_suspend_check_pwr_stats(void)
777 {
778         /* cpus and l2 are powered off later */
779         unsigned long pwrgate_partid_mask =
780                 (1 << TEGRA_POWERGATE_HEG)      |
781                 (1 << TEGRA_POWERGATE_SATA)     |
782                 (1 << TEGRA_POWERGATE_3D1)      |
783                 (1 << TEGRA_POWERGATE_3D)       |
784                 (1 << TEGRA_POWERGATE_VENC)     |
785                 (1 << TEGRA_POWERGATE_PCIE)     |
786                 (1 << TEGRA_POWERGATE_VDEC)     |
787                 (1 << TEGRA_POWERGATE_MPE);
788
789         int partid;
790
791         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
792                 if ((1 << partid) & pwrgate_partid_mask)
793                         if (tegra_powergate_is_powered(partid))
794                                 pr_debug("partition %s is left on before suspend\n",
795                                         tegra_powergate_get_name(partid));
796
797         return;
798 }
799
800 static void tegra_suspend_powergate_control(int partid, bool turn_off)
801 {
802         if (turn_off)
803                 tegra_powergate_partition(partid);
804         else
805                 tegra_unpowergate_partition(partid);
806 }
807
808 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
809 {
810         int err = 0;
811         u32 scratch37 = 0xDEADBEEF;
812         u32 reg;
813
814         bool tegra_suspend_vde_powergated = false;
815
816         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
817                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
818                 err = -ENXIO;
819                 goto fail;
820         }
821
822         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
823                 pr_info("LP0 not used due to unsupported wakeup events\n");
824                 mode = TEGRA_SUSPEND_LP1;
825         }
826
827         /* turn off VDE partition in LP1 */
828         if (mode == TEGRA_SUSPEND_LP1 &&
829                 tegra_powergate_is_powered(TEGRA_POWERGATE_VDEC)) {
830                 pr_info("turning off partition %s in LP1\n",
831                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
832                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, true);
833                 tegra_suspend_vde_powergated = true;
834         }
835
836         tegra_common_suspend();
837
838         tegra_pm_set(mode);
839
840         if (pdata && pdata->board_suspend)
841                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
842
843         local_fiq_disable();
844
845         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
846
847         if (mode == TEGRA_SUSPEND_LP0) {
848                 tegra_tsc_suspend();
849                 tegra_lp0_suspend_mc();
850                 tegra_tsc_wait_for_suspend();
851         }
852
853         cpu_cluster_pm_enter();
854         save_pll_state();
855         prepare_flow_controller();
856
857         if (tegra_get_chipid() != TEGRA_CHIPID_TEGRA13)
858                 flush_cache_all();
859
860         if (mode == TEGRA_SUSPEND_LP2)
861                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
862         else
863                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
864
865         resume_entry_time = 0;
866         if (mode != TEGRA_SUSPEND_LP0)
867                 resume_entry_time = tegra_read_usec_raw();
868
869         if (mode == TEGRA_SUSPEND_LP0) {
870                 tegra_tsc_resume();
871                 tegra_lp0_resume_mc();
872                 tegra_tsc_wait_for_resume();
873         }
874
875         /* if scratch37 was clobbered during LP1, restore it */
876         if (scratch37 != 0xDEADBEEF)
877                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
878
879         restore_csite_clock();
880         clear_flow_controller();
881         cpu_cluster_pm_exit();
882
883         /* for platforms where the core & CPU power requests are
884          * combined as a single request to the PMU, transition out
885          * of LP0 state by temporarily enabling both requests
886          */
887         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
888                 reg = readl(pmc + PMC_CTRL);
889                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
890                 pmc_32kwritel(reg, PMC_CTRL);
891                 reg &= ~TEGRA_POWER_PWRREQ_OE;
892                 pmc_32kwritel(reg, PMC_CTRL);
893         }
894
895         if (pdata && pdata->board_resume)
896                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
897
898         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
899
900         local_fiq_enable();
901
902         tegra_common_resume();
903
904         /* turn on VDE partition in LP1 */
905         if (mode == TEGRA_SUSPEND_LP1 && tegra_suspend_vde_powergated) {
906                 pr_info("turning on partition %s in LP1\n",
907                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
908                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, false);
909         }
910
911 fail:
912         return err;
913 }
914
915 static int tegra_suspend_valid(suspend_state_t state)
916 {
917         int valid = 1;
918
919         /* LP0 is not supported on T132 A01 silicon */
920         if (tegra_get_chipid() == TEGRA_CHIPID_TEGRA13 && \
921                 tegra_revision == TEGRA_REVISION_A01)
922                 valid = 0;
923
924         return valid;
925 }
926
927 static int tegra_suspend_prepare_late(void)
928 {
929         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) ||
930                         (current_suspend_mode == TEGRA_SUSPEND_LP1))
931                 tegra_suspend_check_pwr_stats();
932
933         return 0;
934 }
935
936 static void tegra_suspend_finish(void)
937 {
938         if (pdata && pdata->cpu_resume_boost) {
939                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
940                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
941                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
942         }
943 }
944
945 static const struct platform_suspend_ops tegra_suspend_ops = {
946         .prepare_late   = tegra_suspend_prepare_late,
947         .valid          = tegra_suspend_valid,
948         .finish         = tegra_suspend_finish,
949         .enter          = tegra_suspend_enter,
950 };
951
952 static ssize_t suspend_mode_show(struct kobject *kobj,
953                                         struct kobj_attribute *attr, char *buf)
954 {
955         char *start = buf;
956         char *end = buf + PAGE_SIZE;
957
958         start += scnprintf(start, end - start, "%s ",
959                                 tegra_suspend_name[current_suspend_mode]);
960         start += scnprintf(start, end - start, "\n");
961
962         return start - buf;
963 }
964
965 static ssize_t suspend_mode_store(struct kobject *kobj,
966                                         struct kobj_attribute *attr,
967                                         const char *buf, size_t n)
968 {
969         int len;
970         const char *name_ptr;
971         enum tegra_suspend_mode new_mode;
972
973         name_ptr = buf;
974         while (*name_ptr && !isspace(*name_ptr))
975                 name_ptr++;
976         len = name_ptr - buf;
977         if (!len)
978                 goto bad_name;
979         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
980         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
981                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
982                 pr_info("Illegal tegra suspend state: %s\n", buf);
983                 goto bad_name;
984         }
985
986         for (new_mode = TEGRA_SUSPEND_NONE;
987                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
988                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
989                         current_suspend_mode = new_mode;
990                         break;
991                 }
992         }
993
994 bad_name:
995         return n;
996 }
997
998 static struct kobj_attribute suspend_mode_attribute =
999         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1000
1001 static ssize_t suspend_resume_time_show(struct kobject *kobj,
1002                                         struct kobj_attribute *attr,
1003                                         char *buf)
1004 {
1005         return sprintf(buf, "%ums\n", ((u32)resume_time / 1000));
1006 }
1007
1008 static struct kobj_attribute suspend_resume_time_attribute =
1009         __ATTR(resume_time, 0444, suspend_resume_time_show, 0);
1010
1011 static ssize_t suspend_time_show(struct kobject *kobj,
1012                                         struct kobj_attribute *attr,
1013                                         char *buf)
1014 {
1015         return sprintf(buf, "%ums\n", ((u32)suspend_time / 1000));
1016 }
1017
1018 static struct kobj_attribute suspend_time_attribute =
1019         __ATTR(suspend_time, 0444, suspend_time_show, 0);
1020
1021 static struct kobject *suspend_kobj;
1022
1023 static int tegra_pm_enter_suspend(void)
1024 {
1025         suspend_cpu_dfll_mode(0);
1026         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1027         return 0;
1028 }
1029
1030 static void tegra_pm_enter_resume(void)
1031 {
1032         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1033         resume_cpu_dfll_mode(0);
1034 }
1035
1036 static void tegra_pm_enter_shutdown(void)
1037 {
1038         suspend_cpu_dfll_mode(0);
1039         pr_info("Shutting down tegra ...\n");
1040 }
1041
1042 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1043         .suspend = tegra_pm_enter_suspend,
1044         .resume = tegra_pm_enter_resume,
1045         .save = tegra_pm_enter_suspend,
1046         .restore = tegra_pm_enter_resume,
1047         .shutdown = tegra_pm_enter_shutdown,
1048 };
1049
1050 static __init int tegra_pm_enter_syscore_init(void)
1051 {
1052         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1053         return 0;
1054 }
1055 subsys_initcall(tegra_pm_enter_syscore_init);
1056 #endif
1057
1058 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1059 {
1060         u32 reg;
1061         u32 mode;
1062         struct pmc_pm_data *pm_dat;
1063         bool is_board_pdata = true;
1064
1065 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1066         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1067         BUG_ON(IS_ERR(tegra_dfll));
1068 #endif
1069         tegra_pclk = clk_get_sys(NULL, "pclk");
1070         BUG_ON(IS_ERR(tegra_pclk));
1071         tegra_clk_m = clk_get_sys(NULL, "clk_m");
1072         BUG_ON(IS_ERR(tegra_clk_m));
1073
1074         /* create the pdata from DT information */
1075         pm_dat = tegra_get_pm_data();
1076         if (pm_dat) {
1077                 pr_err("PMC dt information non-NULL %s\n", __func__);
1078                 is_board_pdata = false;
1079                 pdata = kzalloc(sizeof(struct tegra_suspend_platform_data),
1080                         GFP_KERNEL);
1081                 if (pm_dat->combined_req != plat->combined_req) {
1082                         pr_err("PMC DT attribute combined_req=%d, board value=%d\n",
1083                                 pm_dat->combined_req, plat->combined_req);
1084                         pdata->combined_req = plat->combined_req;
1085                 } else {
1086                         pdata->combined_req = pm_dat->combined_req;
1087                 }
1088                 if (pm_dat->sysclkreq_high != plat->sysclkreq_high) {
1089                         pr_err("PMC DT attribute sysclkreq_high=%d, board value=%d\n",
1090                                 pm_dat->sysclkreq_high, plat->sysclkreq_high);
1091                         pdata->sysclkreq_high = plat->sysclkreq_high;
1092                 } else {
1093                         pdata->sysclkreq_high = pm_dat->sysclkreq_high;
1094                 }
1095                 if (pm_dat->corereq_high != plat->corereq_high) {
1096                         pr_err("PMC DT attribute corereq_high=%d, board value=%d\n",
1097                                 pm_dat->corereq_high, plat->corereq_high);
1098                         pdata->corereq_high = plat->corereq_high;
1099                 } else {
1100                         pdata->corereq_high = pm_dat->corereq_high;
1101                 }
1102                 if (pm_dat->cpu_off_time != plat->cpu_off_timer) {
1103                         pr_err("PMC DT attribute cpu_off_timer=%d, board value=%ld\n",
1104                                 pm_dat->cpu_off_time, plat->cpu_off_timer);
1105                         pdata->cpu_off_timer = plat->cpu_off_timer;
1106                 } else {
1107                         pdata->cpu_off_timer = pm_dat->cpu_off_time;
1108                 }
1109                 if (pm_dat->cpu_good_time != plat->cpu_timer) {
1110                         pr_err("PMC DT attribute cpu_timer=%d, board value=%ld\n",
1111                                 pm_dat->cpu_good_time, plat->cpu_timer);
1112                         pdata->cpu_timer = plat->cpu_timer;
1113                 } else {
1114                         pdata->cpu_timer = pm_dat->cpu_good_time;
1115                 }
1116                 if (pm_dat->suspend_mode != plat->suspend_mode) {
1117                         pr_err("PMC DT attribute suspend_mode=%d, board value=%d\n",
1118                                 pm_dat->suspend_mode, plat->suspend_mode);
1119                         pdata->suspend_mode = plat->suspend_mode;
1120                 } else {
1121                         pdata->suspend_mode = pm_dat->suspend_mode;
1122                 }
1123                 if (pm_dat->cpu_suspend_freq != plat->cpu_suspend_freq) {
1124                         pr_err("PMC DT attribute cpu_suspend_freq=%d, cpu_suspend_freq=%d\n",
1125                                 pm_dat->cpu_suspend_freq,
1126                                 plat->cpu_suspend_freq);
1127                         pdata->cpu_suspend_freq = plat->cpu_suspend_freq;
1128                 } else {
1129                         pdata->cpu_suspend_freq = pm_dat->cpu_suspend_freq;
1130                 }
1131
1132                 /* FIXME: pmc_pm_data fields to be reused
1133                  *      core_osc_time, core_pmu_time, core_off_time
1134                  *      units of above fields is uSec while
1135                  *      platform data values are in ticks
1136                  */
1137                 /* FIXME: pmc_pm_data unused by downstream code
1138                  *      cpu_pwr_good_en, lp0_vec_size, lp0_vec_phy_addr
1139                  */
1140                 /* FIXME: add missing DT bindings taken from platform data */
1141                 pdata->core_timer = plat->core_timer;
1142                 pdata->core_off_timer = plat->core_off_timer;
1143                 pdata->board_suspend = plat->board_suspend;
1144                 pdata->board_resume = plat->board_resume;
1145                 pdata->sysclkreq_gpio = plat->sysclkreq_gpio;
1146                 pdata->cpu_lp2_min_residency = plat->cpu_lp2_min_residency;
1147                 pdata->cpu_resume_boost = plat->cpu_resume_boost;
1148 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
1149                 pdata->min_residency_vmin_fmin = plat->min_residency_vmin_fmin;
1150                 pdata->min_residency_ncpu_slow = plat->min_residency_ncpu_slow;
1151                 pdata->min_residency_ncpu_fast = plat->min_residency_ncpu_fast;
1152                 pdata->min_residency_crail = plat->min_residency_crail;
1153                 pdata->crail_up_early = plat->crail_up_early;
1154 #endif
1155                 pdata->min_residency_mclk_stop = plat->min_residency_mclk_stop;
1156                 pdata->usb_vbus_internal_wake = plat->usb_vbus_internal_wake;
1157                 pdata->usb_id_internal_wake = plat->usb_id_internal_wake;
1158                 pdata->suspend_dfll_bypass = plat->suspend_dfll_bypass;
1159                 pdata->resume_dfll_bypass = plat->resume_dfll_bypass;
1160         } else {
1161                 pr_err("PMC board data used in %s\n", __func__);
1162                 pdata = plat;
1163         }
1164         (void)reg;
1165         (void)mode;
1166
1167         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1168                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1169
1170 #ifndef CONFIG_PM_SLEEP
1171         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1172                 pr_warn("%s: Suspend requires CONFIG_PM_SLEEP -- "
1173                            "disabling suspend\n", __func__);
1174                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1175         }
1176 #else
1177
1178         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1179                 tegra_lp0_vec_relocate) {
1180                 unsigned char *reloc_lp0;
1181                 unsigned long tmp;
1182                 void __iomem *orig;
1183                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1184                                         GFP_KERNEL);
1185                 WARN_ON(!reloc_lp0);
1186                 if (!reloc_lp0) {
1187                         pr_err("%s: Failed to allocate reloc_lp0\n",
1188                                 __func__);
1189                         goto out;
1190                 }
1191
1192                 /* Avoid a kmemleak false positive. The allocated memory
1193                  * block is later referenced by a physical address (i.e.
1194                  * tegra_lp0_vec_start) which kmemleak can't detect.
1195                  */
1196                 kmemleak_not_leak(reloc_lp0);
1197
1198                 orig = ioremap_wc(tegra_lp0_vec_start, tegra_lp0_vec_size);
1199                 WARN_ON(!orig);
1200                 if (!orig) {
1201                         pr_err("%s: Failed to map tegra_lp0_vec_start %llx\n",
1202                                 __func__, tegra_lp0_vec_start);
1203                         kfree(reloc_lp0);
1204                         goto out;
1205                 }
1206
1207                 tmp = (unsigned long) reloc_lp0;
1208                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1209                 reloc_lp0 = (unsigned char *)tmp;
1210                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1211                 wmb();
1212                 iounmap(orig);
1213                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1214         }
1215
1216 out:
1217         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1218                 pr_warn("%s: Suspend mode LP0 requested, no lp0_vec "
1219                            "provided by bootlader -- disabling LP0\n",
1220                            __func__);
1221                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1222         }
1223
1224         /* Initialize SOC-specific data */
1225 #ifdef CONFIG_ARCH_TEGRA_13x_SOC
1226         tegra_soc_suspend_init();
1227
1228         if (!tegra_tear_down_cpu || !tegra_sleep_core_finish) {
1229                 pr_err("%s: unable to obtain suspend info -- "
1230                                 "disabling LP0\n", __func__);
1231                 current_suspend_mode = TEGRA_SUSPEND_LP2;
1232         }
1233 #endif
1234
1235         /* Always enable CPU power request; just normal polarity is supported */
1236         reg = readl(pmc + PMC_CTRL);
1237         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1238         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1239         pmc_32kwritel(reg, PMC_CTRL);
1240
1241         /* Configure core power request and system clock control if LP0
1242            is supported */
1243         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1244         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1245
1246         reg = readl(pmc + PMC_CTRL);
1247
1248         if (!pdata->sysclkreq_high)
1249                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1250         else
1251                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1252
1253         if (!pdata->corereq_high)
1254                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1255         else
1256                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1257
1258         /* configure output inverters while the request is tristated */
1259         pmc_32kwritel(reg, PMC_CTRL);
1260
1261         /* now enable requests */
1262         reg |= TEGRA_POWER_SYSCLK_OE;
1263         if (!pdata->combined_req)
1264                 reg |= TEGRA_POWER_PWRREQ_OE;
1265         pmc_32kwritel(reg, PMC_CTRL);
1266
1267         if (pdata->sysclkreq_gpio) {
1268                 reg = readl(pmc + PMC_DPAD_ORIDE);
1269                 reg &= ~TEGRA_DPAD_ORIDE_SYS_CLK_REQ;
1270                 pmc_32kwritel(reg, PMC_DPAD_ORIDE);
1271         }
1272
1273         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1274                 tegra_lp0_suspend_init();
1275
1276         suspend_set_ops(&tegra_suspend_ops);
1277
1278         /* Create /sys/power/suspend/type */
1279         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1280         if (suspend_kobj) {
1281                 if (sysfs_create_file(suspend_kobj,
1282                                                 &suspend_mode_attribute.attr))
1283                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1284                                                                 __func__);
1285                 if (sysfs_create_file(suspend_kobj,
1286                                         &suspend_resume_time_attribute.attr))
1287                         pr_err("%s: sysfs_create_file resume_time failed!\n",
1288                                                                 __func__);
1289                 if (sysfs_create_file(suspend_kobj,
1290                                         &suspend_time_attribute.attr))
1291                         pr_err("%s: sysfs_create_file suspend_time failed!\n",
1292                                                                 __func__);
1293         }
1294
1295         /* clear io dpd settings before kernel */
1296         tegra_bl_io_dpd_cleanup();
1297
1298 #endif
1299         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1300                 tegra_pd_in_idle(false);
1301
1302         current_suspend_mode = plat->suspend_mode;
1303 }
1304
1305 #ifdef CONFIG_ARM_ARCH_TIMER
1306
1307 static u32 tsc_suspend_start;
1308 static u32 tsc_resume_start;
1309
1310 #define pmc_writel(value, reg) \
1311                 writel(value, pmc + (reg))
1312 #define pmc_readl(reg) \
1313                 readl(pmc + (reg))
1314
1315 #define PMC_DPD_ENABLE_TSC_MULT_ENABLE  (1 << 1)
1316 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
1317 #define PMC_DPD_ENABLE_ON               (1 << 0)
1318 #endif
1319
1320 #define PMC_TSC_MULT                    0x2b4
1321 #define PMC_TSC_MULT_FREQ_STS           (1 << 16)
1322
1323 #define TSC_TIMEOUT_US                  32
1324
1325 void tegra_tsc_suspend(void)
1326 {
1327         u32 reg = pmc_readl(PMC_DPD_ENABLE);
1328         BUG_ON(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE);
1329         reg |= PMC_DPD_ENABLE_TSC_MULT_ENABLE;
1330         pmc_writel(reg, PMC_DPD_ENABLE);
1331         tsc_suspend_start = tegra_read_usec_raw();
1332 }
1333
1334 void tegra_tsc_resume(void)
1335 {
1336         u32 reg = pmc_readl(PMC_DPD_ENABLE);
1337         BUG_ON(!(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE));
1338         reg &= ~PMC_DPD_ENABLE_TSC_MULT_ENABLE;
1339 #if defined(CONFIG_ARCH_TEGRA_12x_SOC)
1340         /*
1341          * FIXME: T12x SW WAR -
1342          * Resume ensures DPD_ENABLE is 0 when writing
1343          * TSC_MULT_ENABLE, else PMC wake status gets reset
1344          */
1345         reg &= ~PMC_DPD_ENABLE_ON;
1346 #endif
1347         pmc_writel(reg, PMC_DPD_ENABLE);
1348         tsc_resume_start = tegra_read_usec_raw();
1349 }
1350
1351 void tegra_tsc_wait_for_suspend(void)
1352 {
1353         while ((tegra_read_usec_raw() - tsc_suspend_start) <
1354                 TSC_TIMEOUT_US) {
1355                 if (pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS)
1356                         break;
1357                 cpu_relax();
1358         }
1359 }
1360
1361 void tegra_tsc_wait_for_resume(void)
1362 {
1363         while ((tegra_read_usec_raw() - tsc_resume_start) <
1364                 TSC_TIMEOUT_US) {
1365                 if (!(pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS))
1366                         break;
1367                 cpu_relax();
1368         }
1369 }
1370 #endif