Revert "arm: tegra: optimize L2 enable/disable paths for secureos"
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58 #include <asm/smp_plat.h>
59
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62 #include <mach/hardware.h>
63
64 #include "board.h"
65 #include "clock.h"
66 #include "common.h"
67 #include "cpuidle.h"
68 #include "fuse.h"
69 #include "gic.h"
70 #include "iomap.h"
71 #include "pm.h"
72 #include "pm-irq.h"
73 #include "reset.h"
74 #include "sleep.h"
75 #include "timer.h"
76 #include "dvfs.h"
77 #include "cpu-tegra.h"
78
79 struct suspend_context {
80         /*
81          * The next 7 values are referenced by offset in __restart_plls
82          * in headsmp-t2.S, and should not be moved
83          */
84         u32 pllx_misc;
85         u32 pllx_base;
86         u32 pllp_misc;
87         u32 pllp_base;
88         u32 pllp_outa;
89         u32 pllp_outb;
90         u32 pll_timeout;
91
92         u32 cpu_burst;
93         u32 clk_csite_src;
94         u32 cclk_divider;
95
96         u32 mc[3];
97         u8 uart[5];
98
99         struct tegra_twd_context twd;
100 #ifdef CONFIG_ARM_ARCH_TIMER
101         struct arch_timer_context arch_timer;
102 #endif
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
107 static pgd_t *tegra_pgd;
108 static DEFINE_SPINLOCK(tegra_lp2_lock);
109 static cpumask_t tegra_in_lp2;
110 static cpumask_t *iram_cpu_lp2_mask;
111 static unsigned long *iram_cpu_lp1_mask;
112 static u8 *iram_save;
113 static unsigned long iram_save_size;
114 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
115 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
116 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
117 static int tegra_last_pclk;
118 #endif
119
120 struct suspend_context tegra_sctx;
121
122 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
123 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
124 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
125 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
126 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
127 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
128 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
129 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
130 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
131
132 #define PMC_CTRL                0x0
133 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
134 #define PMC_WAKE_MASK           0xc
135 #define PMC_WAKE_LEVEL          0x10
136 #define PMC_DPAD_ORIDE          0x1C
137 #define PMC_WAKE_DELAY          0xe0
138 #define PMC_DPD_SAMPLE          0x20
139 #define PMC_IO_DPD_REQ_0        0x1b8
140 #define PMC_IO_DPD2_REQ_0       0X1C0
141
142 #define PMC_WAKE_STATUS         0x14
143 #define PMC_SW_WAKE_STATUS      0x18
144 #define PMC_COREPWRGOOD_TIMER   0x3c
145 #define PMC_CPUPWRGOOD_TIMER    0xc8
146 #define PMC_CPUPWROFF_TIMER     0xcc
147 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
148
149 #define PMC_PWRGATE_TOGGLE      0x30
150 #define PWRGATE_TOGGLE_START    (1 << 8)
151 #define UN_PWRGATE_CPU          \
152         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
153
154 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
155 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
156 #endif
157
158 #define CLK_RESET_CCLK_BURST    0x20
159 #define CLK_RESET_CCLK_DIVIDER  0x24
160 #define CLK_RESET_PLLC_BASE     0x80
161 #define CLK_RESET_PLLM_BASE     0x90
162 #define CLK_RESET_PLLX_BASE     0xe0
163 #define CLK_RESET_PLLX_MISC     0xe4
164 #define CLK_RESET_PLLP_BASE     0xa0
165 #define CLK_RESET_PLLP_OUTA     0xa4
166 #define CLK_RESET_PLLP_OUTB     0xa8
167 #define CLK_RESET_PLLP_MISC     0xac
168
169 #define CLK_RESET_SOURCE_CSITE  0x1d4
170
171 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
172 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
173 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
174 #define CLK_RESET_CCLK_IDLE_POLICY         1
175 #define CLK_RESET_CCLK_RUN_POLICY          2
176 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
177 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
178
179 #define EMC_MRW_0               0x0e8
180 #define EMC_MRW_DEV_SELECTN     30
181 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
182
183 #define MC_SECURITY_START       0x6c
184 #define MC_SECURITY_SIZE        0x70
185 #define MC_SECURITY_CFG2        0x7c
186
187 static struct clk *tegra_pclk;
188 static const struct tegra_suspend_platform_data *pdata;
189 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
190
191 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
192 enum tegra_cluster_switch_time_id {
193         tegra_cluster_switch_time_id_start = 0,
194         tegra_cluster_switch_time_id_prolog,
195         tegra_cluster_switch_time_id_switch,
196         tegra_cluster_switch_time_id_epilog,
197         tegra_cluster_switch_time_id_max
198 };
199
200 static unsigned long
201                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
202 #define tegra_cluster_switch_time(flags, id) \
203         do { \
204                 barrier(); \
205                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
206                         void __iomem *timer_us = \
207                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
208                         if (id < tegra_cluster_switch_time_id_max) \
209                                 tegra_cluster_switch_times[id] = \
210                                                         readl(timer_us); \
211                                 wmb(); \
212                 } \
213                 barrier(); \
214         } while(0)
215 #else
216 #define tegra_cluster_switch_time(flags, id) do {} while(0)
217 #endif
218
219 #ifdef CONFIG_PM_SLEEP
220 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
221         [TEGRA_SUSPEND_NONE]    = "none",
222         [TEGRA_SUSPEND_LP2]     = "lp2",
223         [TEGRA_SUSPEND_LP1]     = "lp1",
224         [TEGRA_SUSPEND_LP0]     = "lp0",
225 };
226
227 unsigned long tegra_cpu_power_good_time(void)
228 {
229         if (WARN_ON_ONCE(!pdata))
230                 return 5000;
231
232         return pdata->cpu_timer;
233 }
234
235 unsigned long tegra_cpu_power_off_time(void)
236 {
237         if (WARN_ON_ONCE(!pdata))
238                 return 5000;
239
240         return pdata->cpu_off_timer;
241 }
242
243 unsigned long tegra_cpu_lp2_min_residency(void)
244 {
245         if (WARN_ON_ONCE(!pdata))
246                 return 2000;
247
248         return pdata->cpu_lp2_min_residency;
249 }
250
251 /*
252  * create_suspend_pgtable
253  *
254  * Creates a page table with identity mappings of physical memory and IRAM
255  * for use when the MMU is off, in addition to all the regular kernel mappings.
256  */
257 static __init int create_suspend_pgtable(void)
258 {
259         tegra_pgd = pgd_alloc(&init_mm);
260         if (!tegra_pgd)
261                 return -ENOMEM;
262
263         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
264         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
265                 high_memory, 0);
266         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
267                 IO_IRAM_VIRT + SECTION_SIZE, 0);
268
269         /* inner/outer write-back/write-allocate, sharable */
270         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
271
272         return 0;
273 }
274
275 /* ensures that sufficient time is passed for a register write to
276  * serialize into the 32KHz domain */
277 static void pmc_32kwritel(u32 val, unsigned long offs)
278 {
279         writel(val, pmc + offs);
280         udelay(130);
281 }
282
283 static void set_power_timers(unsigned long us_on, unsigned long us_off,
284                              long rate)
285 {
286         static unsigned long last_us_off = 0;
287         unsigned long long ticks;
288         unsigned long long pclk;
289
290         if (WARN_ON_ONCE(rate <= 0))
291                 pclk = 100000000;
292         else
293                 pclk = rate;
294
295         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
296                 ticks = (us_on * pclk) + 999999ull;
297                 do_div(ticks, 1000000);
298                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
299
300                 ticks = (us_off * pclk) + 999999ull;
301                 do_div(ticks, 1000000);
302                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
303                 wmb();
304         }
305         tegra_last_pclk = pclk;
306         last_us_off = us_off;
307 }
308
309 /*
310  * restore_cpu_complex
311  *
312  * restores cpu clock setting, clears flow controller
313  *
314  * Always called on CPU 0.
315  */
316 static void restore_cpu_complex(u32 mode)
317 {
318         int cpu = smp_processor_id();
319         unsigned int reg;
320 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
321         unsigned int policy;
322 #endif
323
324         BUG_ON(cpu != 0);
325
326 #ifdef CONFIG_SMP
327         cpu = cpu_logical_map(cpu);
328 #endif
329
330 /*
331  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
332  * or restored by common clock suspend/resume procedures. Hence, we don't
333  * need it here.
334  */
335 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
336         /* Is CPU complex already running on PLLX? */
337         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
338         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
339
340         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
341                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
342         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
343                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
344         else
345                 BUG();
346
347         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
348                 /* restore PLLX settings if CPU is on different PLL */
349                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
350                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
351
352                 /* wait for PLL stabilization if PLLX was enabled */
353                 if (tegra_sctx.pllx_base & (1<<30)) {
354 #if USE_PLL_LOCK_BITS
355                         /* Enable lock detector */
356                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
357                         reg |= 1<<18;
358                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
359                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
360                                  (1<<27)))
361                                 cpu_relax();
362
363                         udelay(PLL_POST_LOCK_DELAY);
364 #else
365                         udelay(300);
366 #endif
367                 }
368         }
369
370         /* Restore original burst policy setting for calls resulting from CPU
371            LP2 in idle or system suspend; keep cluster switch prolog setting
372            intact. */
373         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
374                 writel(tegra_sctx.cclk_divider, clk_rst +
375                        CLK_RESET_CCLK_DIVIDER);
376                 writel(tegra_sctx.cpu_burst, clk_rst +
377                        CLK_RESET_CCLK_BURST);
378         }
379 #endif
380         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
381
382         /* Do not power-gate CPU 0 when flow controlled */
383         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
384         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
385         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
386         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
387         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
388         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
389         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
390
391         /* If an immedidate cluster switch is being perfomed, restore the
392            local timer registers. For calls resulting from CPU LP2 in
393            idle or system suspend, the local timer was shut down and
394            timekeeping switched over to the global system timer. In this
395            case keep local timer disabled, and restore only periodic load. */
396         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
397                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
398 #ifdef CONFIG_ARM_ARCH_TIMER
399                 tegra_sctx.arch_timer.cntp_ctl = 0;
400 #endif
401 #ifdef CONFIG_HAVE_ARM_TWD
402                 tegra_sctx.twd.twd_ctrl = 0;
403 #endif
404         }
405 #ifdef CONFIG_ARM_ARCH_TIMER
406         arch_timer_resume(&tegra_sctx.arch_timer);
407 #endif
408 #ifdef CONFIG_HAVE_ARM_TWD
409         tegra_twd_resume(&tegra_sctx.twd);
410 #endif
411 }
412
413 /*
414  * suspend_cpu_complex
415  *
416  * saves pll state for use by restart_plls, prepares flow controller for
417  * transition to suspend state
418  *
419  * Must always be called on cpu 0.
420  */
421 static void suspend_cpu_complex(u32 mode)
422 {
423         int cpu = smp_processor_id();
424         unsigned int reg;
425         int i;
426
427         BUG_ON(cpu != 0);
428
429 #ifdef CONFIG_SMP
430         cpu = cpu_logical_map(cpu);
431 #endif
432         /* switch coresite to clk_m, save off original source */
433         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
434         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
435
436         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
437         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
438         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
439         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
440         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
441         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
442         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
443         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
444
445 #ifdef CONFIG_HAVE_ARM_TWD
446         tegra_twd_suspend(&tegra_sctx.twd);
447 #endif
448 #ifdef CONFIG_ARM_ARCH_TIMER
449         arch_timer_suspend(&tegra_sctx.arch_timer);
450 #endif
451
452         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
453         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
454         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
455         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
456         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
457 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
458         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
459 #else
460         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
461 #endif
462         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
463         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
464
465         for (i = 0; i < num_possible_cpus(); i++) {
466                 if (i == cpu)
467                         continue;
468                 reg = readl(FLOW_CTRL_CPU_CSR(i));
469                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
470                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
471                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
472         }
473
474         tegra_gic_cpu_disable(true);
475 }
476
477 void tegra_clear_cpu_in_lp2(int cpu)
478 {
479         spin_lock(&tegra_lp2_lock);
480         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
481         cpumask_clear_cpu(cpu, &tegra_in_lp2);
482
483         /* Update the IRAM copy used by the reset handler. The IRAM copy
484            can't use used directly by cpumask_clear_cpu() because it uses
485            LDREX/STREX which requires the addressed location to be inner
486            cacheable and sharable which IRAM isn't. */
487         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
488         dsb();
489
490         spin_unlock(&tegra_lp2_lock);
491 }
492
493 bool tegra_set_cpu_in_lp2(int cpu)
494 {
495         bool last_cpu = false;
496
497         spin_lock(&tegra_lp2_lock);
498         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
499         cpumask_set_cpu(cpu, &tegra_in_lp2);
500
501         /* Update the IRAM copy used by the reset handler. The IRAM copy
502            can't use used directly by cpumask_set_cpu() because it uses
503            LDREX/STREX which requires the addressed location to be inner
504            cacheable and sharable which IRAM isn't. */
505         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
506         dsb();
507
508         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
509                 last_cpu = true;
510 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
511         else if (cpu == 1)
512                 tegra2_cpu_set_resettable_soon();
513 #endif
514
515         spin_unlock(&tegra_lp2_lock);
516         return last_cpu;
517 }
518
519 static void tegra_sleep_core(enum tegra_suspend_mode mode,
520                              unsigned long v2p)
521 {
522 #ifdef CONFIG_TRUSTED_FOUNDATIONS
523         if (mode == TEGRA_SUSPEND_LP0) {
524                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
525                                   virt_to_phys(tegra_resume));
526         } else {
527                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
528                                   (TEGRA_RESET_HANDLER_BASE +
529                                    tegra_cpu_reset_handler_offset));
530         }
531 #endif
532 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
533         cpu_suspend(v2p, tegra2_sleep_core_finish);
534 #else
535         cpu_suspend(v2p, tegra3_sleep_core_finish);
536 #endif
537 }
538
539 static inline void tegra_sleep_cpu(unsigned long v2p)
540 {
541 #ifdef CONFIG_TRUSTED_FOUNDATIONS
542         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
543                           (TEGRA_RESET_HANDLER_BASE +
544                            tegra_cpu_reset_handler_offset));
545 #endif
546         cpu_suspend(v2p, tegra_sleep_cpu_finish);
547 }
548
549 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
550 {
551         u32 reg;
552         unsigned int remain;
553 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
554         pgd_t *pgd;
555 #endif
556
557         /* Only the last cpu down does the final suspend steps */
558         reg = readl(pmc + PMC_CTRL);
559         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
560         if (pdata->combined_req)
561                 reg &= ~TEGRA_POWER_PWRREQ_OE;
562         else
563                 reg |= TEGRA_POWER_PWRREQ_OE;
564
565         reg &= ~TEGRA_POWER_EFFECT_LP0;
566         writel(reg, pmc + PMC_CTRL);
567
568         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
569
570         /*
571          * We can use clk_get_rate_all_locked() here, because all other cpus
572          * are in LP2 state and irqs are disabled
573          */
574         if (flags & TEGRA_POWER_CLUSTER_MASK) {
575                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
576                 set_power_timers(pdata->cpu_timer, 2,
577                         clk_get_rate_all_locked(tegra_pclk));
578                 if (flags & TEGRA_POWER_CLUSTER_G) {
579                         /*
580                          * To reduce the vdd_cpu up latency when LP->G
581                          * transition. Before the transition, enable
582                          * the vdd_cpu rail.
583                          */
584                         if (is_lp_cluster())
585                                 writel(UN_PWRGATE_CPU,
586                                        pmc + PMC_PWRGATE_TOGGLE);
587                 }
588                 tegra_cluster_switch_prolog(flags);
589         } else {
590                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
591                         clk_get_rate_all_locked(tegra_pclk));
592 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
593                 reg = readl(FLOW_CTRL_CPU_CSR(0));
594                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
595                 if (is_lp_cluster()) {
596                         /* for LP cluster, there is no option for rail gating */
597                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
598                                                 TEGRA_POWER_CLUSTER_PART_MASK)
599                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
600                         else if (flags)
601                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
602                 }
603                 else {
604                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
605                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
606                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
607                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
608                 }
609                 writel(reg, FLOW_CTRL_CPU_CSR(0));
610 #endif
611         }
612
613         if (sleep_time)
614                 tegra_lp2_set_trigger(sleep_time);
615
616         cpu_cluster_pm_enter();
617         suspend_cpu_complex(flags);
618         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
619 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
620         flush_cache_all();
621         /*
622          * No need to flush complete L2. Cleaning kernel and IO mappings
623          * is enough for the LP code sequence that has L2 disabled but
624          * MMU on.
625          */
626         pgd = cpu_get_pgd();
627         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
628                           __pa(pgd + PTRS_PER_PGD));
629         outer_disable();
630 #endif
631         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
632
633         tegra_init_cache(false);
634         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
635         restore_cpu_complex(flags);
636         cpu_cluster_pm_exit();
637
638         remain = tegra_lp2_timer_remain();
639         if (sleep_time)
640                 tegra_lp2_set_trigger(0);
641
642         if (flags & TEGRA_POWER_CLUSTER_MASK) {
643                 tegra_cluster_switch_epilog(flags);
644                 if (is_idle_task(current))
645                         trace_cpu_cluster_rcuidle(POWER_CPU_CLUSTER_DONE);
646                 else
647                         trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
648         }
649         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
650
651 #if INSTRUMENT_CLUSTER_SWITCH
652         if (flags & TEGRA_POWER_CLUSTER_MASK) {
653                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
654                         is_lp_cluster() ? "G=>LP" : "LP=>G",
655                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
656                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
657                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
658                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
659                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
660                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
661                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
662                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
663         }
664 #endif
665         return remain;
666 }
667
668 static int tegra_common_suspend(void)
669 {
670         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
671
672         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
673         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
674         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
675
676         /* copy the reset vector and SDRAM shutdown code into IRAM */
677         memcpy(iram_save, iram_code, iram_save_size);
678         memcpy(iram_code, tegra_iram_start(), iram_save_size);
679
680         return 0;
681 }
682
683 static void tegra_common_resume(void)
684 {
685         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
686 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
687         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
688 #endif
689
690         /* Clear DPD sample */
691         writel(0x0, pmc + PMC_DPD_SAMPLE);
692
693         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
694         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
695         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
696 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
697         /* trigger emc mode write */
698         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
699 #endif
700         /* clear scratch registers shared by suspend and the reset pen */
701         writel(0x0, pmc + PMC_SCRATCH39);
702         writel(0x0, pmc + PMC_SCRATCH41);
703
704         /* restore IRAM */
705         memcpy(iram_code, iram_save, iram_save_size);
706 }
707
708 static int tegra_suspend_prepare_late(void)
709 {
710 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
711         disable_irq(INT_SYS_STATS_MON);
712 #endif
713         return 0;
714 }
715
716 static void tegra_suspend_wake(void)
717 {
718 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
719         enable_irq(INT_SYS_STATS_MON);
720 #endif
721 }
722
723 static void tegra_pm_set(enum tegra_suspend_mode mode)
724 {
725         u32 reg, boot_flag;
726         unsigned long rate = 32768;
727
728         reg = readl(pmc + PMC_CTRL);
729         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
730         if (pdata->combined_req)
731                 reg &= ~TEGRA_POWER_PWRREQ_OE;
732         else
733                 reg |= TEGRA_POWER_PWRREQ_OE;
734         reg &= ~TEGRA_POWER_EFFECT_LP0;
735
736         switch (mode) {
737         case TEGRA_SUSPEND_LP0:
738 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
739                 rate = clk_get_rate_all_locked(tegra_pclk);
740 #endif
741                 if (pdata->combined_req) {
742                         reg |= TEGRA_POWER_PWRREQ_OE;
743                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
744                 }
745
746                 /*
747                  * LP0 boots through the AVP, which then resumes the AVP to
748                  * the address in scratch 39, and the cpu to the address in
749                  * scratch 41 to tegra_resume
750                  */
751                 writel(0x0, pmc + PMC_SCRATCH39);
752
753                 /* Enable DPD sample to trigger sampling pads data and direction
754                  * in which pad will be driven during lp0 mode*/
755                 writel(0x1, pmc + PMC_DPD_SAMPLE);
756 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
757                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ_0);
758                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ_0);
759 #endif
760 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
761                 /* this is needed only for T11x, not for other chips */
762                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
763 #endif
764
765                 /* Set warmboot flag */
766                 boot_flag = readl(pmc + PMC_SCRATCH0);
767                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
768
769                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
770
771                 reg |= TEGRA_POWER_EFFECT_LP0;
772                 /* No break here. LP0 code falls through to write SCRATCH41 */
773         case TEGRA_SUSPEND_LP1:
774                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
775                 wmb();
776                 break;
777         case TEGRA_SUSPEND_LP2:
778                 rate = clk_get_rate(tegra_pclk);
779                 break;
780         case TEGRA_SUSPEND_NONE:
781                 return;
782         default:
783                 BUG();
784         }
785
786         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
787
788         pmc_32kwritel(reg, PMC_CTRL);
789 }
790
791 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
792         [TEGRA_SUSPEND_NONE] = "none",
793         [TEGRA_SUSPEND_LP2] = "LP2",
794         [TEGRA_SUSPEND_LP1] = "LP1",
795         [TEGRA_SUSPEND_LP0] = "LP0",
796 };
797
798 static int tegra_suspend_enter(suspend_state_t state)
799 {
800         int ret;
801         ktime_t delta;
802         struct timespec ts_entry, ts_exit;
803
804         if (pdata && pdata->board_suspend)
805                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
806
807         read_persistent_clock(&ts_entry);
808
809         ret = tegra_suspend_dram(current_suspend_mode, 0);
810         if (ret) {
811                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
812                 goto abort_suspend;
813         }
814
815         read_persistent_clock(&ts_exit);
816
817         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
818                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
819
820                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
821                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
822                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
823                 else
824                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
825         }
826
827 abort_suspend:
828         if (pdata && pdata->board_resume)
829                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
830
831         return ret;
832 }
833
834 static void tegra_suspend_check_pwr_stats(void)
835 {
836         /* cpus and l2 are powered off later */
837         unsigned long pwrgate_partid_mask =
838 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
839                 (1 << TEGRA_POWERGATE_HEG)      |
840                 (1 << TEGRA_POWERGATE_SATA)     |
841                 (1 << TEGRA_POWERGATE_3D1)      |
842 #endif
843                 (1 << TEGRA_POWERGATE_3D)       |
844                 (1 << TEGRA_POWERGATE_VENC)     |
845                 (1 << TEGRA_POWERGATE_PCIE)     |
846                 (1 << TEGRA_POWERGATE_VDEC)     |
847                 (1 << TEGRA_POWERGATE_MPE);
848
849         int partid;
850
851         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
852                 if ((1 << partid) & pwrgate_partid_mask)
853                         if (tegra_powergate_is_powered(partid))
854                                 pr_warning("partition %s is left on before suspend\n",
855                                         tegra_powergate_get_name(partid));
856
857         return;
858 }
859
860 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
861 {
862         int err = 0;
863         u32 scratch37 = 0xDEADBEEF;
864         u32 reg;
865
866         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
867                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
868                 err = -ENXIO;
869                 goto fail;
870         }
871
872         if (tegra_is_voice_call_active()) {
873                 /* backup the current value of scratch37 */
874                 scratch37 = readl(pmc + PMC_SCRATCH37);
875
876                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
877                 reg = TEGRA_POWER_LP1_AUDIO;
878                 pmc_32kwritel(reg, PMC_SCRATCH37);
879         }
880
881         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
882                 pr_info("LP0 not used due to unsupported wakeup events\n");
883                 mode = TEGRA_SUSPEND_LP1;
884         }
885
886         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
887                 tegra_suspend_check_pwr_stats();
888
889         tegra_common_suspend();
890
891         tegra_pm_set(mode);
892
893         if (pdata && pdata->board_suspend)
894                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
895
896         local_fiq_disable();
897
898         trace_cpu_suspend(CPU_SUSPEND_START);
899
900         if (mode == TEGRA_SUSPEND_LP0) {
901 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
902                 reg = readl(pmc + PMC_SCRATCH4);
903                 if (is_lp_cluster())
904                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
905                 else
906                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
907                 pmc_32kwritel(reg, PMC_SCRATCH4);
908 #endif
909                 tegra_tsc_suspend();
910                 tegra_lp0_suspend_mc();
911                 tegra_cpu_reset_handler_save();
912                 tegra_tsc_wait_for_suspend();
913         }
914         else if (mode == TEGRA_SUSPEND_LP1)
915                 *iram_cpu_lp1_mask = 1;
916
917         suspend_cpu_complex(flags);
918
919 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
920         /* In case of LP0, program external power gating accordinly */
921         if (mode == TEGRA_SUSPEND_LP0) {
922                 reg = readl(FLOW_CTRL_CPU_CSR(0));
923                 if (is_lp_cluster())
924                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
925                 else
926                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
927                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
928         }
929 #endif
930
931         flush_cache_all();
932         outer_flush_all();
933         outer_disable();
934
935         if (mode == TEGRA_SUSPEND_LP2)
936                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
937         else
938                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
939
940         tegra_init_cache(true);
941
942         if (mode == TEGRA_SUSPEND_LP0) {
943 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
944                 reg = readl(pmc+PMC_CTRL);
945                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
946                 pmc_32kwritel(reg, PMC_CTRL);
947 #endif
948                 tegra_tsc_resume();
949                 tegra_cpu_reset_handler_restore();
950                 tegra_lp0_resume_mc();
951                 tegra_tsc_wait_for_resume();
952         } else if (mode == TEGRA_SUSPEND_LP1)
953                 *iram_cpu_lp1_mask = 0;
954
955         /* if scratch37 was clobbered during LP1, restore it */
956         if (scratch37 != 0xDEADBEEF)
957                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
958
959         restore_cpu_complex(flags);
960
961         /* for platforms where the core & CPU power requests are
962          * combined as a single request to the PMU, transition out
963          * of LP0 state by temporarily enabling both requests
964          */
965         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
966                 reg = readl(pmc + PMC_CTRL);
967                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
968                 pmc_32kwritel(reg, PMC_CTRL);
969                 reg &= ~TEGRA_POWER_PWRREQ_OE;
970                 pmc_32kwritel(reg, PMC_CTRL);
971         }
972
973         if (pdata && pdata->board_resume)
974                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
975
976         trace_cpu_suspend(CPU_SUSPEND_DONE);
977
978         local_fiq_enable();
979
980         tegra_common_resume();
981
982 fail:
983         return err;
984 }
985
986 /*
987  * Function pointers to optional board specific function
988  */
989 void (*tegra_deep_sleep)(int);
990 EXPORT_SYMBOL(tegra_deep_sleep);
991
992 static int tegra_suspend_prepare(void)
993 {
994         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
995                 tegra_deep_sleep(1);
996         return 0;
997 }
998
999 static void tegra_suspend_finish(void)
1000 {
1001         if (pdata && pdata->cpu_resume_boost) {
1002                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1003                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1004                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1005         }
1006
1007         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1008                 tegra_deep_sleep(0);
1009 }
1010
1011 static const struct platform_suspend_ops tegra_suspend_ops = {
1012         .valid          = suspend_valid_only_mem,
1013         .prepare        = tegra_suspend_prepare,
1014         .finish         = tegra_suspend_finish,
1015         .prepare_late   = tegra_suspend_prepare_late,
1016         .wake           = tegra_suspend_wake,
1017         .enter          = tegra_suspend_enter,
1018 };
1019
1020 static ssize_t suspend_mode_show(struct kobject *kobj,
1021                                         struct kobj_attribute *attr, char *buf)
1022 {
1023         char *start = buf;
1024         char *end = buf + PAGE_SIZE;
1025
1026         start += scnprintf(start, end - start, "%s ", \
1027                                 tegra_suspend_name[current_suspend_mode]);
1028         start += scnprintf(start, end - start, "\n");
1029
1030         return start - buf;
1031 }
1032
1033 static ssize_t suspend_mode_store(struct kobject *kobj,
1034                                         struct kobj_attribute *attr,
1035                                         const char *buf, size_t n)
1036 {
1037         int len;
1038         const char *name_ptr;
1039         enum tegra_suspend_mode new_mode;
1040
1041         name_ptr = buf;
1042         while (*name_ptr && !isspace(*name_ptr))
1043                 name_ptr++;
1044         len = name_ptr - buf;
1045         if (!len)
1046                 goto bad_name;
1047         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1048         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1049                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1050                 pr_info("Illegal tegra suspend state: %s\n", buf);
1051                 goto bad_name;
1052         }
1053
1054         for (new_mode = TEGRA_SUSPEND_NONE; \
1055                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1056                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1057                         current_suspend_mode = new_mode;
1058                         break;
1059                 }
1060         }
1061
1062 bad_name:
1063         return n;
1064 }
1065
1066 static struct kobj_attribute suspend_mode_attribute =
1067         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1068
1069 static struct kobject *suspend_kobj;
1070
1071 static int tegra_pm_enter_suspend(void)
1072 {
1073         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1074         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1075                 tegra_lp0_cpu_mode(true);
1076         return 0;
1077 }
1078
1079 static void tegra_pm_enter_resume(void)
1080 {
1081         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1082                 tegra_lp0_cpu_mode(false);
1083         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1084 }
1085
1086 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1087         .suspend = tegra_pm_enter_suspend,
1088         .resume = tegra_pm_enter_resume,
1089 };
1090
1091 static __init int tegra_pm_enter_syscore_init(void)
1092 {
1093         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1094         return 0;
1095 }
1096 subsys_initcall(tegra_pm_enter_syscore_init);
1097 #endif
1098
1099 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1100 {
1101         u32 reg;
1102         u32 mode;
1103
1104         tegra_pclk = clk_get_sys(NULL, "pclk");
1105         BUG_ON(IS_ERR(tegra_pclk));
1106         pdata = plat;
1107         (void)reg;
1108         (void)mode;
1109
1110         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1111                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1112
1113 #ifndef CONFIG_PM_SLEEP
1114         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1115                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1116                            "disabling suspend\n", __func__);
1117                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1118         }
1119 #else
1120         if (create_suspend_pgtable() < 0) {
1121                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1122                                 __func__);
1123                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1124                 goto fail;
1125         }
1126
1127         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1128             (tegra_revision == TEGRA_REVISION_A01) &&
1129             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1130                 /* Tegra 3 A01 supports only LP1 */
1131                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1132                            "-- disabling LP0\n", __func__);
1133                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1134         }
1135         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1136                 tegra_lp0_vec_relocate) {
1137                 unsigned char *reloc_lp0;
1138                 unsigned long tmp;
1139                 void __iomem *orig;
1140                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1141                                         GFP_KERNEL);
1142                 WARN_ON(!reloc_lp0);
1143                 if (!reloc_lp0) {
1144                         pr_err("%s: Failed to allocate reloc_lp0\n",
1145                                 __func__);
1146                         goto out;
1147                 }
1148
1149                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1150                 WARN_ON(!orig);
1151                 if (!orig) {
1152                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1153                                 __func__, tegra_lp0_vec_start);
1154                         kfree(reloc_lp0);
1155                         goto out;
1156                 }
1157
1158                 tmp = (unsigned long) reloc_lp0;
1159                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1160                 reloc_lp0 = (unsigned char *)tmp;
1161                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1162                 iounmap(orig);
1163                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1164         }
1165
1166 out:
1167         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1168                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1169                            "provided by bootlader -- disabling LP0\n",
1170                            __func__);
1171                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1172         }
1173
1174         iram_save_size = tegra_iram_end() - tegra_iram_start();
1175
1176         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1177         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1178                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1179                        "-- LP0/LP1 unavailable\n", __func__);
1180                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1181         }
1182
1183 #ifdef CONFIG_TEGRA_LP1_950
1184         if (pdata->lp1_lowvolt_support) {
1185                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1186                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1187                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1188
1189                 lp1_core_lowvolt = 0;
1190                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1191                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1192
1193                 lp1_core_highvolt = 0;
1194                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1195                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1196         }
1197 #endif
1198         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1199         /* Initialize scratch registers used for CPU LP2 synchronization */
1200         writel(0, pmc + PMC_SCRATCH37);
1201         writel(0, pmc + PMC_SCRATCH38);
1202         writel(0, pmc + PMC_SCRATCH39);
1203         writel(0, pmc + PMC_SCRATCH41);
1204
1205         /* Always enable CPU power request; just normal polarity is supported */
1206         reg = readl(pmc + PMC_CTRL);
1207         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1208         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1209         pmc_32kwritel(reg, PMC_CTRL);
1210
1211         /* Configure core power request and system clock control if LP0
1212            is supported */
1213         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1214         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1215
1216         reg = readl(pmc + PMC_CTRL);
1217
1218         if (!pdata->sysclkreq_high)
1219                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1220         else
1221                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1222
1223         if (!pdata->corereq_high)
1224                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1225         else
1226                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1227
1228         /* configure output inverters while the request is tristated */
1229         pmc_32kwritel(reg, PMC_CTRL);
1230
1231         /* now enable requests */
1232         reg |= TEGRA_POWER_SYSCLK_OE;
1233         if (!pdata->combined_req)
1234                 reg |= TEGRA_POWER_PWRREQ_OE;
1235         pmc_32kwritel(reg, PMC_CTRL);
1236
1237         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1238                 tegra_lp0_suspend_init();
1239
1240         suspend_set_ops(&tegra_suspend_ops);
1241
1242         /* Create /sys/power/suspend/type */
1243         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1244         if (suspend_kobj) {
1245                 if (sysfs_create_file(suspend_kobj, \
1246                                                 &suspend_mode_attribute.attr))
1247                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1248                                                                 __func__);
1249         }
1250
1251         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1252         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1253
1254         /* clear io dpd settings before kernel */
1255         tegra_bl_io_dpd_cleanup();
1256
1257 fail:
1258 #endif
1259         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1260                 tegra_lp2_in_idle(false);
1261
1262         current_suspend_mode = plat->suspend_mode;
1263 }
1264
1265 unsigned long debug_uart_port_base = 0;
1266 EXPORT_SYMBOL(debug_uart_port_base);
1267
1268 static int tegra_debug_uart_suspend(void)
1269 {
1270         void __iomem *uart;
1271         u32 lcr;
1272
1273         if (!debug_uart_port_base)
1274                 return 0;
1275
1276         uart = IO_ADDRESS(debug_uart_port_base);
1277
1278         lcr = readb(uart + UART_LCR * 4);
1279
1280         tegra_sctx.uart[0] = lcr;
1281         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1282
1283         /* DLAB = 0 */
1284         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1285
1286         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1287
1288         /* DLAB = 1 */
1289         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1290
1291         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1292         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1293
1294         writeb(lcr, uart + UART_LCR * 4);
1295
1296         return 0;
1297 }
1298
1299 static void tegra_debug_uart_resume(void)
1300 {
1301         void __iomem *uart;
1302         u32 lcr;
1303
1304         if (!debug_uart_port_base)
1305                 return;
1306
1307         uart = IO_ADDRESS(debug_uart_port_base);
1308
1309         lcr = tegra_sctx.uart[0];
1310
1311         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1312
1313         /* DLAB = 0 */
1314         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1315
1316         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1317                         uart + UART_FCR * 4);
1318
1319         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1320
1321         /* DLAB = 1 */
1322         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1323
1324         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1325         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1326
1327         writeb(lcr, uart + UART_LCR * 4);
1328 }
1329
1330 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1331         .suspend = tegra_debug_uart_suspend,
1332         .resume = tegra_debug_uart_resume,
1333 };
1334
1335 struct clk *debug_uart_clk = NULL;
1336 EXPORT_SYMBOL(debug_uart_clk);
1337
1338 void tegra_console_uart_suspend(void)
1339 {
1340         if (console_suspend_enabled && debug_uart_clk)
1341                 clk_disable(debug_uart_clk);
1342 }
1343
1344 void tegra_console_uart_resume(void)
1345 {
1346         if (console_suspend_enabled && debug_uart_clk)
1347                 clk_enable(debug_uart_clk);
1348 }
1349
1350 static int tegra_debug_uart_syscore_init(void)
1351 {
1352         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1353         return 0;
1354 }
1355 arch_initcall(tegra_debug_uart_syscore_init);