ARM: tegra11: Save & restore timer registers
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58 #include <asm/smp_plat.h>
59
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62 #include <mach/hardware.h>
63
64 #include "board.h"
65 #include "clock.h"
66 #include "common.h"
67 #include "cpuidle.h"
68 #include "fuse.h"
69 #include "gic.h"
70 #include "iomap.h"
71 #include "pm.h"
72 #include "pm-irq.h"
73 #include "reset.h"
74 #include "sleep.h"
75 #include "timer.h"
76 #include "dvfs.h"
77 #include "cpu-tegra.h"
78
79 struct suspend_context {
80         /*
81          * The next 7 values are referenced by offset in __restart_plls
82          * in headsmp-t2.S, and should not be moved
83          */
84         u32 pllx_misc;
85         u32 pllx_base;
86         u32 pllp_misc;
87         u32 pllp_base;
88         u32 pllp_outa;
89         u32 pllp_outb;
90         u32 pll_timeout;
91
92         u32 cpu_burst;
93         u32 clk_csite_src;
94         u32 cclk_divider;
95
96         u32 mc[3];
97         u8 uart[5];
98
99         struct tegra_twd_context twd;
100 #ifdef CONFIG_ARM_ARCH_TIMER
101         struct arch_timer_context arch_timer;
102 #endif
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
107 static pgd_t *tegra_pgd;
108 static DEFINE_SPINLOCK(tegra_lp2_lock);
109 static cpumask_t tegra_in_lp2;
110 static cpumask_t *iram_cpu_lp2_mask;
111 static unsigned long *iram_cpu_lp1_mask;
112 static u8 *iram_save;
113 static unsigned long iram_save_size;
114 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
115 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
116 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
117 static int tegra_last_pclk;
118 #endif
119
120 struct suspend_context tegra_sctx;
121
122 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
123 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
124 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
125 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
126 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
127 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
128 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
129 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
130
131 #define PMC_CTRL                0x0
132 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
133 #define PMC_WAKE_MASK           0xc
134 #define PMC_WAKE_LEVEL          0x10
135 #define PMC_DPAD_ORIDE          0x1C
136 #define PMC_WAKE_DELAY          0xe0
137 #define PMC_DPD_SAMPLE          0x20
138 #define PMC_IO_DPD_REQ_0        0x1b8
139 #define PMC_IO_DPD2_REQ_0       0X1C0
140
141 #define PMC_WAKE_STATUS         0x14
142 #define PMC_SW_WAKE_STATUS      0x18
143 #define PMC_COREPWRGOOD_TIMER   0x3c
144 #define PMC_CPUPWRGOOD_TIMER    0xc8
145 #define PMC_CPUPWROFF_TIMER     0xcc
146 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
147
148 #define PMC_PWRGATE_TOGGLE      0x30
149 #define PWRGATE_TOGGLE_START    (1 << 8)
150 #define UN_PWRGATE_CPU          \
151         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
152
153 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
154 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
155 #endif
156
157 #define CLK_RESET_CCLK_BURST    0x20
158 #define CLK_RESET_CCLK_DIVIDER  0x24
159 #define CLK_RESET_PLLC_BASE     0x80
160 #define CLK_RESET_PLLM_BASE     0x90
161 #define CLK_RESET_PLLX_BASE     0xe0
162 #define CLK_RESET_PLLX_MISC     0xe4
163 #define CLK_RESET_PLLP_BASE     0xa0
164 #define CLK_RESET_PLLP_OUTA     0xa4
165 #define CLK_RESET_PLLP_OUTB     0xa8
166 #define CLK_RESET_PLLP_MISC     0xac
167
168 #define CLK_RESET_SOURCE_CSITE  0x1d4
169
170 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
171 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
172 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
173 #define CLK_RESET_CCLK_IDLE_POLICY         1
174 #define CLK_RESET_CCLK_RUN_POLICY          2
175 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
176 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
177
178 #define EMC_MRW_0               0x0e8
179 #define EMC_MRW_DEV_SELECTN     30
180 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
181
182 #define MC_SECURITY_START       0x6c
183 #define MC_SECURITY_SIZE        0x70
184 #define MC_SECURITY_CFG2        0x7c
185
186 struct dvfs_rail *tegra_cpu_rail;
187 static struct dvfs_rail *tegra_core_rail;
188 static struct clk *tegra_pclk;
189 static const struct tegra_suspend_platform_data *pdata;
190 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
191
192 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
193 enum tegra_cluster_switch_time_id {
194         tegra_cluster_switch_time_id_start = 0,
195         tegra_cluster_switch_time_id_prolog,
196         tegra_cluster_switch_time_id_switch,
197         tegra_cluster_switch_time_id_epilog,
198         tegra_cluster_switch_time_id_max
199 };
200
201 static unsigned long
202                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
203 #define tegra_cluster_switch_time(flags, id) \
204         do { \
205                 barrier(); \
206                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
207                         void __iomem *timer_us = \
208                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
209                         if (id < tegra_cluster_switch_time_id_max) \
210                                 tegra_cluster_switch_times[id] = \
211                                                         readl(timer_us); \
212                                 wmb(); \
213                 } \
214                 barrier(); \
215         } while(0)
216 #else
217 #define tegra_cluster_switch_time(flags, id) do {} while(0)
218 #endif
219
220 #ifdef CONFIG_PM_SLEEP
221 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
222         [TEGRA_SUSPEND_NONE]    = "none",
223         [TEGRA_SUSPEND_LP2]     = "lp2",
224         [TEGRA_SUSPEND_LP1]     = "lp1",
225         [TEGRA_SUSPEND_LP0]     = "lp0",
226 };
227
228 unsigned long tegra_cpu_power_good_time(void)
229 {
230         if (WARN_ON_ONCE(!pdata))
231                 return 5000;
232
233         return pdata->cpu_timer;
234 }
235
236 unsigned long tegra_cpu_power_off_time(void)
237 {
238         if (WARN_ON_ONCE(!pdata))
239                 return 5000;
240
241         return pdata->cpu_off_timer;
242 }
243
244 unsigned long tegra_cpu_lp2_min_residency(void)
245 {
246         if (WARN_ON_ONCE(!pdata))
247                 return 2000;
248
249         return pdata->cpu_lp2_min_residency;
250 }
251
252 /*
253  * create_suspend_pgtable
254  *
255  * Creates a page table with identity mappings of physical memory and IRAM
256  * for use when the MMU is off, in addition to all the regular kernel mappings.
257  */
258 static __init int create_suspend_pgtable(void)
259 {
260         tegra_pgd = pgd_alloc(&init_mm);
261         if (!tegra_pgd)
262                 return -ENOMEM;
263
264         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
265         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
266                 high_memory, 0);
267         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
268                 IO_IRAM_VIRT + SECTION_SIZE, 0);
269
270         /* inner/outer write-back/write-allocate, sharable */
271         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
272
273         return 0;
274 }
275
276 /* ensures that sufficient time is passed for a register write to
277  * serialize into the 32KHz domain */
278 static void pmc_32kwritel(u32 val, unsigned long offs)
279 {
280         writel(val, pmc + offs);
281         udelay(130);
282 }
283
284 static void set_power_timers(unsigned long us_on, unsigned long us_off,
285                              long rate)
286 {
287         static unsigned long last_us_off = 0;
288         unsigned long long ticks;
289         unsigned long long pclk;
290
291         if (WARN_ON_ONCE(rate <= 0))
292                 pclk = 100000000;
293         else
294                 pclk = rate;
295
296         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
297                 ticks = (us_on * pclk) + 999999ull;
298                 do_div(ticks, 1000000);
299                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
300
301                 ticks = (us_off * pclk) + 999999ull;
302                 do_div(ticks, 1000000);
303                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
304                 wmb();
305         }
306         tegra_last_pclk = pclk;
307         last_us_off = us_off;
308 }
309
310 /*
311  * restore_cpu_complex
312  *
313  * restores cpu clock setting, clears flow controller
314  *
315  * Always called on CPU 0.
316  */
317 static void restore_cpu_complex(u32 mode)
318 {
319         int cpu = smp_processor_id();
320         unsigned int reg, policy;
321
322         BUG_ON(cpu != 0);
323
324 #ifdef CONFIG_SMP
325         cpu = cpu_logical_map(cpu);
326 #endif
327         /* Is CPU complex already running on PLLX? */
328         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
329         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
330
331         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
332                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
333         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
334                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
335         else
336                 BUG();
337
338         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
339                 /* restore PLLX settings if CPU is on different PLL */
340                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
341                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
342
343                 /* wait for PLL stabilization if PLLX was enabled */
344                 if (tegra_sctx.pllx_base & (1<<30)) {
345 #if USE_PLL_LOCK_BITS
346                         /* Enable lock detector */
347                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
348                         reg |= 1<<18;
349                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
350                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
351                                  (1<<27)))
352                                 cpu_relax();
353
354                         udelay(PLL_POST_LOCK_DELAY);
355 #else
356                         udelay(300);
357 #endif
358                 }
359         }
360
361         /* Restore original burst policy setting for calls resulting from CPU
362            LP2 in idle or system suspend; keep cluster switch prolog setting
363            intact. */
364         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
365                 writel(tegra_sctx.cclk_divider, clk_rst +
366                        CLK_RESET_CCLK_DIVIDER);
367                 writel(tegra_sctx.cpu_burst, clk_rst +
368                        CLK_RESET_CCLK_BURST);
369         }
370
371         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
372
373         /* Do not power-gate CPU 0 when flow controlled */
374         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
375         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
376         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
377         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
378         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
379         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
380         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
381
382         /* If an immedidate cluster switch is being perfomed, restore the
383            local timer registers. For calls resulting from CPU LP2 in
384            idle or system suspend, the local timer was shut down and
385            timekeeping switched over to the global system timer. In this
386            case keep local timer disabled, and restore only periodic load. */
387         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
388                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
389 #ifdef CONFIG_ARM_ARCH_TIMER
390                 tegra_sctx.arch_timer.cntp_ctl = 0;
391 #endif
392 #ifdef CONFIG_HAVE_ARM_TWD
393                 tegra_sctx.twd.twd_ctrl = 0;
394 #endif
395         }
396 #ifdef CONFIG_ARM_ARCH_TIMER
397         arch_timer_resume(&tegra_sctx.arch_timer);
398 #endif
399 #ifdef CONFIG_HAVE_ARM_TWD
400         tegra_twd_resume(&tegra_sctx.twd);
401 #endif
402 }
403
404 /*
405  * suspend_cpu_complex
406  *
407  * saves pll state for use by restart_plls, prepares flow controller for
408  * transition to suspend state
409  *
410  * Must always be called on cpu 0.
411  */
412 static void suspend_cpu_complex(u32 mode)
413 {
414         int cpu = smp_processor_id();
415         unsigned int reg;
416         int i;
417
418         BUG_ON(cpu != 0);
419
420 #ifdef CONFIG_SMP
421         cpu = cpu_logical_map(cpu);
422 #endif
423         /* switch coresite to clk_m, save off original source */
424         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
425         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
426
427         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
428         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
429         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
430         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
431         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
432         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
433         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
434         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
435
436 #ifdef CONFIG_HAVE_ARM_TWD
437         tegra_twd_suspend(&tegra_sctx.twd);
438 #endif
439 #ifdef CONFIG_ARM_ARCH_TIMER
440         arch_timer_suspend(&tegra_sctx.arch_timer);
441 #endif
442
443         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
444         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
445         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
446         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
447         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
448 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
449         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
450 #else
451         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
452 #endif
453         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
454         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
455
456         for (i = 0; i < num_possible_cpus(); i++) {
457                 if (i == cpu)
458                         continue;
459                 reg = readl(FLOW_CTRL_CPU_CSR(i));
460                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
461                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
462                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
463         }
464
465         tegra_gic_cpu_disable(true);
466 }
467
468 void tegra_clear_cpu_in_lp2(int cpu)
469 {
470         spin_lock(&tegra_lp2_lock);
471         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
472         cpumask_clear_cpu(cpu, &tegra_in_lp2);
473
474         /* Update the IRAM copy used by the reset handler. The IRAM copy
475            can't use used directly by cpumask_clear_cpu() because it uses
476            LDREX/STREX which requires the addressed location to be inner
477            cacheable and sharable which IRAM isn't. */
478         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
479         dsb();
480
481         spin_unlock(&tegra_lp2_lock);
482 }
483
484 bool tegra_set_cpu_in_lp2(int cpu)
485 {
486         bool last_cpu = false;
487
488         spin_lock(&tegra_lp2_lock);
489         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
490         cpumask_set_cpu(cpu, &tegra_in_lp2);
491
492         /* Update the IRAM copy used by the reset handler. The IRAM copy
493            can't use used directly by cpumask_set_cpu() because it uses
494            LDREX/STREX which requires the addressed location to be inner
495            cacheable and sharable which IRAM isn't. */
496         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
497         dsb();
498
499         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
500                 last_cpu = true;
501 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
502         else if (cpu == 1)
503                 tegra2_cpu_set_resettable_soon();
504 #endif
505
506         spin_unlock(&tegra_lp2_lock);
507         return last_cpu;
508 }
509
510 static void tegra_sleep_core(enum tegra_suspend_mode mode,
511                              unsigned long v2p)
512 {
513 #ifdef CONFIG_TRUSTED_FOUNDATIONS
514         if (mode == TEGRA_SUSPEND_LP0) {
515                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
516                                   virt_to_phys(tegra_resume));
517         } else {
518                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
519                                   (TEGRA_RESET_HANDLER_BASE +
520                                    tegra_cpu_reset_handler_offset));
521         }
522 #endif
523 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
524         cpu_suspend(v2p, tegra2_sleep_core_finish);
525 #else
526         cpu_suspend(v2p, tegra3_sleep_core_finish);
527 #endif
528 }
529
530 static inline void tegra_sleep_cpu(unsigned long v2p)
531 {
532 #ifdef CONFIG_TRUSTED_FOUNDATIONS
533         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
534                           (TEGRA_RESET_HANDLER_BASE +
535                            tegra_cpu_reset_handler_offset));
536 #endif
537         cpu_suspend(v2p, tegra_sleep_cpu_finish);
538 }
539
540 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
541 {
542         u32 reg;
543         unsigned int remain;
544         pgd_t *pgd;
545
546         /* Only the last cpu down does the final suspend steps */
547         reg = readl(pmc + PMC_CTRL);
548         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
549         if (pdata->combined_req)
550                 reg &= ~TEGRA_POWER_PWRREQ_OE;
551         else
552                 reg |= TEGRA_POWER_PWRREQ_OE;
553
554         reg &= ~TEGRA_POWER_EFFECT_LP0;
555         writel(reg, pmc + PMC_CTRL);
556
557         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
558
559         /*
560          * We can use clk_get_rate_all_locked() here, because all other cpus
561          * are in LP2 state and irqs are disabled
562          */
563         if (flags & TEGRA_POWER_CLUSTER_MASK) {
564                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
565                 set_power_timers(pdata->cpu_timer, 0,
566                         clk_get_rate_all_locked(tegra_pclk));
567                 if (flags & TEGRA_POWER_CLUSTER_G) {
568                         /*
569                          * To reduce the vdd_cpu up latency when LP->G
570                          * transition. Before the transition, enable
571                          * the vdd_cpu rail.
572                          */
573                         if (is_lp_cluster())
574                                 writel(UN_PWRGATE_CPU,
575                                        pmc + PMC_PWRGATE_TOGGLE);
576                 }
577                 tegra_cluster_switch_prolog(flags);
578         } else {
579                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
580                         clk_get_rate_all_locked(tegra_pclk));
581 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
582                 reg = readl(FLOW_CTRL_CPU_CSR(0));
583                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
584                 if (is_lp_cluster()) {
585                         /* for LP cluster, there is no option for rail gating */
586                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
587                                                 TEGRA_POWER_CLUSTER_PART_MASK)
588                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
589                         else if (flags)
590                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
591                 }
592                 else {
593                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
594                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
595                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
596                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
597                 }
598                 writel(reg, FLOW_CTRL_CPU_CSR(0));
599 #endif
600         }
601
602         if (sleep_time)
603                 tegra_lp2_set_trigger(sleep_time);
604
605         cpu_cluster_pm_enter();
606         suspend_cpu_complex(flags);
607         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
608         flush_cache_all();
609         /*
610          * No need to flush complete L2. Cleaning kernel and IO mappings
611          * is enough for the LP code sequence that has L2 disabled but
612          * MMU on.
613          */
614         pgd = cpu_get_pgd();
615         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
616                           __pa(pgd + PTRS_PER_PGD));
617         outer_disable();
618
619         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
620
621         tegra_init_cache(false);
622         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
623         restore_cpu_complex(flags);
624         cpu_cluster_pm_exit();
625
626         remain = tegra_lp2_timer_remain();
627         if (sleep_time)
628                 tegra_lp2_set_trigger(0);
629
630         if (flags & TEGRA_POWER_CLUSTER_MASK) {
631                 tegra_cluster_switch_epilog(flags);
632                 trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
633         }
634         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
635
636 #if INSTRUMENT_CLUSTER_SWITCH
637         if (flags & TEGRA_POWER_CLUSTER_MASK) {
638                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
639                         is_lp_cluster() ? "G=>LP" : "LP=>G",
640                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
641                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
642                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
643                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
644                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
645                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
646                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
647                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
648         }
649 #endif
650         return remain;
651 }
652
653 static int tegra_common_suspend(void)
654 {
655         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
656
657         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
658         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
659         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
660
661         /* copy the reset vector and SDRAM shutdown code into IRAM */
662         memcpy(iram_save, iram_code, iram_save_size);
663         memcpy(iram_code, tegra_iram_start(), iram_save_size);
664
665         return 0;
666 }
667
668 static void tegra_common_resume(void)
669 {
670         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
671 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
672         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
673 #endif
674
675         /* Clear DPD sample */
676         writel(0x0, pmc + PMC_DPD_SAMPLE);
677
678         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
679         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
680         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
681 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
682         /* trigger emc mode write */
683         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
684 #endif
685         /* clear scratch registers shared by suspend and the reset pen */
686         writel(0x0, pmc + PMC_SCRATCH39);
687         writel(0x0, pmc + PMC_SCRATCH41);
688
689         /* restore IRAM */
690         memcpy(iram_code, iram_save, iram_save_size);
691 }
692
693 static int tegra_suspend_prepare_late(void)
694 {
695 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
696         disable_irq(INT_SYS_STATS_MON);
697 #endif
698         return 0;
699 }
700
701 static void tegra_suspend_wake(void)
702 {
703 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
704         enable_irq(INT_SYS_STATS_MON);
705 #endif
706 }
707
708 static void tegra_pm_set(enum tegra_suspend_mode mode)
709 {
710         u32 reg, boot_flag;
711         unsigned long rate = 32768;
712
713         reg = readl(pmc + PMC_CTRL);
714         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
715         if (pdata->combined_req)
716                 reg &= ~TEGRA_POWER_PWRREQ_OE;
717         else
718                 reg |= TEGRA_POWER_PWRREQ_OE;
719         reg &= ~TEGRA_POWER_EFFECT_LP0;
720
721         switch (mode) {
722         case TEGRA_SUSPEND_LP0:
723 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
724                 rate = clk_get_rate_all_locked(tegra_pclk);
725 #endif
726                 if (pdata->combined_req) {
727                         reg |= TEGRA_POWER_PWRREQ_OE;
728                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
729                 }
730
731                 /*
732                  * LP0 boots through the AVP, which then resumes the AVP to
733                  * the address in scratch 39, and the cpu to the address in
734                  * scratch 41 to tegra_resume
735                  */
736                 writel(0x0, pmc + PMC_SCRATCH39);
737
738                 /* Enable DPD sample to trigger sampling pads data and direction
739                  * in which pad will be driven during lp0 mode*/
740                 writel(0x1, pmc + PMC_DPD_SAMPLE);
741 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
742                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ_0);
743                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ_0);
744 #endif
745                 /* Set warmboot flag */
746                 boot_flag = readl(pmc + PMC_SCRATCH0);
747                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
748
749                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
750
751                 reg |= TEGRA_POWER_EFFECT_LP0;
752                 /* No break here. LP0 code falls through to write SCRATCH41 */
753         case TEGRA_SUSPEND_LP1:
754                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
755                 wmb();
756                 break;
757         case TEGRA_SUSPEND_LP2:
758                 rate = clk_get_rate(tegra_pclk);
759                 break;
760         case TEGRA_SUSPEND_NONE:
761                 return;
762         default:
763                 BUG();
764         }
765
766         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
767
768         pmc_32kwritel(reg, PMC_CTRL);
769 }
770
771 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
772         [TEGRA_SUSPEND_NONE] = "none",
773         [TEGRA_SUSPEND_LP2] = "LP2",
774         [TEGRA_SUSPEND_LP1] = "LP1",
775         [TEGRA_SUSPEND_LP0] = "LP0",
776 };
777
778 static int tegra_suspend_enter(suspend_state_t state)
779 {
780         int ret;
781         ktime_t delta;
782         struct timespec ts_entry, ts_exit;
783
784         if (pdata && pdata->board_suspend)
785                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
786
787         read_persistent_clock(&ts_entry);
788
789         ret = tegra_suspend_dram(current_suspend_mode, 0);
790         if (ret) {
791                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
792                 goto abort_suspend;
793         }
794
795         read_persistent_clock(&ts_exit);
796
797         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
798                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
799
800                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
801                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
802                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
803                 else
804                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
805         }
806
807 abort_suspend:
808         if (pdata && pdata->board_resume)
809                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
810
811         return ret;
812 }
813
814 static void tegra_suspend_check_pwr_stats(void)
815 {
816         /* cpus and l2 are powered off later */
817         unsigned long pwrgate_partid_mask =
818 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
819                 (1 << TEGRA_POWERGATE_HEG)      |
820                 (1 << TEGRA_POWERGATE_SATA)     |
821                 (1 << TEGRA_POWERGATE_3D1)      |
822 #endif
823                 (1 << TEGRA_POWERGATE_3D)       |
824                 (1 << TEGRA_POWERGATE_VENC)     |
825                 (1 << TEGRA_POWERGATE_PCIE)     |
826                 (1 << TEGRA_POWERGATE_VDEC)     |
827                 (1 << TEGRA_POWERGATE_MPE);
828
829         int partid;
830
831         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
832                 if ((1 << partid) & pwrgate_partid_mask)
833                         if (tegra_powergate_is_powered(partid))
834                                 pr_warning("partition %s is left on before suspend\n",
835                                         tegra_powergate_get_name(partid));
836
837         return;
838 }
839
840 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
841 {
842         int err = 0;
843         u32 scratch37 = 0xDEADBEEF;
844
845         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
846                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
847                 err = -ENXIO;
848                 goto fail;
849         }
850
851         if (tegra_is_voice_call_active()) {
852                 u32 reg;
853
854                 /* backup the current value of scratch37 */
855                 scratch37 = readl(pmc + PMC_SCRATCH37);
856
857                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
858                 reg = TEGRA_POWER_LP1_AUDIO;
859                 pmc_32kwritel(reg, PMC_SCRATCH37);
860         }
861
862         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
863                 pr_info("LP0 not used due to unsupported wakeup events\n");
864                 mode = TEGRA_SUSPEND_LP1;
865         }
866
867         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
868                 tegra_suspend_check_pwr_stats();
869
870         tegra_common_suspend();
871
872         tegra_pm_set(mode);
873
874         if (pdata && pdata->board_suspend)
875                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
876
877         local_fiq_disable();
878
879         trace_cpu_suspend(CPU_SUSPEND_START);
880
881         if (mode == TEGRA_SUSPEND_LP0) {
882 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
883                 u32 reg = readl(pmc + PMC_SCRATCH4);
884                 if (is_lp_cluster())
885                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
886                 else
887                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
888                 pmc_32kwritel(reg, PMC_SCRATCH4);
889 #endif
890                 tegra_tsc_suspend();
891                 tegra_lp0_suspend_mc();
892                 tegra_cpu_reset_handler_save();
893                 tegra_tsc_wait_for_suspend();
894         }
895         else if (mode == TEGRA_SUSPEND_LP1)
896                 *iram_cpu_lp1_mask = 1;
897
898         suspend_cpu_complex(flags);
899
900         flush_cache_all();
901         outer_flush_all();
902         outer_disable();
903
904         if (mode == TEGRA_SUSPEND_LP2)
905                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
906         else
907                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
908
909         tegra_init_cache(true);
910
911         if (mode == TEGRA_SUSPEND_LP0) {
912                 tegra_tsc_resume();
913                 tegra_cpu_reset_handler_restore();
914                 tegra_lp0_resume_mc();
915                 tegra_tsc_wait_for_resume();
916         } else if (mode == TEGRA_SUSPEND_LP1)
917                 *iram_cpu_lp1_mask = 0;
918
919         /* if scratch37 was clobbered during LP1, restore it */
920         if (scratch37 != 0xDEADBEEF)
921                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
922
923         restore_cpu_complex(flags);
924
925         /* for platforms where the core & CPU power requests are
926          * combined as a single request to the PMU, transition out
927          * of LP0 state by temporarily enabling both requests
928          */
929         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
930                 u32 reg;
931                 reg = readl(pmc + PMC_CTRL);
932                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
933                 pmc_32kwritel(reg, PMC_CTRL);
934                 reg &= ~TEGRA_POWER_PWRREQ_OE;
935                 pmc_32kwritel(reg, PMC_CTRL);
936         }
937
938         if (pdata && pdata->board_resume)
939                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
940
941         trace_cpu_suspend(CPU_SUSPEND_DONE);
942
943         local_fiq_enable();
944
945         tegra_common_resume();
946
947 fail:
948         return err;
949 }
950
951 /*
952  * Function pointers to optional board specific function
953  */
954 void (*tegra_deep_sleep)(int);
955 EXPORT_SYMBOL(tegra_deep_sleep);
956
957 static int tegra_suspend_prepare(void)
958 {
959         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
960                 tegra_deep_sleep(1);
961         return 0;
962 }
963
964 static void tegra_suspend_finish(void)
965 {
966         if (pdata && pdata->cpu_resume_boost) {
967                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
968                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
969                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
970         }
971
972         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
973                 tegra_deep_sleep(0);
974 }
975
976 static const struct platform_suspend_ops tegra_suspend_ops = {
977         .valid          = suspend_valid_only_mem,
978         .prepare        = tegra_suspend_prepare,
979         .finish         = tegra_suspend_finish,
980         .prepare_late   = tegra_suspend_prepare_late,
981         .wake           = tegra_suspend_wake,
982         .enter          = tegra_suspend_enter,
983 };
984
985 static ssize_t suspend_mode_show(struct kobject *kobj,
986                                         struct kobj_attribute *attr, char *buf)
987 {
988         char *start = buf;
989         char *end = buf + PAGE_SIZE;
990
991         start += scnprintf(start, end - start, "%s ", \
992                                 tegra_suspend_name[current_suspend_mode]);
993         start += scnprintf(start, end - start, "\n");
994
995         return start - buf;
996 }
997
998 static ssize_t suspend_mode_store(struct kobject *kobj,
999                                         struct kobj_attribute *attr,
1000                                         const char *buf, size_t n)
1001 {
1002         int len;
1003         const char *name_ptr;
1004         enum tegra_suspend_mode new_mode;
1005
1006         name_ptr = buf;
1007         while (*name_ptr && !isspace(*name_ptr))
1008                 name_ptr++;
1009         len = name_ptr - buf;
1010         if (!len)
1011                 goto bad_name;
1012         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1013         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1014                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1015                 pr_info("Illegal tegra suspend state: %s\n", buf);
1016                 goto bad_name;
1017         }
1018
1019         for (new_mode = TEGRA_SUSPEND_NONE; \
1020                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1021                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1022                         current_suspend_mode = new_mode;
1023                         break;
1024                 }
1025         }
1026
1027 bad_name:
1028         return n;
1029 }
1030
1031 static struct kobj_attribute suspend_mode_attribute =
1032         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1033
1034 static struct kobject *suspend_kobj;
1035
1036 static int tegra_pm_enter_suspend(void)
1037 {
1038         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1039         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1040                 tegra_lp0_cpu_mode(true);
1041         return 0;
1042 }
1043
1044 static void tegra_pm_enter_resume(void)
1045 {
1046         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1047                 tegra_lp0_cpu_mode(false);
1048         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1049 }
1050
1051 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1052         .suspend = tegra_pm_enter_suspend,
1053         .resume = tegra_pm_enter_resume,
1054 };
1055
1056 static __init int tegra_pm_enter_syscore_init(void)
1057 {
1058         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1059         return 0;
1060 }
1061 subsys_initcall(tegra_pm_enter_syscore_init);
1062 #endif
1063
1064 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1065 {
1066         u32 reg;
1067         u32 mode;
1068
1069         tegra_cpu_rail = tegra_dvfs_get_rail_by_name("vdd_cpu");
1070         tegra_core_rail = tegra_dvfs_get_rail_by_name("vdd_core");
1071         tegra_pclk = clk_get_sys(NULL, "pclk");
1072         BUG_ON(IS_ERR(tegra_pclk));
1073         pdata = plat;
1074         (void)reg;
1075         (void)mode;
1076
1077         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1078                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1079
1080 #ifndef CONFIG_PM_SLEEP
1081         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1082                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1083                            "disabling suspend\n", __func__);
1084                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1085         }
1086 #else
1087         if (create_suspend_pgtable() < 0) {
1088                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1089                                 __func__);
1090                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1091                 goto fail;
1092         }
1093
1094         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1095             (tegra_revision == TEGRA_REVISION_A01) &&
1096             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1097                 /* Tegra 3 A01 supports only LP1 */
1098                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1099                            "-- disabling LP0\n", __func__);
1100                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1101         }
1102         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1103                 tegra_lp0_vec_relocate) {
1104                 unsigned char *reloc_lp0;
1105                 unsigned long tmp;
1106                 void __iomem *orig;
1107                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1108                                         GFP_KERNEL);
1109                 WARN_ON(!reloc_lp0);
1110                 if (!reloc_lp0) {
1111                         pr_err("%s: Failed to allocate reloc_lp0\n",
1112                                 __func__);
1113                         goto out;
1114                 }
1115
1116                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1117                 WARN_ON(!orig);
1118                 if (!orig) {
1119                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1120                                 __func__, tegra_lp0_vec_start);
1121                         kfree(reloc_lp0);
1122                         goto out;
1123                 }
1124
1125                 tmp = (unsigned long) reloc_lp0;
1126                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1127                 reloc_lp0 = (unsigned char *)tmp;
1128                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1129                 iounmap(orig);
1130                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1131         }
1132
1133 out:
1134         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1135                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1136                            "provided by bootlader -- disabling LP0\n",
1137                            __func__);
1138                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1139         }
1140
1141         iram_save_size = tegra_iram_end() - tegra_iram_start();
1142
1143         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1144         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1145                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1146                        "-- LP0/LP1 unavailable\n", __func__);
1147                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1148         }
1149
1150         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1151         /* Initialize scratch registers used for CPU LP2 synchronization */
1152         writel(0, pmc + PMC_SCRATCH37);
1153         writel(0, pmc + PMC_SCRATCH38);
1154         writel(0, pmc + PMC_SCRATCH39);
1155         writel(0, pmc + PMC_SCRATCH41);
1156
1157         /* Always enable CPU power request; just normal polarity is supported */
1158         reg = readl(pmc + PMC_CTRL);
1159         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1160         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1161         pmc_32kwritel(reg, PMC_CTRL);
1162
1163         /* Configure core power request and system clock control if LP0
1164            is supported */
1165         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1166         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1167
1168         reg = readl(pmc + PMC_CTRL);
1169
1170         if (!pdata->sysclkreq_high)
1171                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1172         else
1173                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1174
1175         if (!pdata->corereq_high)
1176                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1177         else
1178                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1179
1180         /* configure output inverters while the request is tristated */
1181         pmc_32kwritel(reg, PMC_CTRL);
1182
1183         /* now enable requests */
1184         reg |= TEGRA_POWER_SYSCLK_OE;
1185         if (!pdata->combined_req)
1186                 reg |= TEGRA_POWER_PWRREQ_OE;
1187         pmc_32kwritel(reg, PMC_CTRL);
1188
1189         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1190                 tegra_lp0_suspend_init();
1191
1192         suspend_set_ops(&tegra_suspend_ops);
1193
1194         /* Create /sys/power/suspend/type */
1195         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1196         if (suspend_kobj) {
1197                 if (sysfs_create_file(suspend_kobj, \
1198                                                 &suspend_mode_attribute.attr))
1199                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1200                                                                 __func__);
1201         }
1202
1203         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1204         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1205 fail:
1206 #endif
1207         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1208                 tegra_lp2_in_idle(false);
1209
1210         current_suspend_mode = plat->suspend_mode;
1211 }
1212
1213 unsigned long debug_uart_port_base = 0;
1214 EXPORT_SYMBOL(debug_uart_port_base);
1215
1216 static int tegra_debug_uart_suspend(void)
1217 {
1218         void __iomem *uart;
1219         u32 lcr;
1220
1221         if (!debug_uart_port_base)
1222                 return 0;
1223
1224         uart = IO_ADDRESS(debug_uart_port_base);
1225
1226         lcr = readb(uart + UART_LCR * 4);
1227
1228         tegra_sctx.uart[0] = lcr;
1229         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1230
1231         /* DLAB = 0 */
1232         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1233
1234         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1235
1236         /* DLAB = 1 */
1237         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1238
1239         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1240         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1241
1242         writeb(lcr, uart + UART_LCR * 4);
1243
1244         return 0;
1245 }
1246
1247 static void tegra_debug_uart_resume(void)
1248 {
1249         void __iomem *uart;
1250         u32 lcr;
1251
1252         if (!debug_uart_port_base)
1253                 return;
1254
1255         uart = IO_ADDRESS(debug_uart_port_base);
1256
1257         lcr = tegra_sctx.uart[0];
1258
1259         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1260
1261         /* DLAB = 0 */
1262         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1263
1264         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1265                         uart + UART_FCR * 4);
1266
1267         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1268
1269         /* DLAB = 1 */
1270         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1271
1272         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1273         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1274
1275         writeb(lcr, uart + UART_LCR * 4);
1276 }
1277
1278 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1279         .suspend = tegra_debug_uart_suspend,
1280         .resume = tegra_debug_uart_resume,
1281 };
1282
1283 struct clk *debug_uart_clk = NULL;
1284 EXPORT_SYMBOL(debug_uart_clk);
1285
1286 void tegra_console_uart_suspend(void)
1287 {
1288         if (console_suspend_enabled && debug_uart_clk)
1289                 clk_disable(debug_uart_clk);
1290 }
1291
1292 void tegra_console_uart_resume(void)
1293 {
1294         if (console_suspend_enabled && debug_uart_clk)
1295                 clk_enable(debug_uart_clk);
1296 }
1297
1298 static int tegra_debug_uart_syscore_init(void)
1299 {
1300         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1301         return 0;
1302 }
1303 arch_initcall(tegra_debug_uart_syscore_init);