ARM: tegra: pm: adding timestamp to suspend trace
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58 #include <asm/smp_plat.h>
59
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62 #include <mach/hardware.h>
63
64 #include "board.h"
65 #include "clock.h"
66 #include "common.h"
67 #include "cpuidle.h"
68 #include "fuse.h"
69 #include "gic.h"
70 #include "iomap.h"
71 #include "pm.h"
72 #include "pm-irq.h"
73 #include "reset.h"
74 #include "sleep.h"
75 #include "timer.h"
76 #include "dvfs.h"
77 #include "cpu-tegra.h"
78
79 struct suspend_context {
80         /*
81          * The next 7 values are referenced by offset in __restart_plls
82          * in headsmp-t2.S, and should not be moved
83          */
84         u32 pllx_misc;
85         u32 pllx_base;
86         u32 pllp_misc;
87         u32 pllp_base;
88         u32 pllp_outa;
89         u32 pllp_outb;
90         u32 pll_timeout;
91
92         u32 cpu_burst;
93         u32 clk_csite_src;
94         u32 cclk_divider;
95
96         u32 mc[3];
97         u8 uart[5];
98
99         struct tegra_twd_context twd;
100 #ifdef CONFIG_ARM_ARCH_TIMER
101         struct arch_timer_context arch_timer;
102 #endif
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
107 static pgd_t *tegra_pgd;
108 static DEFINE_SPINLOCK(tegra_lp2_lock);
109 static cpumask_t tegra_in_lp2;
110 static cpumask_t *iram_cpu_lp2_mask;
111 static unsigned long *iram_cpu_lp1_mask;
112 static u8 *iram_save;
113 static unsigned long iram_save_size;
114 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
115 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
116 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
117 static int tegra_last_pclk;
118 #endif
119
120 struct suspend_context tegra_sctx;
121
122 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
123 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
124 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
125 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
126 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
127 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
128 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
129 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
130 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
131
132 #define PMC_CTRL                0x0
133 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
134 #define PMC_WAKE_MASK           0xc
135 #define PMC_WAKE_LEVEL          0x10
136 #define PMC_DPAD_ORIDE          0x1C
137 #define PMC_WAKE_DELAY          0xe0
138 #define PMC_DPD_SAMPLE          0x20
139
140 #define PMC_WAKE_STATUS         0x14
141 #define PMC_SW_WAKE_STATUS      0x18
142 #define PMC_COREPWRGOOD_TIMER   0x3c
143 #define PMC_CPUPWRGOOD_TIMER    0xc8
144 #define PMC_CPUPWROFF_TIMER     0xcc
145 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
146
147 #define PMC_PWRGATE_TOGGLE      0x30
148 #define PWRGATE_TOGGLE_START    (1 << 8)
149 #define UN_PWRGATE_CPU          \
150         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
151
152 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
153 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
154 #endif
155
156 #define CLK_RESET_CCLK_BURST    0x20
157 #define CLK_RESET_CCLK_DIVIDER  0x24
158 #define CLK_RESET_PLLC_BASE     0x80
159 #define CLK_RESET_PLLM_BASE     0x90
160 #define CLK_RESET_PLLX_BASE     0xe0
161 #define CLK_RESET_PLLX_MISC     0xe4
162 #define CLK_RESET_PLLP_BASE     0xa0
163 #define CLK_RESET_PLLP_OUTA     0xa4
164 #define CLK_RESET_PLLP_OUTB     0xa8
165 #define CLK_RESET_PLLP_MISC     0xac
166
167 #define CLK_RESET_SOURCE_CSITE  0x1d4
168
169 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
170 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
171 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
172 #define CLK_RESET_CCLK_IDLE_POLICY         1
173 #define CLK_RESET_CCLK_RUN_POLICY          2
174 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
175 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
176
177 #define EMC_MRW_0               0x0e8
178 #define EMC_MRW_DEV_SELECTN     30
179 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
180
181 #define MC_SECURITY_START       0x6c
182 #define MC_SECURITY_SIZE        0x70
183 #define MC_SECURITY_CFG2        0x7c
184
185 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
186 static struct clk *tegra_dfll;
187 #endif
188 static struct clk *tegra_pclk;
189 static const struct tegra_suspend_platform_data *pdata;
190 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
191
192 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
193 enum tegra_cluster_switch_time_id {
194         tegra_cluster_switch_time_id_start = 0,
195         tegra_cluster_switch_time_id_prolog,
196         tegra_cluster_switch_time_id_switch,
197         tegra_cluster_switch_time_id_epilog,
198         tegra_cluster_switch_time_id_max
199 };
200
201 static unsigned long
202                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
203 #define tegra_cluster_switch_time(flags, id) \
204         do { \
205                 barrier(); \
206                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
207                         void __iomem *timer_us = \
208                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
209                         if (id < tegra_cluster_switch_time_id_max) \
210                                 tegra_cluster_switch_times[id] = \
211                                                         readl(timer_us); \
212                                 wmb(); \
213                 } \
214                 barrier(); \
215         } while(0)
216 #else
217 #define tegra_cluster_switch_time(flags, id) do {} while(0)
218 #endif
219
220 #ifdef CONFIG_PM_SLEEP
221 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
222         [TEGRA_SUSPEND_NONE]    = "none",
223         [TEGRA_SUSPEND_LP2]     = "lp2",
224         [TEGRA_SUSPEND_LP1]     = "lp1",
225         [TEGRA_SUSPEND_LP0]     = "lp0",
226 };
227
228 unsigned long tegra_cpu_power_good_time(void)
229 {
230         if (WARN_ON_ONCE(!pdata))
231                 return 5000;
232
233         return pdata->cpu_timer;
234 }
235
236 unsigned long tegra_cpu_power_off_time(void)
237 {
238         if (WARN_ON_ONCE(!pdata))
239                 return 5000;
240
241         return pdata->cpu_off_timer;
242 }
243
244 unsigned long tegra_cpu_lp2_min_residency(void)
245 {
246         if (WARN_ON_ONCE(!pdata))
247                 return 2000;
248
249         return pdata->cpu_lp2_min_residency;
250 }
251
252 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
253 unsigned long tegra_min_residency_noncpu(void)
254 {
255         return pdata->min_residency_noncpu;
256 }
257
258 unsigned long tegra_min_residency_crail(void)
259 {
260         return pdata->min_residency_crail;
261 }
262 #endif
263
264 static void suspend_cpu_dfll_mode(void)
265 {
266 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
267         /* If DFLL is used as CPU clock source go to open loop mode */
268         if (!is_lp_cluster() && tegra_dfll &&
269             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
270                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
271 #endif
272 }
273
274 static void resume_cpu_dfll_mode(void)
275 {
276 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
277         /* If DFLL is used as CPU clock source restore closed loop mode */
278         if (!is_lp_cluster() && tegra_dfll &&
279             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
280                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
281 #endif
282 }
283
284 /*
285  * create_suspend_pgtable
286  *
287  * Creates a page table with identity mappings of physical memory and IRAM
288  * for use when the MMU is off, in addition to all the regular kernel mappings.
289  */
290 static __init int create_suspend_pgtable(void)
291 {
292         tegra_pgd = pgd_alloc(&init_mm);
293         if (!tegra_pgd)
294                 return -ENOMEM;
295
296         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
297         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
298                 high_memory, 0);
299         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
300                 IO_IRAM_VIRT + SECTION_SIZE, 0);
301
302         /* inner/outer write-back/write-allocate, sharable */
303         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
304
305         return 0;
306 }
307
308 /* ensures that sufficient time is passed for a register write to
309  * serialize into the 32KHz domain */
310 static void pmc_32kwritel(u32 val, unsigned long offs)
311 {
312         writel(val, pmc + offs);
313         udelay(130);
314 }
315
316 static void set_power_timers(unsigned long us_on, unsigned long us_off,
317                              long rate)
318 {
319         static unsigned long last_us_off = 0;
320         unsigned long long ticks;
321         unsigned long long pclk;
322
323         if (WARN_ON_ONCE(rate <= 0))
324                 pclk = 100000000;
325         else
326                 pclk = rate;
327
328         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
329                 ticks = (us_on * pclk) + 999999ull;
330                 do_div(ticks, 1000000);
331                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
332
333                 ticks = (us_off * pclk) + 999999ull;
334                 do_div(ticks, 1000000);
335                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
336                 wmb();
337         }
338         tegra_last_pclk = pclk;
339         last_us_off = us_off;
340 }
341
342 /*
343  * restore_cpu_complex
344  *
345  * restores cpu clock setting, clears flow controller
346  *
347  * Always called on CPU 0.
348  */
349 static void restore_cpu_complex(u32 mode)
350 {
351         int cpu = cpu_logical_map(smp_processor_id());
352         unsigned int reg;
353 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
354         unsigned int policy;
355 #endif
356
357 /*
358  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
359  * or restored by common clock suspend/resume procedures. Hence, we don't
360  * need it here.
361  */
362 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
363         /* Is CPU complex already running on PLLX? */
364         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
365         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
366
367         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
368                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
369         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
370                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
371         else
372                 BUG();
373
374         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
375                 /* restore PLLX settings if CPU is on different PLL */
376                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
377                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
378
379                 /* wait for PLL stabilization if PLLX was enabled */
380                 if (tegra_sctx.pllx_base & (1<<30)) {
381 #if USE_PLL_LOCK_BITS
382                         /* Enable lock detector */
383                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
384                         reg |= 1<<18;
385                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
386                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
387                                  (1<<27)))
388                                 cpu_relax();
389
390                         udelay(PLL_POST_LOCK_DELAY);
391 #else
392                         udelay(300);
393 #endif
394                 }
395         }
396
397         /* Restore original burst policy setting for calls resulting from CPU
398            LP2 in idle or system suspend; keep cluster switch prolog setting
399            intact. */
400         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
401                 writel(tegra_sctx.cclk_divider, clk_rst +
402                        CLK_RESET_CCLK_DIVIDER);
403                 writel(tegra_sctx.cpu_burst, clk_rst +
404                        CLK_RESET_CCLK_BURST);
405         }
406 #endif
407         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
408
409         /* Do not power-gate CPU 0 when flow controlled */
410         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
411         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
412         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
413         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
414         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
415         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
416         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
417
418         /* If an immedidate cluster switch is being perfomed, restore the
419            local timer registers. For calls resulting from CPU LP2 in
420            idle or system suspend, the local timer was shut down and
421            timekeeping switched over to the global system timer. In this
422            case keep local timer disabled, and restore only periodic load. */
423         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
424                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
425 #ifdef CONFIG_ARM_ARCH_TIMER
426                 tegra_sctx.arch_timer.cntp_ctl = 0;
427 #endif
428 #ifdef CONFIG_HAVE_ARM_TWD
429                 tegra_sctx.twd.twd_ctrl = 0;
430 #endif
431         }
432 #ifdef CONFIG_ARM_ARCH_TIMER
433         arch_timer_resume(&tegra_sctx.arch_timer);
434 #endif
435 #ifdef CONFIG_HAVE_ARM_TWD
436         tegra_twd_resume(&tegra_sctx.twd);
437 #endif
438 }
439
440 /*
441  * suspend_cpu_complex
442  *
443  * saves pll state for use by restart_plls, prepares flow controller for
444  * transition to suspend state
445  *
446  * Must always be called on cpu 0.
447  */
448 static void suspend_cpu_complex(u32 mode)
449 {
450         int cpu = cpu_logical_map(smp_processor_id());
451         unsigned int reg;
452         int i;
453
454         BUG_ON(cpu != 0);
455
456         /* switch coresite to clk_m, save off original source */
457         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
458         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
459
460         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
461         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
462         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
463         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
464         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
465         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
466         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
467         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
468
469 #ifdef CONFIG_HAVE_ARM_TWD
470         tegra_twd_suspend(&tegra_sctx.twd);
471 #endif
472 #ifdef CONFIG_ARM_ARCH_TIMER
473         arch_timer_suspend(&tegra_sctx.arch_timer);
474 #endif
475
476         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
477         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
478         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
479         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
480         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
481 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
482         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
483 #else
484         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
485 #endif
486         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
487         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
488
489         for (i = 0; i < num_possible_cpus(); i++) {
490                 if (i == cpu)
491                         continue;
492                 reg = readl(FLOW_CTRL_CPU_CSR(i));
493                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
494                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
495                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
496         }
497
498         tegra_gic_cpu_disable(true);
499 }
500
501 void tegra_clear_cpu_in_pd(int cpu)
502 {
503         spin_lock(&tegra_lp2_lock);
504         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
505         cpumask_clear_cpu(cpu, &tegra_in_lp2);
506
507         /* Update the IRAM copy used by the reset handler. The IRAM copy
508            can't use used directly by cpumask_clear_cpu() because it uses
509            LDREX/STREX which requires the addressed location to be inner
510            cacheable and sharable which IRAM isn't. */
511         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
512         dsb();
513
514         spin_unlock(&tegra_lp2_lock);
515 }
516
517 bool tegra_set_cpu_in_pd(int cpu)
518 {
519         bool last_cpu = false;
520
521         spin_lock(&tegra_lp2_lock);
522         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
523         cpumask_set_cpu(cpu, &tegra_in_lp2);
524
525         /* Update the IRAM copy used by the reset handler. The IRAM copy
526            can't use used directly by cpumask_set_cpu() because it uses
527            LDREX/STREX which requires the addressed location to be inner
528            cacheable and sharable which IRAM isn't. */
529         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
530         dsb();
531
532         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
533                 last_cpu = true;
534 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
535         else if (cpu == 1)
536                 tegra2_cpu_set_resettable_soon();
537 #endif
538
539         spin_unlock(&tegra_lp2_lock);
540         return last_cpu;
541 }
542
543 static void tegra_sleep_core(enum tegra_suspend_mode mode,
544                              unsigned long v2p)
545 {
546 #ifdef CONFIG_TRUSTED_FOUNDATIONS
547         if (mode == TEGRA_SUSPEND_LP0) {
548                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
549                                   virt_to_phys(tegra_resume));
550         } else {
551                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
552                                   (TEGRA_RESET_HANDLER_BASE +
553                                    tegra_cpu_reset_handler_offset));
554         }
555 #endif
556 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
557         cpu_suspend(v2p, tegra2_sleep_core_finish);
558 #else
559         cpu_suspend(v2p, tegra3_sleep_core_finish);
560 #endif
561 }
562
563 static inline void tegra_sleep_cpu(unsigned long v2p)
564 {
565 #ifdef CONFIG_TRUSTED_FOUNDATIONS
566         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
567                           (TEGRA_RESET_HANDLER_BASE +
568                            tegra_cpu_reset_handler_offset));
569 #endif
570         cpu_suspend(v2p, tegra_sleep_cpu_finish);
571 }
572
573 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
574                                         unsigned int flags)
575 {
576         u32 reg;
577         unsigned int remain;
578 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
579         pgd_t *pgd;
580 #endif
581
582         /* Only the last cpu down does the final suspend steps */
583         reg = readl(pmc + PMC_CTRL);
584         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
585         if (pdata->combined_req)
586                 reg &= ~TEGRA_POWER_PWRREQ_OE;
587         else
588                 reg |= TEGRA_POWER_PWRREQ_OE;
589
590         reg &= ~TEGRA_POWER_EFFECT_LP0;
591         writel(reg, pmc + PMC_CTRL);
592
593         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
594
595         /*
596          * We can use clk_get_rate_all_locked() here, because all other cpus
597          * are in LP2 state and irqs are disabled
598          */
599         if (flags & TEGRA_POWER_CLUSTER_MASK) {
600                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
601                 set_power_timers(pdata->cpu_timer, 2,
602                         clk_get_rate_all_locked(tegra_pclk));
603                 if (flags & TEGRA_POWER_CLUSTER_G) {
604                         /*
605                          * To reduce the vdd_cpu up latency when LP->G
606                          * transition. Before the transition, enable
607                          * the vdd_cpu rail.
608                          */
609                         if (is_lp_cluster()) {
610 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
611                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
612                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
613                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
614 #else
615                                 writel(UN_PWRGATE_CPU,
616                                        pmc + PMC_PWRGATE_TOGGLE);
617 #endif
618                         }
619                 }
620                 tegra_cluster_switch_prolog(flags);
621         } else {
622                 suspend_cpu_dfll_mode();
623                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
624                         clk_get_rate_all_locked(tegra_pclk));
625 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
626                 reg = readl(FLOW_CTRL_CPU_CSR(0));
627                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
628                 if (is_lp_cluster()) {
629                         /* for LP cluster, there is no option for rail gating */
630                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
631                                                 TEGRA_POWER_CLUSTER_PART_MASK)
632                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
633                         else if (flags)
634                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
635                 }
636                 else {
637                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
638                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
639                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
640                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
641                 }
642                 writel(reg, FLOW_CTRL_CPU_CSR(0));
643 #endif
644         }
645
646         if (sleep_time)
647                 tegra_pd_set_trigger(sleep_time);
648
649         cpu_cluster_pm_enter();
650         suspend_cpu_complex(flags);
651         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
652 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
653         flush_cache_all();
654         /*
655          * No need to flush complete L2. Cleaning kernel and IO mappings
656          * is enough for the LP code sequence that has L2 disabled but
657          * MMU on.
658          */
659         pgd = cpu_get_pgd();
660         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
661                           __pa(pgd + PTRS_PER_PGD));
662         outer_disable();
663 #endif
664         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
665
666         tegra_init_cache(false);
667         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
668         restore_cpu_complex(flags);
669         cpu_cluster_pm_exit();
670
671         remain = tegra_pd_timer_remain();
672         if (sleep_time)
673                 tegra_pd_set_trigger(0);
674
675         if (flags & TEGRA_POWER_CLUSTER_MASK) {
676                 tegra_cluster_switch_epilog(flags);
677                 if (is_idle_task(current))
678                         trace_cpu_cluster_rcuidle(POWER_CPU_CLUSTER_DONE);
679                 else
680                         trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
681         } else {
682                 resume_cpu_dfll_mode();
683         }
684         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
685
686 #if INSTRUMENT_CLUSTER_SWITCH
687         if (flags & TEGRA_POWER_CLUSTER_MASK) {
688                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
689                         is_lp_cluster() ? "G=>LP" : "LP=>G",
690                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
691                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
692                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
693                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
694                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
695                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
696                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
697                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
698         }
699 #endif
700         return remain;
701 }
702
703 static int tegra_common_suspend(void)
704 {
705         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
706
707         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
708         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
709         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
710
711         /* copy the reset vector and SDRAM shutdown code into IRAM */
712         memcpy(iram_save, iram_code, iram_save_size);
713         memcpy(iram_code, tegra_iram_start(), iram_save_size);
714
715         return 0;
716 }
717
718 static void tegra_common_resume(void)
719 {
720         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
721 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
722         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
723 #endif
724
725         /* Clear DPD sample */
726         writel(0x0, pmc + PMC_DPD_SAMPLE);
727
728         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
729         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
730         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
731 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
732         /* trigger emc mode write */
733         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
734 #endif
735         /* clear scratch registers shared by suspend and the reset pen */
736         writel(0x0, pmc + PMC_SCRATCH39);
737         writel(0x0, pmc + PMC_SCRATCH41);
738
739         /* restore IRAM */
740         memcpy(iram_code, iram_save, iram_save_size);
741 }
742
743 static int tegra_suspend_prepare_late(void)
744 {
745 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
746         disable_irq(INT_SYS_STATS_MON);
747 #endif
748         return 0;
749 }
750
751 static void tegra_suspend_wake(void)
752 {
753 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
754         enable_irq(INT_SYS_STATS_MON);
755 #endif
756 }
757
758 static void tegra_pm_set(enum tegra_suspend_mode mode)
759 {
760         u32 reg, boot_flag;
761         unsigned long rate = 32768;
762
763         reg = readl(pmc + PMC_CTRL);
764         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
765         if (pdata->combined_req)
766                 reg &= ~TEGRA_POWER_PWRREQ_OE;
767         else
768                 reg |= TEGRA_POWER_PWRREQ_OE;
769         reg &= ~TEGRA_POWER_EFFECT_LP0;
770
771         switch (mode) {
772         case TEGRA_SUSPEND_LP0:
773 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
774                 rate = clk_get_rate_all_locked(tegra_pclk);
775 #endif
776                 if (pdata->combined_req) {
777                         reg |= TEGRA_POWER_PWRREQ_OE;
778                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
779                 }
780
781                 /*
782                  * LP0 boots through the AVP, which then resumes the AVP to
783                  * the address in scratch 39, and the cpu to the address in
784                  * scratch 41 to tegra_resume
785                  */
786                 writel(0x0, pmc + PMC_SCRATCH39);
787
788                 /* Enable DPD sample to trigger sampling pads data and direction
789                  * in which pad will be driven during lp0 mode*/
790                 writel(0x1, pmc + PMC_DPD_SAMPLE);
791 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
792                 /* this is needed only for T11x, not for other chips */
793                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
794 #endif
795
796                 /* Set warmboot flag */
797                 boot_flag = readl(pmc + PMC_SCRATCH0);
798                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
799
800                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
801
802                 reg |= TEGRA_POWER_EFFECT_LP0;
803                 /* No break here. LP0 code falls through to write SCRATCH41 */
804         case TEGRA_SUSPEND_LP1:
805                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
806                 wmb();
807                 break;
808         case TEGRA_SUSPEND_LP2:
809                 rate = clk_get_rate(tegra_pclk);
810                 break;
811         case TEGRA_SUSPEND_NONE:
812                 return;
813         default:
814                 BUG();
815         }
816
817         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
818
819         pmc_32kwritel(reg, PMC_CTRL);
820 }
821
822 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
823         [TEGRA_SUSPEND_NONE] = "none",
824         [TEGRA_SUSPEND_LP2] = "LP2",
825         [TEGRA_SUSPEND_LP1] = "LP1",
826         [TEGRA_SUSPEND_LP0] = "LP0",
827 };
828
829 static int tegra_suspend_enter(suspend_state_t state)
830 {
831         int ret;
832         ktime_t delta;
833         struct timespec ts_entry, ts_exit;
834
835         if (pdata && pdata->board_suspend)
836                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
837
838         read_persistent_clock(&ts_entry);
839
840         ret = tegra_suspend_dram(current_suspend_mode, 0);
841         if (ret) {
842                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
843                 goto abort_suspend;
844         }
845
846         read_persistent_clock(&ts_exit);
847
848         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
849                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
850
851                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
852                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
853                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
854                 else
855                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
856         }
857
858 abort_suspend:
859         if (pdata && pdata->board_resume)
860                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
861
862         return ret;
863 }
864
865 static void tegra_suspend_check_pwr_stats(void)
866 {
867         /* cpus and l2 are powered off later */
868         unsigned long pwrgate_partid_mask =
869 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
870                 (1 << TEGRA_POWERGATE_HEG)      |
871                 (1 << TEGRA_POWERGATE_SATA)     |
872                 (1 << TEGRA_POWERGATE_3D1)      |
873 #endif
874                 (1 << TEGRA_POWERGATE_3D)       |
875                 (1 << TEGRA_POWERGATE_VENC)     |
876                 (1 << TEGRA_POWERGATE_PCIE)     |
877                 (1 << TEGRA_POWERGATE_VDEC)     |
878                 (1 << TEGRA_POWERGATE_MPE);
879
880         int partid;
881
882         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
883                 if ((1 << partid) & pwrgate_partid_mask)
884                         if (tegra_powergate_is_powered(partid))
885                                 pr_warning("partition %s is left on before suspend\n",
886                                         tegra_powergate_get_name(partid));
887
888         return;
889 }
890
891 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
892 {
893         int err = 0;
894         u32 scratch37 = 0xDEADBEEF;
895         u32 reg;
896
897         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
898                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
899                 err = -ENXIO;
900                 goto fail;
901         }
902
903         if (tegra_is_voice_call_active()) {
904                 /* backup the current value of scratch37 */
905                 scratch37 = readl(pmc + PMC_SCRATCH37);
906
907                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
908                 reg = TEGRA_POWER_LP1_AUDIO;
909                 pmc_32kwritel(reg, PMC_SCRATCH37);
910         }
911
912         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
913                 pr_info("LP0 not used due to unsupported wakeup events\n");
914                 mode = TEGRA_SUSPEND_LP1;
915         }
916
917         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
918                 tegra_suspend_check_pwr_stats();
919
920         tegra_common_suspend();
921
922         tegra_pm_set(mode);
923
924         if (pdata && pdata->board_suspend)
925                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
926
927         local_fiq_disable();
928
929         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
930
931         if (mode == TEGRA_SUSPEND_LP0) {
932 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
933                 reg = readl(pmc + PMC_SCRATCH4);
934                 if (is_lp_cluster())
935                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
936                 else
937                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
938                 pmc_32kwritel(reg, PMC_SCRATCH4);
939 #endif
940                 tegra_tsc_suspend();
941                 tegra_lp0_suspend_mc();
942                 tegra_cpu_reset_handler_save();
943                 tegra_tsc_wait_for_suspend();
944                 tegra_smp_clear_power_mask();
945         }
946         else if (mode == TEGRA_SUSPEND_LP1)
947                 *iram_cpu_lp1_mask = 1;
948
949         suspend_cpu_complex(flags);
950
951 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
952         /* In case of LP0/1, program external power gating accordinly */
953         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
954                 reg = readl(FLOW_CTRL_CPU_CSR(0));
955                 if (is_lp_cluster())
956                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
957                 else
958                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
959                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
960         }
961 #endif
962
963         flush_cache_all();
964         outer_flush_all();
965         outer_disable();
966
967         if (mode == TEGRA_SUSPEND_LP2)
968                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
969         else
970                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
971
972         tegra_init_cache(true);
973
974         if (mode == TEGRA_SUSPEND_LP0) {
975
976                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
977                 * Otherwise it is creating issue in cluster switch after LP0 *
978 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
979                 reg = readl(pmc+PMC_CTRL);
980                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
981                 pmc_32kwritel(reg, PMC_CTRL);
982 #endif
983                 */
984
985                 tegra_tsc_resume();
986                 tegra_cpu_reset_handler_restore();
987                 tegra_lp0_resume_mc();
988                 tegra_tsc_wait_for_resume();
989         } else if (mode == TEGRA_SUSPEND_LP1)
990                 *iram_cpu_lp1_mask = 0;
991
992         /* if scratch37 was clobbered during LP1, restore it */
993         if (scratch37 != 0xDEADBEEF)
994                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
995
996         restore_cpu_complex(flags);
997
998         /* for platforms where the core & CPU power requests are
999          * combined as a single request to the PMU, transition out
1000          * of LP0 state by temporarily enabling both requests
1001          */
1002         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1003                 reg = readl(pmc + PMC_CTRL);
1004                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1005                 pmc_32kwritel(reg, PMC_CTRL);
1006                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1007                 pmc_32kwritel(reg, PMC_CTRL);
1008         }
1009
1010         if (pdata && pdata->board_resume)
1011                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1012
1013         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1014
1015         local_fiq_enable();
1016
1017         tegra_common_resume();
1018
1019 fail:
1020         return err;
1021 }
1022
1023 /*
1024  * Function pointers to optional board specific function
1025  */
1026 void (*tegra_deep_sleep)(int);
1027 EXPORT_SYMBOL(tegra_deep_sleep);
1028
1029 static int tegra_suspend_prepare(void)
1030 {
1031         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1032                 tegra_deep_sleep(1);
1033         return 0;
1034 }
1035
1036 static void tegra_suspend_finish(void)
1037 {
1038         if (pdata && pdata->cpu_resume_boost) {
1039                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1040                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1041                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1042         }
1043
1044         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1045                 tegra_deep_sleep(0);
1046 }
1047
1048 static const struct platform_suspend_ops tegra_suspend_ops = {
1049         .valid          = suspend_valid_only_mem,
1050         .prepare        = tegra_suspend_prepare,
1051         .finish         = tegra_suspend_finish,
1052         .prepare_late   = tegra_suspend_prepare_late,
1053         .wake           = tegra_suspend_wake,
1054         .enter          = tegra_suspend_enter,
1055 };
1056
1057 static ssize_t suspend_mode_show(struct kobject *kobj,
1058                                         struct kobj_attribute *attr, char *buf)
1059 {
1060         char *start = buf;
1061         char *end = buf + PAGE_SIZE;
1062
1063         start += scnprintf(start, end - start, "%s ", \
1064                                 tegra_suspend_name[current_suspend_mode]);
1065         start += scnprintf(start, end - start, "\n");
1066
1067         return start - buf;
1068 }
1069
1070 static ssize_t suspend_mode_store(struct kobject *kobj,
1071                                         struct kobj_attribute *attr,
1072                                         const char *buf, size_t n)
1073 {
1074         int len;
1075         const char *name_ptr;
1076         enum tegra_suspend_mode new_mode;
1077
1078         name_ptr = buf;
1079         while (*name_ptr && !isspace(*name_ptr))
1080                 name_ptr++;
1081         len = name_ptr - buf;
1082         if (!len)
1083                 goto bad_name;
1084         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1085         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1086                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1087                 pr_info("Illegal tegra suspend state: %s\n", buf);
1088                 goto bad_name;
1089         }
1090
1091         for (new_mode = TEGRA_SUSPEND_NONE; \
1092                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1093                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1094                         current_suspend_mode = new_mode;
1095                         break;
1096                 }
1097         }
1098
1099 bad_name:
1100         return n;
1101 }
1102
1103 static struct kobj_attribute suspend_mode_attribute =
1104         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1105
1106 static struct kobject *suspend_kobj;
1107
1108 static int tegra_pm_enter_suspend(void)
1109 {
1110         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1111         suspend_cpu_dfll_mode();
1112         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1113                 tegra_lp0_cpu_mode(true);
1114         return 0;
1115 }
1116
1117 static void tegra_pm_enter_resume(void)
1118 {
1119         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1120                 tegra_lp0_cpu_mode(false);
1121         resume_cpu_dfll_mode();
1122         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1123 }
1124
1125 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1126         .suspend = tegra_pm_enter_suspend,
1127         .resume = tegra_pm_enter_resume,
1128 };
1129
1130 static __init int tegra_pm_enter_syscore_init(void)
1131 {
1132         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1133         return 0;
1134 }
1135 subsys_initcall(tegra_pm_enter_syscore_init);
1136 #endif
1137
1138 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1139 {
1140         u32 reg;
1141         u32 mode;
1142
1143 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1144         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1145         BUG_ON(IS_ERR(tegra_dfll));
1146 #endif
1147         tegra_pclk = clk_get_sys(NULL, "pclk");
1148         BUG_ON(IS_ERR(tegra_pclk));
1149         pdata = plat;
1150         (void)reg;
1151         (void)mode;
1152
1153         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1154                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1155
1156 #ifndef CONFIG_PM_SLEEP
1157         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1158                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1159                            "disabling suspend\n", __func__);
1160                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1161         }
1162 #else
1163         if (create_suspend_pgtable() < 0) {
1164                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1165                                 __func__);
1166                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1167                 goto fail;
1168         }
1169
1170         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1171             (tegra_revision == TEGRA_REVISION_A01) &&
1172             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1173                 /* Tegra 3 A01 supports only LP1 */
1174                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1175                            "-- disabling LP0\n", __func__);
1176                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1177         }
1178         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1179                 tegra_lp0_vec_relocate) {
1180                 unsigned char *reloc_lp0;
1181                 unsigned long tmp;
1182                 void __iomem *orig;
1183                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1184                                         GFP_KERNEL);
1185                 WARN_ON(!reloc_lp0);
1186                 if (!reloc_lp0) {
1187                         pr_err("%s: Failed to allocate reloc_lp0\n",
1188                                 __func__);
1189                         goto out;
1190                 }
1191
1192                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1193                 WARN_ON(!orig);
1194                 if (!orig) {
1195                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1196                                 __func__, tegra_lp0_vec_start);
1197                         kfree(reloc_lp0);
1198                         goto out;
1199                 }
1200
1201                 tmp = (unsigned long) reloc_lp0;
1202                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1203                 reloc_lp0 = (unsigned char *)tmp;
1204                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1205                 iounmap(orig);
1206                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1207         }
1208
1209 out:
1210         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1211                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1212                            "provided by bootlader -- disabling LP0\n",
1213                            __func__);
1214                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1215         }
1216
1217         iram_save_size = tegra_iram_end() - tegra_iram_start();
1218
1219         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1220         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1221                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1222                        "-- LP0/LP1 unavailable\n", __func__);
1223                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1224         }
1225
1226 #ifdef CONFIG_TEGRA_LP1_950
1227         if (pdata->lp1_lowvolt_support) {
1228                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1229                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1230                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1231
1232                 lp1_core_lowvolt = 0;
1233                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1234                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1235
1236                 lp1_core_highvolt = 0;
1237                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1238                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1239         }
1240 #endif
1241         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1242         /* Initialize scratch registers used for CPU LP2 synchronization */
1243         writel(0, pmc + PMC_SCRATCH37);
1244         writel(0, pmc + PMC_SCRATCH38);
1245         writel(0, pmc + PMC_SCRATCH39);
1246         writel(0, pmc + PMC_SCRATCH41);
1247
1248         /* Always enable CPU power request; just normal polarity is supported */
1249         reg = readl(pmc + PMC_CTRL);
1250         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1251         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1252         pmc_32kwritel(reg, PMC_CTRL);
1253
1254         /* Configure core power request and system clock control if LP0
1255            is supported */
1256         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1257         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1258
1259         reg = readl(pmc + PMC_CTRL);
1260
1261         if (!pdata->sysclkreq_high)
1262                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1263         else
1264                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1265
1266         if (!pdata->corereq_high)
1267                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1268         else
1269                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1270
1271         /* configure output inverters while the request is tristated */
1272         pmc_32kwritel(reg, PMC_CTRL);
1273
1274         /* now enable requests */
1275         reg |= TEGRA_POWER_SYSCLK_OE;
1276         if (!pdata->combined_req)
1277                 reg |= TEGRA_POWER_PWRREQ_OE;
1278         pmc_32kwritel(reg, PMC_CTRL);
1279
1280         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1281                 tegra_lp0_suspend_init();
1282
1283         suspend_set_ops(&tegra_suspend_ops);
1284
1285         /* Create /sys/power/suspend/type */
1286         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1287         if (suspend_kobj) {
1288                 if (sysfs_create_file(suspend_kobj, \
1289                                                 &suspend_mode_attribute.attr))
1290                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1291                                                                 __func__);
1292         }
1293
1294         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1295         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1296
1297         /* clear io dpd settings before kernel */
1298         tegra_bl_io_dpd_cleanup();
1299
1300 fail:
1301 #endif
1302         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1303                 tegra_pd_in_idle(false);
1304
1305         current_suspend_mode = plat->suspend_mode;
1306 }
1307
1308 unsigned long debug_uart_port_base = 0;
1309 EXPORT_SYMBOL(debug_uart_port_base);
1310
1311 static int tegra_debug_uart_suspend(void)
1312 {
1313         void __iomem *uart;
1314         u32 lcr;
1315
1316         if (!debug_uart_port_base)
1317                 return 0;
1318
1319         uart = IO_ADDRESS(debug_uart_port_base);
1320
1321         lcr = readb(uart + UART_LCR * 4);
1322
1323         tegra_sctx.uart[0] = lcr;
1324         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1325
1326         /* DLAB = 0 */
1327         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1328
1329         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1330
1331         /* DLAB = 1 */
1332         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1333
1334         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1335         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1336
1337         writeb(lcr, uart + UART_LCR * 4);
1338
1339         return 0;
1340 }
1341
1342 static void tegra_debug_uart_resume(void)
1343 {
1344         void __iomem *uart;
1345         u32 lcr;
1346
1347         if (!debug_uart_port_base)
1348                 return;
1349
1350         uart = IO_ADDRESS(debug_uart_port_base);
1351
1352         lcr = tegra_sctx.uart[0];
1353
1354         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1355
1356         /* DLAB = 0 */
1357         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1358
1359         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1360                         uart + UART_FCR * 4);
1361
1362         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1363
1364         /* DLAB = 1 */
1365         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1366
1367         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1368         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1369
1370         writeb(lcr, uart + UART_LCR * 4);
1371 }
1372
1373 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1374         .suspend = tegra_debug_uart_suspend,
1375         .resume = tegra_debug_uart_resume,
1376 };
1377
1378 struct clk *debug_uart_clk = NULL;
1379 EXPORT_SYMBOL(debug_uart_clk);
1380
1381 void tegra_console_uart_suspend(void)
1382 {
1383         if (console_suspend_enabled && debug_uart_clk)
1384                 tegra_clk_disable_unprepare(debug_uart_clk);
1385 }
1386
1387 void tegra_console_uart_resume(void)
1388 {
1389         if (console_suspend_enabled && debug_uart_clk)
1390                 tegra_clk_prepare_enable(debug_uart_clk);
1391 }
1392
1393 static int tegra_debug_uart_syscore_init(void)
1394 {
1395         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1396         return 0;
1397 }
1398 arch_initcall(tegra_debug_uart_syscore_init);