ARM: Tegra: Tegra specific traces to own module
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <asm/cacheflush.h>
50 #include <asm/idmap.h>
51 #include <asm/localtimer.h>
52 #include <asm/pgalloc.h>
53 #include <asm/pgtable.h>
54 #include <asm/tlbflush.h>
55 #include <asm/suspend.h>
56 #include <asm/smp_plat.h>
57
58 #include <mach/irqs.h>
59 #include <mach/powergate.h>
60 #include <mach/hardware.h>
61
62 #include "board.h"
63 #include "clock.h"
64 #include "common.h"
65 #include "cpuidle.h"
66 #include "fuse.h"
67 #include "gic.h"
68 #include "iomap.h"
69 #include "pm.h"
70 #include "pm-irq.h"
71 #include "reset.h"
72 #include "sleep.h"
73 #include "timer.h"
74 #include "dvfs.h"
75 #include "cpu-tegra.h"
76
77 #include <trace/events/nvpower.h>
78 #define CREATE_TRACE_POINTS
79 #include <trace/events/nvpower.h>
80
81 struct suspend_context {
82         /*
83          * The next 7 values are referenced by offset in __restart_plls
84          * in headsmp-t2.S, and should not be moved
85          */
86         u32 pllx_misc;
87         u32 pllx_base;
88         u32 pllp_misc;
89         u32 pllp_base;
90         u32 pllp_outa;
91         u32 pllp_outb;
92         u32 pll_timeout;
93
94         u32 cpu_burst;
95         u32 clk_csite_src;
96         u32 cclk_divider;
97
98         u32 mc[3];
99         u8 uart[5];
100
101         struct tegra_twd_context twd;
102 #ifdef CONFIG_ARM_ARCH_TIMER
103         struct arch_timer_context arch_timer;
104 #endif
105 };
106
107 #ifdef CONFIG_PM_SLEEP
108 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
109 static pgd_t *tegra_pgd;
110 static DEFINE_SPINLOCK(tegra_lp2_lock);
111 static cpumask_t tegra_in_lp2;
112 static cpumask_t *iram_cpu_lp2_mask;
113 static unsigned long *iram_cpu_lp1_mask;
114 static u8 *iram_save;
115 static unsigned long iram_save_size;
116 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
117 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
118 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
119 static int tegra_last_pclk;
120 #endif
121
122 struct suspend_context tegra_sctx;
123
124 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
125 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
126 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
127 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
128 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
129 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
130 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
131 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
132 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
133
134 #define PMC_CTRL                0x0
135 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
136 #define PMC_WAKE_MASK           0xc
137 #define PMC_WAKE_LEVEL          0x10
138 #define PMC_DPAD_ORIDE          0x1C
139 #define PMC_WAKE_DELAY          0xe0
140 #define PMC_DPD_SAMPLE          0x20
141
142 #define PMC_WAKE_STATUS         0x14
143 #define PMC_SW_WAKE_STATUS      0x18
144 #define PMC_COREPWRGOOD_TIMER   0x3c
145 #define PMC_CPUPWRGOOD_TIMER    0xc8
146 #define PMC_CPUPWROFF_TIMER     0xcc
147 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
148
149 #define PMC_PWRGATE_TOGGLE      0x30
150 #define PWRGATE_TOGGLE_START    (1 << 8)
151 #define UN_PWRGATE_CPU          \
152         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
153
154 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
155 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
156 #endif
157
158 #define CLK_RESET_CCLK_BURST    0x20
159 #define CLK_RESET_CCLK_DIVIDER  0x24
160 #define CLK_RESET_PLLC_BASE     0x80
161 #define CLK_RESET_PLLM_BASE     0x90
162 #define CLK_RESET_PLLX_BASE     0xe0
163 #define CLK_RESET_PLLX_MISC     0xe4
164 #define CLK_RESET_PLLP_BASE     0xa0
165 #define CLK_RESET_PLLP_OUTA     0xa4
166 #define CLK_RESET_PLLP_OUTB     0xa8
167 #define CLK_RESET_PLLP_MISC     0xac
168
169 #define CLK_RESET_SOURCE_CSITE  0x1d4
170
171 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
172 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
173 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
174 #define CLK_RESET_CCLK_IDLE_POLICY         1
175 #define CLK_RESET_CCLK_RUN_POLICY          2
176 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
177 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
178
179 #define EMC_MRW_0               0x0e8
180 #define EMC_MRW_DEV_SELECTN     30
181 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
182
183 #define MC_SECURITY_START       0x6c
184 #define MC_SECURITY_SIZE        0x70
185 #define MC_SECURITY_CFG2        0x7c
186
187 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
188 static struct clk *tegra_dfll;
189 #endif
190 static struct clk *tegra_pclk;
191 static const struct tegra_suspend_platform_data *pdata;
192 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
193
194 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
195 enum tegra_cluster_switch_time_id {
196         tegra_cluster_switch_time_id_start = 0,
197         tegra_cluster_switch_time_id_prolog,
198         tegra_cluster_switch_time_id_switch,
199         tegra_cluster_switch_time_id_epilog,
200         tegra_cluster_switch_time_id_max
201 };
202
203 static unsigned long
204                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
205 #define tegra_cluster_switch_time(flags, id) \
206         do { \
207                 barrier(); \
208                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
209                         void __iomem *timer_us = \
210                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
211                         if (id < tegra_cluster_switch_time_id_max) \
212                                 tegra_cluster_switch_times[id] = \
213                                                         readl(timer_us); \
214                                 wmb(); \
215                 } \
216                 barrier(); \
217         } while(0)
218 #else
219 #define tegra_cluster_switch_time(flags, id) do {} while(0)
220 #endif
221
222 #ifdef CONFIG_PM_SLEEP
223 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
224         [TEGRA_SUSPEND_NONE]    = "none",
225         [TEGRA_SUSPEND_LP2]     = "lp2",
226         [TEGRA_SUSPEND_LP1]     = "lp1",
227         [TEGRA_SUSPEND_LP0]     = "lp0",
228 };
229
230 unsigned long tegra_cpu_power_good_time(void)
231 {
232         if (WARN_ON_ONCE(!pdata))
233                 return 5000;
234
235         return pdata->cpu_timer;
236 }
237
238 unsigned long tegra_cpu_power_off_time(void)
239 {
240         if (WARN_ON_ONCE(!pdata))
241                 return 5000;
242
243         return pdata->cpu_off_timer;
244 }
245
246 unsigned long tegra_cpu_lp2_min_residency(void)
247 {
248         if (WARN_ON_ONCE(!pdata))
249                 return 2000;
250
251         return pdata->cpu_lp2_min_residency;
252 }
253
254 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
255 unsigned long tegra_min_residency_noncpu(void)
256 {
257         return pdata->min_residency_noncpu;
258 }
259
260 unsigned long tegra_min_residency_crail(void)
261 {
262         return pdata->min_residency_crail;
263 }
264 #endif
265
266 static void suspend_cpu_dfll_mode(void)
267 {
268 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
269         /* If DFLL is used as CPU clock source go to open loop mode */
270         if (!is_lp_cluster() && tegra_dfll &&
271             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
272                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
273 #endif
274 }
275
276 static void resume_cpu_dfll_mode(void)
277 {
278 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
279         /* If DFLL is used as CPU clock source restore closed loop mode */
280         if (!is_lp_cluster() && tegra_dfll &&
281             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
282                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
283 #endif
284 }
285
286 /*
287  * create_suspend_pgtable
288  *
289  * Creates a page table with identity mappings of physical memory and IRAM
290  * for use when the MMU is off, in addition to all the regular kernel mappings.
291  */
292 static __init int create_suspend_pgtable(void)
293 {
294         tegra_pgd = pgd_alloc(&init_mm);
295         if (!tegra_pgd)
296                 return -ENOMEM;
297
298         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
299         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
300                 high_memory, 0);
301         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
302                 IO_IRAM_VIRT + SECTION_SIZE, 0);
303
304         /* inner/outer write-back/write-allocate, sharable */
305         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
306
307         return 0;
308 }
309
310 /* ensures that sufficient time is passed for a register write to
311  * serialize into the 32KHz domain */
312 static void pmc_32kwritel(u32 val, unsigned long offs)
313 {
314         writel(val, pmc + offs);
315         udelay(130);
316 }
317
318 static void set_power_timers(unsigned long us_on, unsigned long us_off,
319                              long rate)
320 {
321         static unsigned long last_us_off = 0;
322         unsigned long long ticks;
323         unsigned long long pclk;
324
325         if (WARN_ON_ONCE(rate <= 0))
326                 pclk = 100000000;
327         else
328                 pclk = rate;
329
330         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
331                 ticks = (us_on * pclk) + 999999ull;
332                 do_div(ticks, 1000000);
333                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
334
335                 ticks = (us_off * pclk) + 999999ull;
336                 do_div(ticks, 1000000);
337                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
338                 wmb();
339         }
340         tegra_last_pclk = pclk;
341         last_us_off = us_off;
342 }
343
344 /*
345  * restore_cpu_complex
346  *
347  * restores cpu clock setting, clears flow controller
348  *
349  * Always called on CPU 0.
350  */
351 static void restore_cpu_complex(u32 mode)
352 {
353         int cpu = cpu_logical_map(smp_processor_id());
354         unsigned int reg;
355 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
356         unsigned int policy;
357 #endif
358
359 /*
360  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
361  * or restored by common clock suspend/resume procedures. Hence, we don't
362  * need it here.
363  */
364 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
365         /* Is CPU complex already running on PLLX? */
366         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
367         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
368
369         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
370                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
371         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
372                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
373         else
374                 BUG();
375
376         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
377                 /* restore PLLX settings if CPU is on different PLL */
378                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
379                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
380
381                 /* wait for PLL stabilization if PLLX was enabled */
382                 if (tegra_sctx.pllx_base & (1<<30)) {
383 #if USE_PLL_LOCK_BITS
384                         /* Enable lock detector */
385                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
386                         reg |= 1<<18;
387                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
388                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
389                                  (1<<27)))
390                                 cpu_relax();
391
392                         udelay(PLL_POST_LOCK_DELAY);
393 #else
394                         udelay(300);
395 #endif
396                 }
397         }
398
399         /* Restore original burst policy setting for calls resulting from CPU
400            LP2 in idle or system suspend; keep cluster switch prolog setting
401            intact. */
402         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
403                 writel(tegra_sctx.cclk_divider, clk_rst +
404                        CLK_RESET_CCLK_DIVIDER);
405                 writel(tegra_sctx.cpu_burst, clk_rst +
406                        CLK_RESET_CCLK_BURST);
407         }
408 #endif
409         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
410
411         /* Do not power-gate CPU 0 when flow controlled */
412         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
413         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
414         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
415         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
416         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
417         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
418         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
419
420         /* If an immedidate cluster switch is being perfomed, restore the
421            local timer registers. For calls resulting from CPU LP2 in
422            idle or system suspend, the local timer was shut down and
423            timekeeping switched over to the global system timer. In this
424            case keep local timer disabled, and restore only periodic load. */
425         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
426                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
427 #ifdef CONFIG_ARM_ARCH_TIMER
428                 tegra_sctx.arch_timer.cntp_ctl = 0;
429 #endif
430 #ifdef CONFIG_HAVE_ARM_TWD
431                 tegra_sctx.twd.twd_ctrl = 0;
432 #endif
433         }
434 #ifdef CONFIG_ARM_ARCH_TIMER
435         arch_timer_resume(&tegra_sctx.arch_timer);
436 #endif
437 #ifdef CONFIG_HAVE_ARM_TWD
438         tegra_twd_resume(&tegra_sctx.twd);
439 #endif
440 }
441
442 /*
443  * suspend_cpu_complex
444  *
445  * saves pll state for use by restart_plls, prepares flow controller for
446  * transition to suspend state
447  *
448  * Must always be called on cpu 0.
449  */
450 static void suspend_cpu_complex(u32 mode)
451 {
452         int cpu = cpu_logical_map(smp_processor_id());
453         unsigned int reg;
454         int i;
455
456         BUG_ON(cpu != 0);
457
458         /* switch coresite to clk_m, save off original source */
459         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
460         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
461
462         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
463         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
464         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
465         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
466         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
467         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
468         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
469         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
470
471 #ifdef CONFIG_HAVE_ARM_TWD
472         tegra_twd_suspend(&tegra_sctx.twd);
473 #endif
474 #ifdef CONFIG_ARM_ARCH_TIMER
475         arch_timer_suspend(&tegra_sctx.arch_timer);
476 #endif
477
478         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
479         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
480         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
481         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
482         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
483 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
484         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
485 #else
486         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
487 #endif
488         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
489         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
490
491         for (i = 0; i < num_possible_cpus(); i++) {
492                 if (i == cpu)
493                         continue;
494                 reg = readl(FLOW_CTRL_CPU_CSR(i));
495                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
496                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
497                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
498         }
499
500         tegra_gic_cpu_disable(true);
501 }
502
503 void tegra_clear_cpu_in_pd(int cpu)
504 {
505         spin_lock(&tegra_lp2_lock);
506         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
507         cpumask_clear_cpu(cpu, &tegra_in_lp2);
508
509         /* Update the IRAM copy used by the reset handler. The IRAM copy
510            can't use used directly by cpumask_clear_cpu() because it uses
511            LDREX/STREX which requires the addressed location to be inner
512            cacheable and sharable which IRAM isn't. */
513         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
514         dsb();
515
516         spin_unlock(&tegra_lp2_lock);
517 }
518
519 bool tegra_set_cpu_in_pd(int cpu)
520 {
521         bool last_cpu = false;
522
523         spin_lock(&tegra_lp2_lock);
524         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
525         cpumask_set_cpu(cpu, &tegra_in_lp2);
526
527         /* Update the IRAM copy used by the reset handler. The IRAM copy
528            can't use used directly by cpumask_set_cpu() because it uses
529            LDREX/STREX which requires the addressed location to be inner
530            cacheable and sharable which IRAM isn't. */
531         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
532         dsb();
533
534         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
535                 last_cpu = true;
536 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
537         else if (cpu == 1)
538                 tegra2_cpu_set_resettable_soon();
539 #endif
540
541         spin_unlock(&tegra_lp2_lock);
542         return last_cpu;
543 }
544
545 static void tegra_sleep_core(enum tegra_suspend_mode mode,
546                              unsigned long v2p)
547 {
548 #ifdef CONFIG_TRUSTED_FOUNDATIONS
549         if (mode == TEGRA_SUSPEND_LP0) {
550                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
551                                   virt_to_phys(tegra_resume));
552         } else {
553                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
554                                   (TEGRA_RESET_HANDLER_BASE +
555                                    tegra_cpu_reset_handler_offset));
556         }
557 #endif
558 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
559         cpu_suspend(v2p, tegra2_sleep_core_finish);
560 #else
561         cpu_suspend(v2p, tegra3_sleep_core_finish);
562 #endif
563 }
564
565 static inline void tegra_sleep_cpu(unsigned long v2p)
566 {
567 #ifdef CONFIG_TRUSTED_FOUNDATIONS
568         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
569                           (TEGRA_RESET_HANDLER_BASE +
570                            tegra_cpu_reset_handler_offset));
571 #endif
572         cpu_suspend(v2p, tegra_sleep_cpu_finish);
573 }
574
575 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
576                                         unsigned int flags)
577 {
578         u32 reg;
579         unsigned int remain;
580 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
581         pgd_t *pgd;
582 #endif
583
584         /* Only the last cpu down does the final suspend steps */
585         reg = readl(pmc + PMC_CTRL);
586         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
587         if (pdata->combined_req)
588                 reg &= ~TEGRA_POWER_PWRREQ_OE;
589         else
590                 reg |= TEGRA_POWER_PWRREQ_OE;
591
592         reg &= ~TEGRA_POWER_EFFECT_LP0;
593         writel(reg, pmc + PMC_CTRL);
594
595         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
596
597         /*
598          * We can use clk_get_rate_all_locked() here, because all other cpus
599          * are in LP2 state and irqs are disabled
600          */
601         if (flags & TEGRA_POWER_CLUSTER_MASK) {
602                 trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_START);
603                 set_power_timers(pdata->cpu_timer, 2,
604                         clk_get_rate_all_locked(tegra_pclk));
605                 if (flags & TEGRA_POWER_CLUSTER_G) {
606                         /*
607                          * To reduce the vdd_cpu up latency when LP->G
608                          * transition. Before the transition, enable
609                          * the vdd_cpu rail.
610                          */
611                         if (is_lp_cluster()) {
612 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
613                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
614                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
615                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
616 #else
617                                 writel(UN_PWRGATE_CPU,
618                                        pmc + PMC_PWRGATE_TOGGLE);
619 #endif
620                         }
621                 }
622                 tegra_cluster_switch_prolog(flags);
623         } else {
624                 suspend_cpu_dfll_mode();
625                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
626                         clk_get_rate_all_locked(tegra_pclk));
627 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
628                 reg = readl(FLOW_CTRL_CPU_CSR(0));
629                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
630                 if (is_lp_cluster()) {
631                         /* for LP cluster, there is no option for rail gating */
632                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
633                                                 TEGRA_POWER_CLUSTER_PART_MASK)
634                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
635                         else if (flags)
636                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
637                 }
638                 else {
639                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
640                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
641                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
642                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
643                 }
644                 writel(reg, FLOW_CTRL_CPU_CSR(0));
645 #endif
646         }
647
648         if (sleep_time)
649                 tegra_pd_set_trigger(sleep_time);
650
651         cpu_cluster_pm_enter();
652         suspend_cpu_complex(flags);
653         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
654 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
655         flush_cache_all();
656         /*
657          * No need to flush complete L2. Cleaning kernel and IO mappings
658          * is enough for the LP code sequence that has L2 disabled but
659          * MMU on.
660          */
661         pgd = cpu_get_pgd();
662         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
663                           __pa(pgd + PTRS_PER_PGD));
664         outer_disable();
665 #endif
666         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
667
668         tegra_init_cache(false);
669         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
670         restore_cpu_complex(flags);
671         cpu_cluster_pm_exit();
672
673         remain = tegra_pd_timer_remain();
674         if (sleep_time)
675                 tegra_pd_set_trigger(0);
676
677         if (flags & TEGRA_POWER_CLUSTER_MASK) {
678                 tegra_cluster_switch_epilog(flags);
679                 if (is_idle_task(current))
680                         trace_nvcpu_cluster_rcuidle(NVPOWER_CPU_CLUSTER_DONE);
681                 else
682                         trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_DONE);
683         } else {
684                 resume_cpu_dfll_mode();
685         }
686         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
687
688 #if INSTRUMENT_CLUSTER_SWITCH
689         if (flags & TEGRA_POWER_CLUSTER_MASK) {
690                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
691                         is_lp_cluster() ? "G=>LP" : "LP=>G",
692                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
693                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
694                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
695                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
696                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
697                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
698                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
699                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
700         }
701 #endif
702         return remain;
703 }
704
705 static int tegra_common_suspend(void)
706 {
707         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
708
709         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
710         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
711         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
712
713         /* copy the reset vector and SDRAM shutdown code into IRAM */
714         memcpy(iram_save, iram_code, iram_save_size);
715         memcpy(iram_code, tegra_iram_start(), iram_save_size);
716
717         return 0;
718 }
719
720 static void tegra_common_resume(void)
721 {
722         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
723 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
724         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
725 #endif
726
727         /* Clear DPD sample */
728         writel(0x0, pmc + PMC_DPD_SAMPLE);
729
730         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
731         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
732         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
733 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
734         /* trigger emc mode write */
735         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
736 #endif
737         /* clear scratch registers shared by suspend and the reset pen */
738         writel(0x0, pmc + PMC_SCRATCH39);
739         writel(0x0, pmc + PMC_SCRATCH41);
740
741         /* restore IRAM */
742         memcpy(iram_code, iram_save, iram_save_size);
743 }
744
745 static int tegra_suspend_prepare_late(void)
746 {
747 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
748         disable_irq(INT_SYS_STATS_MON);
749 #endif
750         return 0;
751 }
752
753 static void tegra_suspend_wake(void)
754 {
755 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
756         enable_irq(INT_SYS_STATS_MON);
757 #endif
758 }
759
760 static void tegra_pm_set(enum tegra_suspend_mode mode)
761 {
762         u32 reg, boot_flag;
763         unsigned long rate = 32768;
764
765         reg = readl(pmc + PMC_CTRL);
766         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
767         if (pdata->combined_req)
768                 reg &= ~TEGRA_POWER_PWRREQ_OE;
769         else
770                 reg |= TEGRA_POWER_PWRREQ_OE;
771         reg &= ~TEGRA_POWER_EFFECT_LP0;
772
773         switch (mode) {
774         case TEGRA_SUSPEND_LP0:
775 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
776                 rate = clk_get_rate_all_locked(tegra_pclk);
777 #endif
778                 if (pdata->combined_req) {
779                         reg |= TEGRA_POWER_PWRREQ_OE;
780                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
781                 }
782
783                 /*
784                  * LP0 boots through the AVP, which then resumes the AVP to
785                  * the address in scratch 39, and the cpu to the address in
786                  * scratch 41 to tegra_resume
787                  */
788                 writel(0x0, pmc + PMC_SCRATCH39);
789
790                 /* Enable DPD sample to trigger sampling pads data and direction
791                  * in which pad will be driven during lp0 mode*/
792                 writel(0x1, pmc + PMC_DPD_SAMPLE);
793 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
794                 /* this is needed only for T11x, not for other chips */
795                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
796 #endif
797
798                 /* Set warmboot flag */
799                 boot_flag = readl(pmc + PMC_SCRATCH0);
800                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
801
802                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
803
804                 reg |= TEGRA_POWER_EFFECT_LP0;
805                 /* No break here. LP0 code falls through to write SCRATCH41 */
806         case TEGRA_SUSPEND_LP1:
807                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
808                 wmb();
809                 break;
810         case TEGRA_SUSPEND_LP2:
811                 rate = clk_get_rate(tegra_pclk);
812                 break;
813         case TEGRA_SUSPEND_NONE:
814                 return;
815         default:
816                 BUG();
817         }
818
819         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
820
821         pmc_32kwritel(reg, PMC_CTRL);
822 }
823
824 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
825         [TEGRA_SUSPEND_NONE] = "none",
826         [TEGRA_SUSPEND_LP2] = "LP2",
827         [TEGRA_SUSPEND_LP1] = "LP1",
828         [TEGRA_SUSPEND_LP0] = "LP0",
829 };
830
831 static int tegra_suspend_enter(suspend_state_t state)
832 {
833         int ret;
834         ktime_t delta;
835         struct timespec ts_entry, ts_exit;
836
837         if (pdata && pdata->board_suspend)
838                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
839
840         read_persistent_clock(&ts_entry);
841
842         ret = tegra_suspend_dram(current_suspend_mode, 0);
843         if (ret) {
844                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
845                 goto abort_suspend;
846         }
847
848         read_persistent_clock(&ts_exit);
849
850         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
851                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
852
853                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
854                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
855                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
856                 else
857                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
858         }
859
860 abort_suspend:
861         if (pdata && pdata->board_resume)
862                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
863
864         return ret;
865 }
866
867 static void tegra_suspend_check_pwr_stats(void)
868 {
869         /* cpus and l2 are powered off later */
870         unsigned long pwrgate_partid_mask =
871 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
872                 (1 << TEGRA_POWERGATE_HEG)      |
873                 (1 << TEGRA_POWERGATE_SATA)     |
874                 (1 << TEGRA_POWERGATE_3D1)      |
875 #endif
876                 (1 << TEGRA_POWERGATE_3D)       |
877                 (1 << TEGRA_POWERGATE_VENC)     |
878                 (1 << TEGRA_POWERGATE_PCIE)     |
879                 (1 << TEGRA_POWERGATE_VDEC)     |
880                 (1 << TEGRA_POWERGATE_MPE);
881
882         int partid;
883
884         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
885                 if ((1 << partid) & pwrgate_partid_mask)
886                         if (tegra_powergate_is_powered(partid))
887                                 pr_warning("partition %s is left on before suspend\n",
888                                         tegra_powergate_get_name(partid));
889
890         return;
891 }
892
893 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
894 {
895         int err = 0;
896         u32 scratch37 = 0xDEADBEEF;
897         u32 reg;
898
899         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
900                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
901                 err = -ENXIO;
902                 goto fail;
903         }
904
905         if (tegra_is_voice_call_active()) {
906                 /* backup the current value of scratch37 */
907                 scratch37 = readl(pmc + PMC_SCRATCH37);
908
909                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
910                 reg = TEGRA_POWER_LP1_AUDIO;
911                 pmc_32kwritel(reg, PMC_SCRATCH37);
912         }
913
914         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
915                 pr_info("LP0 not used due to unsupported wakeup events\n");
916                 mode = TEGRA_SUSPEND_LP1;
917         }
918
919         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
920                 tegra_suspend_check_pwr_stats();
921
922         tegra_common_suspend();
923
924         tegra_pm_set(mode);
925
926         if (pdata && pdata->board_suspend)
927                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
928
929         local_fiq_disable();
930
931         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
932
933         if (mode == TEGRA_SUSPEND_LP0) {
934 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
935                 reg = readl(pmc + PMC_SCRATCH4);
936                 if (is_lp_cluster())
937                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
938                 else
939                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
940                 pmc_32kwritel(reg, PMC_SCRATCH4);
941 #endif
942                 tegra_tsc_suspend();
943                 tegra_lp0_suspend_mc();
944                 tegra_cpu_reset_handler_save();
945                 tegra_tsc_wait_for_suspend();
946                 tegra_smp_clear_power_mask();
947         }
948         else if (mode == TEGRA_SUSPEND_LP1)
949                 *iram_cpu_lp1_mask = 1;
950
951         suspend_cpu_complex(flags);
952
953 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
954         /* In case of LP0/1, program external power gating accordinly */
955         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
956                 reg = readl(FLOW_CTRL_CPU_CSR(0));
957                 if (is_lp_cluster())
958                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
959                 else
960                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
961                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
962         }
963 #endif
964
965         flush_cache_all();
966         outer_flush_all();
967         outer_disable();
968
969         if (mode == TEGRA_SUSPEND_LP2)
970                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
971         else
972                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
973
974         tegra_init_cache(true);
975
976         if (mode == TEGRA_SUSPEND_LP0) {
977
978                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
979                 * Otherwise it is creating issue in cluster switch after LP0 *
980 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
981                 reg = readl(pmc+PMC_CTRL);
982                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
983                 pmc_32kwritel(reg, PMC_CTRL);
984 #endif
985                 */
986
987                 tegra_tsc_resume();
988                 tegra_cpu_reset_handler_restore();
989                 tegra_lp0_resume_mc();
990                 tegra_tsc_wait_for_resume();
991         } else if (mode == TEGRA_SUSPEND_LP1)
992                 *iram_cpu_lp1_mask = 0;
993
994         /* if scratch37 was clobbered during LP1, restore it */
995         if (scratch37 != 0xDEADBEEF)
996                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
997
998         restore_cpu_complex(flags);
999
1000         /* for platforms where the core & CPU power requests are
1001          * combined as a single request to the PMU, transition out
1002          * of LP0 state by temporarily enabling both requests
1003          */
1004         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1005                 reg = readl(pmc + PMC_CTRL);
1006                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1007                 pmc_32kwritel(reg, PMC_CTRL);
1008                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1009                 pmc_32kwritel(reg, PMC_CTRL);
1010         }
1011
1012         if (pdata && pdata->board_resume)
1013                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1014
1015         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1016
1017         local_fiq_enable();
1018
1019         tegra_common_resume();
1020
1021 fail:
1022         return err;
1023 }
1024
1025 /*
1026  * Function pointers to optional board specific function
1027  */
1028 void (*tegra_deep_sleep)(int);
1029 EXPORT_SYMBOL(tegra_deep_sleep);
1030
1031 static int tegra_suspend_prepare(void)
1032 {
1033         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1034                 tegra_deep_sleep(1);
1035         return 0;
1036 }
1037
1038 static void tegra_suspend_finish(void)
1039 {
1040         if (pdata && pdata->cpu_resume_boost) {
1041                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1042                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1043                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1044         }
1045
1046         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1047                 tegra_deep_sleep(0);
1048 }
1049
1050 static const struct platform_suspend_ops tegra_suspend_ops = {
1051         .valid          = suspend_valid_only_mem,
1052         .prepare        = tegra_suspend_prepare,
1053         .finish         = tegra_suspend_finish,
1054         .prepare_late   = tegra_suspend_prepare_late,
1055         .wake           = tegra_suspend_wake,
1056         .enter          = tegra_suspend_enter,
1057 };
1058
1059 static ssize_t suspend_mode_show(struct kobject *kobj,
1060                                         struct kobj_attribute *attr, char *buf)
1061 {
1062         char *start = buf;
1063         char *end = buf + PAGE_SIZE;
1064
1065         start += scnprintf(start, end - start, "%s ", \
1066                                 tegra_suspend_name[current_suspend_mode]);
1067         start += scnprintf(start, end - start, "\n");
1068
1069         return start - buf;
1070 }
1071
1072 static ssize_t suspend_mode_store(struct kobject *kobj,
1073                                         struct kobj_attribute *attr,
1074                                         const char *buf, size_t n)
1075 {
1076         int len;
1077         const char *name_ptr;
1078         enum tegra_suspend_mode new_mode;
1079
1080         name_ptr = buf;
1081         while (*name_ptr && !isspace(*name_ptr))
1082                 name_ptr++;
1083         len = name_ptr - buf;
1084         if (!len)
1085                 goto bad_name;
1086         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1087         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1088                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1089                 pr_info("Illegal tegra suspend state: %s\n", buf);
1090                 goto bad_name;
1091         }
1092
1093         for (new_mode = TEGRA_SUSPEND_NONE; \
1094                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1095                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1096                         current_suspend_mode = new_mode;
1097                         break;
1098                 }
1099         }
1100
1101 bad_name:
1102         return n;
1103 }
1104
1105 static struct kobj_attribute suspend_mode_attribute =
1106         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1107
1108 static struct kobject *suspend_kobj;
1109
1110 static int tegra_pm_enter_suspend(void)
1111 {
1112         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1113         suspend_cpu_dfll_mode();
1114         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1115                 tegra_lp0_cpu_mode(true);
1116         return 0;
1117 }
1118
1119 static void tegra_pm_enter_resume(void)
1120 {
1121         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1122                 tegra_lp0_cpu_mode(false);
1123         resume_cpu_dfll_mode();
1124         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1125 }
1126
1127 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1128         .suspend = tegra_pm_enter_suspend,
1129         .resume = tegra_pm_enter_resume,
1130 };
1131
1132 static __init int tegra_pm_enter_syscore_init(void)
1133 {
1134         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1135         return 0;
1136 }
1137 subsys_initcall(tegra_pm_enter_syscore_init);
1138 #endif
1139
1140 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1141 {
1142         u32 reg;
1143         u32 mode;
1144
1145 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1146         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1147         BUG_ON(IS_ERR(tegra_dfll));
1148 #endif
1149         tegra_pclk = clk_get_sys(NULL, "pclk");
1150         BUG_ON(IS_ERR(tegra_pclk));
1151         pdata = plat;
1152         (void)reg;
1153         (void)mode;
1154
1155         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1156                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1157
1158 #ifndef CONFIG_PM_SLEEP
1159         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1160                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1161                            "disabling suspend\n", __func__);
1162                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1163         }
1164 #else
1165         if (create_suspend_pgtable() < 0) {
1166                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1167                                 __func__);
1168                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1169                 goto fail;
1170         }
1171
1172         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1173             (tegra_revision == TEGRA_REVISION_A01) &&
1174             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1175                 /* Tegra 3 A01 supports only LP1 */
1176                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1177                            "-- disabling LP0\n", __func__);
1178                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1179         }
1180         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1181                 tegra_lp0_vec_relocate) {
1182                 unsigned char *reloc_lp0;
1183                 unsigned long tmp;
1184                 void __iomem *orig;
1185                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1186                                         GFP_KERNEL);
1187                 WARN_ON(!reloc_lp0);
1188                 if (!reloc_lp0) {
1189                         pr_err("%s: Failed to allocate reloc_lp0\n",
1190                                 __func__);
1191                         goto out;
1192                 }
1193
1194                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1195                 WARN_ON(!orig);
1196                 if (!orig) {
1197                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1198                                 __func__, tegra_lp0_vec_start);
1199                         kfree(reloc_lp0);
1200                         goto out;
1201                 }
1202
1203                 tmp = (unsigned long) reloc_lp0;
1204                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1205                 reloc_lp0 = (unsigned char *)tmp;
1206                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1207                 iounmap(orig);
1208                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1209         }
1210
1211 out:
1212         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1213                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1214                            "provided by bootlader -- disabling LP0\n",
1215                            __func__);
1216                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1217         }
1218
1219         iram_save_size = tegra_iram_end() - tegra_iram_start();
1220
1221         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1222         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1223                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1224                        "-- LP0/LP1 unavailable\n", __func__);
1225                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1226         }
1227
1228 #ifdef CONFIG_TEGRA_LP1_950
1229         if (pdata->lp1_lowvolt_support) {
1230                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1231                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1232                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1233
1234                 lp1_core_lowvolt = 0;
1235                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1236                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1237
1238                 lp1_core_highvolt = 0;
1239                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1240                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1241         }
1242 #endif
1243         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1244         /* Initialize scratch registers used for CPU LP2 synchronization */
1245         writel(0, pmc + PMC_SCRATCH37);
1246         writel(0, pmc + PMC_SCRATCH38);
1247         writel(0, pmc + PMC_SCRATCH39);
1248         writel(0, pmc + PMC_SCRATCH41);
1249
1250         /* Always enable CPU power request; just normal polarity is supported */
1251         reg = readl(pmc + PMC_CTRL);
1252         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1253         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1254         pmc_32kwritel(reg, PMC_CTRL);
1255
1256         /* Configure core power request and system clock control if LP0
1257            is supported */
1258         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1259         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1260
1261         reg = readl(pmc + PMC_CTRL);
1262
1263         if (!pdata->sysclkreq_high)
1264                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1265         else
1266                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1267
1268         if (!pdata->corereq_high)
1269                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1270         else
1271                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1272
1273         /* configure output inverters while the request is tristated */
1274         pmc_32kwritel(reg, PMC_CTRL);
1275
1276         /* now enable requests */
1277         reg |= TEGRA_POWER_SYSCLK_OE;
1278         if (!pdata->combined_req)
1279                 reg |= TEGRA_POWER_PWRREQ_OE;
1280         pmc_32kwritel(reg, PMC_CTRL);
1281
1282         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1283                 tegra_lp0_suspend_init();
1284
1285         suspend_set_ops(&tegra_suspend_ops);
1286
1287         /* Create /sys/power/suspend/type */
1288         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1289         if (suspend_kobj) {
1290                 if (sysfs_create_file(suspend_kobj, \
1291                                                 &suspend_mode_attribute.attr))
1292                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1293                                                                 __func__);
1294         }
1295
1296         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1297         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1298
1299         /* clear io dpd settings before kernel */
1300         tegra_bl_io_dpd_cleanup();
1301
1302 fail:
1303 #endif
1304         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1305                 tegra_pd_in_idle(false);
1306
1307         current_suspend_mode = plat->suspend_mode;
1308 }
1309
1310 unsigned long debug_uart_port_base = 0;
1311 EXPORT_SYMBOL(debug_uart_port_base);
1312
1313 static int tegra_debug_uart_suspend(void)
1314 {
1315         void __iomem *uart;
1316         u32 lcr;
1317
1318         if (!debug_uart_port_base)
1319                 return 0;
1320
1321         uart = IO_ADDRESS(debug_uart_port_base);
1322
1323         lcr = readb(uart + UART_LCR * 4);
1324
1325         tegra_sctx.uart[0] = lcr;
1326         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1327
1328         /* DLAB = 0 */
1329         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1330
1331         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1332
1333         /* DLAB = 1 */
1334         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1335
1336         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1337         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1338
1339         writeb(lcr, uart + UART_LCR * 4);
1340
1341         return 0;
1342 }
1343
1344 static void tegra_debug_uart_resume(void)
1345 {
1346         void __iomem *uart;
1347         u32 lcr;
1348
1349         if (!debug_uart_port_base)
1350                 return;
1351
1352         uart = IO_ADDRESS(debug_uart_port_base);
1353
1354         lcr = tegra_sctx.uart[0];
1355
1356         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1357
1358         /* DLAB = 0 */
1359         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1360
1361         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1362                         uart + UART_FCR * 4);
1363
1364         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1365
1366         /* DLAB = 1 */
1367         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1368
1369         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1370         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1371
1372         writeb(lcr, uart + UART_LCR * 4);
1373 }
1374
1375 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1376         .suspend = tegra_debug_uart_suspend,
1377         .resume = tegra_debug_uart_resume,
1378 };
1379
1380 struct clk *debug_uart_clk = NULL;
1381 EXPORT_SYMBOL(debug_uart_clk);
1382
1383 void tegra_console_uart_suspend(void)
1384 {
1385         if (console_suspend_enabled && debug_uart_clk)
1386                 tegra_clk_disable_unprepare(debug_uart_clk);
1387 }
1388
1389 void tegra_console_uart_resume(void)
1390 {
1391         if (console_suspend_enabled && debug_uart_clk)
1392                 tegra_clk_prepare_enable(debug_uart_clk);
1393 }
1394
1395 static int tegra_debug_uart_syscore_init(void)
1396 {
1397         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1398         return 0;
1399 }
1400 arch_initcall(tegra_debug_uart_syscore_init);