ARM: tegra: t14x: flush complete outer cache
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50 #include <trace/events/nvsecurity.h>
51
52 #include <asm/cacheflush.h>
53 #include <asm/idmap.h>
54 #include <asm/localtimer.h>
55 #include <asm/pgalloc.h>
56 #include <asm/pgtable.h>
57 #include <asm/tlbflush.h>
58 #include <asm/suspend.h>
59 #include <asm/smp_plat.h>
60
61 #include <mach/irqs.h>
62 #include <mach/powergate.h>
63 #include <mach/hardware.h>
64
65 #include "board.h"
66 #include "clock.h"
67 #include "common.h"
68 #include "cpuidle.h"
69 #include "fuse.h"
70 #include "gic.h"
71 #include "iomap.h"
72 #include "pm.h"
73 #include "pm-irq.h"
74 #include "reset.h"
75 #include "sleep.h"
76 #include "timer.h"
77 #include "dvfs.h"
78 #include "cpu-tegra.h"
79
80 #define CREATE_TRACE_POINTS
81 #include <trace/events/nvpower.h>
82
83 struct suspend_context {
84         /*
85          * The next 7 values are referenced by offset in __restart_plls
86          * in headsmp-t2.S, and should not be moved
87          */
88         u32 pllx_misc;
89         u32 pllx_base;
90         u32 pllp_misc;
91         u32 pllp_base;
92         u32 pllp_outa;
93         u32 pllp_outb;
94         u32 pll_timeout;
95
96         u32 cpu_burst;
97         u32 clk_csite_src;
98         u32 cclk_divider;
99
100         u32 mc[3];
101         u8 uart[5];
102
103         struct tegra_twd_context twd;
104 #ifdef CONFIG_ARM_ARCH_TIMER
105         struct arch_timer_context arch_timer;
106 #endif
107 };
108
109 #ifdef CONFIG_PM_SLEEP
110 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
111 static pgd_t *tegra_pgd;
112 static DEFINE_SPINLOCK(tegra_lp2_lock);
113 static cpumask_t tegra_in_lp2;
114 static cpumask_t *iram_cpu_lp2_mask;
115 static unsigned long *iram_cpu_lp1_mask;
116 static u8 *iram_save;
117 static unsigned long iram_save_size;
118 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
119 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
120 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
121 static int tegra_last_pclk;
122 #endif
123
124 struct suspend_context tegra_sctx;
125
126 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
127 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
128 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
129 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
130 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
131 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
132 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
133 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
134 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
135
136 #define PMC_CTRL                0x0
137 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
138 #define PMC_WAKE_MASK           0xc
139 #define PMC_WAKE_LEVEL          0x10
140 #define PMC_DPAD_ORIDE          0x1C
141 #define PMC_WAKE_DELAY          0xe0
142 #define PMC_DPD_SAMPLE          0x20
143 #define PMC_IO_DPD_REQ          0x1B8
144 #define PMC_IO_DPD2_REQ         0x1C0
145
146
147 #define PMC_WAKE_STATUS         0x14
148 #define PMC_SW_WAKE_STATUS      0x18
149 #define PMC_COREPWRGOOD_TIMER   0x3c
150 #define PMC_CPUPWRGOOD_TIMER    0xc8
151 #define PMC_CPUPWROFF_TIMER     0xcc
152 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
153
154 #define PMC_PWRGATE_TOGGLE      0x30
155 #define PWRGATE_TOGGLE_START    (1 << 8)
156 #define UN_PWRGATE_CPU          \
157         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
158
159 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
160 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
161 #endif
162
163 #define CLK_RESET_CCLK_BURST    0x20
164 #define CLK_RESET_CCLK_DIVIDER  0x24
165 #define CLK_RESET_PLLC_BASE     0x80
166 #define CLK_RESET_PLLM_BASE     0x90
167 #define CLK_RESET_PLLX_BASE     0xe0
168 #define CLK_RESET_PLLX_MISC     0xe4
169 #define CLK_RESET_PLLP_BASE     0xa0
170 #define CLK_RESET_PLLP_OUTA     0xa4
171 #define CLK_RESET_PLLP_OUTB     0xa8
172 #define CLK_RESET_PLLP_MISC     0xac
173
174 #define CLK_RESET_SOURCE_CSITE  0x1d4
175
176 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
177 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
178 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
179 #define CLK_RESET_CCLK_IDLE_POLICY         1
180 #define CLK_RESET_CCLK_RUN_POLICY          2
181 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
182 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
183
184 #define EMC_MRW_0               0x0e8
185 #define EMC_MRW_DEV_SELECTN     30
186 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
187
188 #define MC_SECURITY_START       0x6c
189 #define MC_SECURITY_SIZE        0x70
190 #define MC_SECURITY_CFG2        0x7c
191
192 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
193 static struct clk *tegra_dfll;
194 #endif
195 static struct clk *tegra_pclk;
196 static const struct tegra_suspend_platform_data *pdata;
197 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
198
199 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
200 enum tegra_cluster_switch_time_id {
201         tegra_cluster_switch_time_id_start = 0,
202         tegra_cluster_switch_time_id_prolog,
203         tegra_cluster_switch_time_id_switch,
204         tegra_cluster_switch_time_id_epilog,
205         tegra_cluster_switch_time_id_max
206 };
207
208 static unsigned long
209                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
210 #define tegra_cluster_switch_time(flags, id) \
211         do { \
212                 barrier(); \
213                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
214                         void __iomem *timer_us = \
215                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
216                         if (id < tegra_cluster_switch_time_id_max) \
217                                 tegra_cluster_switch_times[id] = \
218                                                         readl(timer_us); \
219                                 wmb(); \
220                 } \
221                 barrier(); \
222         } while(0)
223 #else
224 #define tegra_cluster_switch_time(flags, id) do {} while(0)
225 #endif
226
227 #ifdef CONFIG_PM_SLEEP
228 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
229         [TEGRA_SUSPEND_NONE]    = "none",
230         [TEGRA_SUSPEND_LP2]     = "lp2",
231         [TEGRA_SUSPEND_LP1]     = "lp1",
232         [TEGRA_SUSPEND_LP0]     = "lp0",
233 };
234
235 unsigned long tegra_cpu_power_good_time(void)
236 {
237         if (WARN_ON_ONCE(!pdata))
238                 return 5000;
239
240         return pdata->cpu_timer;
241 }
242
243 unsigned long tegra_cpu_power_off_time(void)
244 {
245         if (WARN_ON_ONCE(!pdata))
246                 return 5000;
247
248         return pdata->cpu_off_timer;
249 }
250
251 unsigned long tegra_cpu_lp2_min_residency(void)
252 {
253         if (WARN_ON_ONCE(!pdata))
254                 return 2000;
255
256         return pdata->cpu_lp2_min_residency;
257 }
258
259 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
260 unsigned long tegra_min_residency_noncpu(void)
261 {
262         return pdata->min_residency_noncpu;
263 }
264
265 unsigned long tegra_min_residency_crail(void)
266 {
267         return pdata->min_residency_crail;
268 }
269 #endif
270
271 static void suspend_cpu_dfll_mode(void)
272 {
273 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
274         /* If DFLL is used as CPU clock source go to open loop mode */
275         if (!is_lp_cluster() && tegra_dfll &&
276             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
277                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
278 #endif
279 }
280
281 static void resume_cpu_dfll_mode(void)
282 {
283 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
284         /* If DFLL is used as CPU clock source restore closed loop mode */
285         if (!is_lp_cluster() && tegra_dfll &&
286             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
287                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
288 #endif
289 }
290
291 /*
292  * create_suspend_pgtable
293  *
294  * Creates a page table with identity mappings of physical memory and IRAM
295  * for use when the MMU is off, in addition to all the regular kernel mappings.
296  */
297 static __init int create_suspend_pgtable(void)
298 {
299         tegra_pgd = pgd_alloc(&init_mm);
300         if (!tegra_pgd)
301                 return -ENOMEM;
302
303         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
304         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
305                 high_memory, 0);
306         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
307                 IO_IRAM_VIRT + SECTION_SIZE, 0);
308
309         /* inner/outer write-back/write-allocate, sharable */
310         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
311
312         return 0;
313 }
314
315 /* ensures that sufficient time is passed for a register write to
316  * serialize into the 32KHz domain */
317 static void pmc_32kwritel(u32 val, unsigned long offs)
318 {
319         writel(val, pmc + offs);
320         udelay(130);
321 }
322
323 static void set_power_timers(unsigned long us_on, unsigned long us_off,
324                              long rate)
325 {
326         static unsigned long last_us_off = 0;
327         unsigned long long ticks;
328         unsigned long long pclk;
329
330         if (WARN_ON_ONCE(rate <= 0))
331                 pclk = 100000000;
332         else
333                 pclk = rate;
334
335         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
336                 ticks = (us_on * pclk) + 999999ull;
337                 do_div(ticks, 1000000);
338                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
339
340                 ticks = (us_off * pclk) + 999999ull;
341                 do_div(ticks, 1000000);
342                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
343                 wmb();
344         }
345         tegra_last_pclk = pclk;
346         last_us_off = us_off;
347 }
348
349 /*
350  * restore_cpu_complex
351  *
352  * restores cpu clock setting, clears flow controller
353  *
354  * Always called on CPU 0.
355  */
356 static void restore_cpu_complex(u32 mode)
357 {
358         int cpu = cpu_logical_map(smp_processor_id());
359         unsigned int reg;
360 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
361         unsigned int policy;
362 #endif
363
364 /*
365  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
366  * or restored by common clock suspend/resume procedures. Hence, we don't
367  * need it here.
368  */
369 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
370         /* Is CPU complex already running on PLLX? */
371         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
372         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
373
374         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
375                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
376         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
377                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
378         else
379                 BUG();
380
381         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
382                 /* restore PLLX settings if CPU is on different PLL */
383                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
384                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
385
386                 /* wait for PLL stabilization if PLLX was enabled */
387                 if (tegra_sctx.pllx_base & (1<<30)) {
388 #if USE_PLL_LOCK_BITS
389                         /* Enable lock detector */
390                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
391                         reg |= 1<<18;
392                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
393                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
394                                  (1<<27)))
395                                 cpu_relax();
396
397                         udelay(PLL_POST_LOCK_DELAY);
398 #else
399                         udelay(300);
400 #endif
401                 }
402         }
403
404         /* Restore original burst policy setting for calls resulting from CPU
405            LP2 in idle or system suspend; keep cluster switch prolog setting
406            intact. */
407         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
408                 writel(tegra_sctx.cclk_divider, clk_rst +
409                        CLK_RESET_CCLK_DIVIDER);
410                 writel(tegra_sctx.cpu_burst, clk_rst +
411                        CLK_RESET_CCLK_BURST);
412         }
413 #endif
414         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
415
416         /* Do not power-gate CPU 0 when flow controlled */
417         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
418         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
419         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
420         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
421         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
422         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
423         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
424
425         /* If an immedidate cluster switch is being perfomed, restore the
426            local timer registers. For calls resulting from CPU LP2 in
427            idle or system suspend, the local timer was shut down and
428            timekeeping switched over to the global system timer. In this
429            case keep local timer disabled, and restore only periodic load. */
430         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
431                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
432 #ifdef CONFIG_ARM_ARCH_TIMER
433                 tegra_sctx.arch_timer.cntp_ctl = 0;
434 #endif
435 #ifdef CONFIG_HAVE_ARM_TWD
436                 tegra_sctx.twd.twd_ctrl = 0;
437 #endif
438         }
439 #ifdef CONFIG_ARM_ARCH_TIMER
440         arch_timer_resume(&tegra_sctx.arch_timer);
441 #endif
442 #ifdef CONFIG_HAVE_ARM_TWD
443         tegra_twd_resume(&tegra_sctx.twd);
444 #endif
445 }
446
447 /*
448  * suspend_cpu_complex
449  *
450  * saves pll state for use by restart_plls, prepares flow controller for
451  * transition to suspend state
452  *
453  * Must always be called on cpu 0.
454  */
455 static void suspend_cpu_complex(u32 mode)
456 {
457         int cpu = cpu_logical_map(smp_processor_id());
458         unsigned int reg;
459         int i;
460
461         BUG_ON(cpu != 0);
462
463         /* switch coresite to clk_m, save off original source */
464         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
465         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
466
467         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
468         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
469         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
470         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
471         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
472         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
473         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
474         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
475
476 #ifdef CONFIG_HAVE_ARM_TWD
477         tegra_twd_suspend(&tegra_sctx.twd);
478 #endif
479 #ifdef CONFIG_ARM_ARCH_TIMER
480         arch_timer_suspend(&tegra_sctx.arch_timer);
481 #endif
482
483         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
484         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
485         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
486         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
487         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
488 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
489         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
490 #else
491         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
492 #endif
493         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
494         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
495
496         for (i = 0; i < num_possible_cpus(); i++) {
497                 if (i == cpu)
498                         continue;
499                 reg = readl(FLOW_CTRL_CPU_CSR(i));
500                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
501                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
502                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
503         }
504
505         tegra_gic_cpu_disable(true);
506 }
507
508 void tegra_clear_cpu_in_pd(int cpu)
509 {
510         spin_lock(&tegra_lp2_lock);
511         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
512         cpumask_clear_cpu(cpu, &tegra_in_lp2);
513
514         /* Update the IRAM copy used by the reset handler. The IRAM copy
515            can't use used directly by cpumask_clear_cpu() because it uses
516            LDREX/STREX which requires the addressed location to be inner
517            cacheable and sharable which IRAM isn't. */
518         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
519         dsb();
520
521         spin_unlock(&tegra_lp2_lock);
522 }
523
524 bool tegra_set_cpu_in_pd(int cpu)
525 {
526         bool last_cpu = false;
527
528         spin_lock(&tegra_lp2_lock);
529         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
530         cpumask_set_cpu(cpu, &tegra_in_lp2);
531
532         /* Update the IRAM copy used by the reset handler. The IRAM copy
533            can't use used directly by cpumask_set_cpu() because it uses
534            LDREX/STREX which requires the addressed location to be inner
535            cacheable and sharable which IRAM isn't. */
536         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
537         dsb();
538
539         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
540                 last_cpu = true;
541 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
542         else if (cpu == 1)
543                 tegra2_cpu_set_resettable_soon();
544 #endif
545
546         spin_unlock(&tegra_lp2_lock);
547         return last_cpu;
548 }
549
550 static void tegra_sleep_core(enum tegra_suspend_mode mode,
551                              unsigned long v2p)
552 {
553 #ifdef CONFIG_TRUSTED_FOUNDATIONS
554         outer_flush_range(__pa(&tegra_resume_timestamps_start),
555                           __pa(&tegra_resume_timestamps_end));
556
557         if (mode == TEGRA_SUSPEND_LP0) {
558                 trace_smc_sleep_core(NVSEC_SMC_START);
559
560                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
561                                   virt_to_phys(tegra_resume));
562         } else {
563                 trace_smc_sleep_core(NVSEC_SMC_START);
564
565                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
566                                   (TEGRA_RESET_HANDLER_BASE +
567                                    tegra_cpu_reset_handler_offset));
568         }
569
570         trace_smc_sleep_core(NVSEC_SMC_DONE);
571 #endif
572 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
573         cpu_suspend(v2p, tegra2_sleep_core_finish);
574 #else
575         cpu_suspend(v2p, tegra3_sleep_core_finish);
576 #endif
577 }
578
579 static inline void tegra_sleep_cpu(unsigned long v2p)
580 {
581 #ifdef CONFIG_TRUSTED_FOUNDATIONS
582         outer_flush_range(__pa(&tegra_resume_timestamps_start),
583                           __pa(&tegra_resume_timestamps_end));
584         trace_smc_sleep_cpu(NVSEC_SMC_START);
585
586         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
587                           (TEGRA_RESET_HANDLER_BASE +
588                            tegra_cpu_reset_handler_offset));
589
590         trace_smc_sleep_cpu(NVSEC_SMC_DONE);
591 #endif
592         cpu_suspend(v2p, tegra_sleep_cpu_finish);
593 }
594
595 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
596                                         unsigned int flags)
597 {
598         u32 reg;
599         unsigned int remain;
600 #ifdef CONFIG_CACHE_L2X0
601         pgd_t *pgd;
602 #endif
603
604         /* Only the last cpu down does the final suspend steps */
605         reg = readl(pmc + PMC_CTRL);
606         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
607         if (pdata->combined_req)
608                 reg &= ~TEGRA_POWER_PWRREQ_OE;
609         else
610                 reg |= TEGRA_POWER_PWRREQ_OE;
611
612         reg &= ~TEGRA_POWER_EFFECT_LP0;
613         writel(reg, pmc + PMC_CTRL);
614
615         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
616
617         /*
618          * We can use clk_get_rate_all_locked() here, because all other cpus
619          * are in LP2 state and irqs are disabled
620          */
621         if (flags & TEGRA_POWER_CLUSTER_MASK) {
622                 trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_START);
623                 set_power_timers(pdata->cpu_timer, 2,
624                         clk_get_rate_all_locked(tegra_pclk));
625                 if (flags & TEGRA_POWER_CLUSTER_G) {
626                         /*
627                          * To reduce the vdd_cpu up latency when LP->G
628                          * transition. Before the transition, enable
629                          * the vdd_cpu rail.
630                          */
631                         if (is_lp_cluster()) {
632 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
633                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
634                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
635                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
636 #else
637                                 writel(UN_PWRGATE_CPU,
638                                        pmc + PMC_PWRGATE_TOGGLE);
639 #endif
640                         }
641                 }
642                 tegra_cluster_switch_prolog(flags);
643         } else {
644                 suspend_cpu_dfll_mode();
645                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
646                         clk_get_rate_all_locked(tegra_pclk));
647 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
648                 reg = readl(FLOW_CTRL_CPU_CSR(0));
649                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
650                 if (is_lp_cluster()) {
651                         /* for LP cluster, there is no option for rail gating */
652                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
653                                                 TEGRA_POWER_CLUSTER_PART_MASK)
654                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
655                         else if (flags)
656                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
657                 }
658                 else {
659                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
660                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
661                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
662                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
663                 }
664                 writel(reg, FLOW_CTRL_CPU_CSR(0));
665 #endif
666         }
667
668         if (sleep_time)
669                 tegra_pd_set_trigger(sleep_time);
670
671         cpu_cluster_pm_enter();
672         suspend_cpu_complex(flags);
673         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
674 #ifdef CONFIG_CACHE_L2X0
675         flush_cache_all();
676 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
677         outer_flush_all();
678 #else
679         /*
680          * No need to flush complete L2. Cleaning kernel and IO mappings
681          * is enough for the LP code sequence that has L2 disabled but
682          * MMU on.
683          */
684         pgd = cpu_get_pgd();
685         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
686                           __pa(pgd + PTRS_PER_PGD));
687 #endif
688         outer_disable();
689 #endif
690         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
691
692 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
693         tegra_init_cache(true);
694 #else
695         tegra_init_cache(false);
696 #endif
697
698 #ifdef CONFIG_TRUSTED_FOUNDATIONS
699 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
700         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
701         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
702 #endif
703 #endif
704
705         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
706         restore_cpu_complex(flags);
707         cpu_cluster_pm_exit();
708
709         remain = tegra_pd_timer_remain();
710         if (sleep_time)
711                 tegra_pd_set_trigger(0);
712
713         if (flags & TEGRA_POWER_CLUSTER_MASK) {
714                 tegra_cluster_switch_epilog(flags);
715                 if (is_idle_task(current))
716                         trace_nvcpu_cluster_rcuidle(NVPOWER_CPU_CLUSTER_DONE);
717                 else
718                         trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_DONE);
719         } else {
720                 resume_cpu_dfll_mode();
721         }
722         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
723
724 #if INSTRUMENT_CLUSTER_SWITCH
725         if (flags & TEGRA_POWER_CLUSTER_MASK) {
726                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
727                         is_lp_cluster() ? "G=>LP" : "LP=>G",
728                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
729                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
730                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
731                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
732                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
733                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
734                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
735                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
736         }
737 #endif
738         return remain;
739 }
740
741 static int tegra_common_suspend(void)
742 {
743         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
744
745         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
746         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
747         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
748
749         /* copy the reset vector and SDRAM shutdown code into IRAM */
750         memcpy(iram_save, iram_code, iram_save_size);
751         memcpy(iram_code, tegra_iram_start(), iram_save_size);
752
753         return 0;
754 }
755
756 static void tegra_common_resume(void)
757 {
758         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
759 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
760         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
761 #endif
762
763         /* Clear DPD sample */
764         writel(0x0, pmc + PMC_DPD_SAMPLE);
765
766         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
767         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
768         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
769 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
770         /* trigger emc mode write */
771         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
772         /* clear scratch registers shared by suspend and the reset pen */
773         writel(0x0, pmc + PMC_SCRATCH39);
774 #endif
775         writel(0x0, pmc + PMC_SCRATCH41);
776
777         /* restore IRAM */
778         memcpy(iram_code, iram_save, iram_save_size);
779 }
780
781 static int tegra_suspend_prepare_late(void)
782 {
783 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
784         disable_irq(INT_SYS_STATS_MON);
785 #endif
786         return 0;
787 }
788
789 static void tegra_suspend_wake(void)
790 {
791 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
792         enable_irq(INT_SYS_STATS_MON);
793 #endif
794 }
795
796 static void tegra_pm_set(enum tegra_suspend_mode mode)
797 {
798         u32 reg, boot_flag;
799         unsigned long rate = 32768;
800
801         reg = readl(pmc + PMC_CTRL);
802         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
803         if (pdata->combined_req)
804                 reg &= ~TEGRA_POWER_PWRREQ_OE;
805         else
806                 reg |= TEGRA_POWER_PWRREQ_OE;
807         reg &= ~TEGRA_POWER_EFFECT_LP0;
808
809         switch (mode) {
810         case TEGRA_SUSPEND_LP0:
811 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
812                 rate = clk_get_rate_all_locked(tegra_pclk);
813 #endif
814                 if (pdata->combined_req) {
815                         reg |= TEGRA_POWER_PWRREQ_OE;
816                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
817                 }
818
819 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
820                 /*
821                  * LP0 boots through the AVP, which then resumes the AVP to
822                  * the address in scratch 39, and the cpu to the address in
823                  * scratch 41 to tegra_resume
824                  */
825                 writel(0x0, pmc + PMC_SCRATCH39);
826 #endif
827
828                 /* Enable DPD sample to trigger sampling pads data and direction
829                  * in which pad will be driven during lp0 mode*/
830                 writel(0x1, pmc + PMC_DPD_SAMPLE);
831 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
832                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
833                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
834 #endif
835
836 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
837                 /* this is needed only for T11x, not for other chips */
838                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
839 #endif
840
841                 /* Set warmboot flag */
842                 boot_flag = readl(pmc + PMC_SCRATCH0);
843                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
844
845                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
846
847                 reg |= TEGRA_POWER_EFFECT_LP0;
848                 /* No break here. LP0 code falls through to write SCRATCH41 */
849         case TEGRA_SUSPEND_LP1:
850                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
851                 wmb();
852                 break;
853         case TEGRA_SUSPEND_LP2:
854                 rate = clk_get_rate(tegra_pclk);
855                 break;
856         case TEGRA_SUSPEND_NONE:
857                 return;
858         default:
859                 BUG();
860         }
861
862         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
863
864         pmc_32kwritel(reg, PMC_CTRL);
865 }
866
867 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
868         [TEGRA_SUSPEND_NONE] = "none",
869         [TEGRA_SUSPEND_LP2] = "LP2",
870         [TEGRA_SUSPEND_LP1] = "LP1",
871         [TEGRA_SUSPEND_LP0] = "LP0",
872 };
873
874 static int tegra_suspend_enter(suspend_state_t state)
875 {
876         int ret;
877         ktime_t delta;
878         struct timespec ts_entry, ts_exit;
879
880         if (pdata && pdata->board_suspend)
881                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
882
883         read_persistent_clock(&ts_entry);
884
885         ret = tegra_suspend_dram(current_suspend_mode, 0);
886         if (ret) {
887                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
888                 goto abort_suspend;
889         }
890
891         read_persistent_clock(&ts_exit);
892
893         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
894                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
895
896                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
897                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
898                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
899                 else
900                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
901         }
902
903 abort_suspend:
904         if (pdata && pdata->board_resume)
905                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
906
907         return ret;
908 }
909
910 static void tegra_suspend_check_pwr_stats(void)
911 {
912         /* cpus and l2 are powered off later */
913         unsigned long pwrgate_partid_mask =
914 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
915                 (1 << TEGRA_POWERGATE_HEG)      |
916                 (1 << TEGRA_POWERGATE_SATA)     |
917                 (1 << TEGRA_POWERGATE_3D1)      |
918 #endif
919                 (1 << TEGRA_POWERGATE_3D)       |
920                 (1 << TEGRA_POWERGATE_VENC)     |
921                 (1 << TEGRA_POWERGATE_PCIE)     |
922                 (1 << TEGRA_POWERGATE_VDEC)     |
923                 (1 << TEGRA_POWERGATE_MPE);
924
925         int partid;
926
927         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
928                 if ((1 << partid) & pwrgate_partid_mask)
929                         if (tegra_powergate_is_powered(partid))
930                                 pr_warning("partition %s is left on before suspend\n",
931                                         tegra_powergate_get_name(partid));
932
933         return;
934 }
935
936 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
937 {
938         int err = 0;
939         u32 scratch37 = 0xDEADBEEF;
940         u32 reg;
941
942         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
943                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
944                 err = -ENXIO;
945                 goto fail;
946         }
947
948         if (tegra_is_voice_call_active()) {
949                 /* backup the current value of scratch37 */
950                 scratch37 = readl(pmc + PMC_SCRATCH37);
951
952                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
953                 reg = TEGRA_POWER_LP1_AUDIO;
954                 pmc_32kwritel(reg, PMC_SCRATCH37);
955         }
956
957         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
958                 pr_info("LP0 not used due to unsupported wakeup events\n");
959                 mode = TEGRA_SUSPEND_LP1;
960         }
961
962         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
963                 tegra_suspend_check_pwr_stats();
964
965         tegra_common_suspend();
966
967         tegra_pm_set(mode);
968
969         if (pdata && pdata->board_suspend)
970                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
971
972         local_fiq_disable();
973
974         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
975
976         if (mode == TEGRA_SUSPEND_LP0) {
977 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
978                 reg = readl(pmc + PMC_SCRATCH4);
979                 if (is_lp_cluster())
980                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
981                 else
982                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
983                 pmc_32kwritel(reg, PMC_SCRATCH4);
984 #endif
985                 tegra_tsc_suspend();
986                 tegra_lp0_suspend_mc();
987                 tegra_cpu_reset_handler_save();
988                 tegra_tsc_wait_for_suspend();
989 #ifndef CONFIG_TEGRA_SIMULATION_PLATFORM
990                 tegra_smp_clear_power_mask();
991 #endif
992         }
993         else if (mode == TEGRA_SUSPEND_LP1)
994                 *iram_cpu_lp1_mask = 1;
995
996         suspend_cpu_complex(flags);
997
998 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
999         /* In case of LP0/1, program external power gating accordinly */
1000         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
1001                 reg = readl(FLOW_CTRL_CPU_CSR(0));
1002                 if (is_lp_cluster())
1003                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
1004                 else
1005                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
1006                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
1007         }
1008 #endif
1009
1010         flush_cache_all();
1011         outer_flush_all();
1012         outer_disable();
1013
1014         if (mode == TEGRA_SUSPEND_LP2)
1015                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
1016         else
1017                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
1018
1019         tegra_init_cache(true);
1020
1021 #ifdef CONFIG_TRUSTED_FOUNDATIONS
1022 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
1023         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
1024         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
1025 #endif
1026
1027         if (mode == TEGRA_SUSPEND_LP0) {
1028                 trace_secureos_init(tegra_resume_entry_time,
1029                         NVSEC_SUSPEND_EXIT_DONE);
1030         }
1031 #endif
1032
1033         if (mode == TEGRA_SUSPEND_LP0) {
1034
1035                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
1036                 * Otherwise it is creating issue in cluster switch after LP0 *
1037 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1038                 reg = readl(pmc+PMC_CTRL);
1039                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
1040                 pmc_32kwritel(reg, PMC_CTRL);
1041 #endif
1042                 */
1043
1044                 tegra_tsc_resume();
1045                 tegra_cpu_reset_handler_restore();
1046                 tegra_lp0_resume_mc();
1047                 tegra_tsc_wait_for_resume();
1048         } else if (mode == TEGRA_SUSPEND_LP1)
1049                 *iram_cpu_lp1_mask = 0;
1050
1051         /* if scratch37 was clobbered during LP1, restore it */
1052         if (scratch37 != 0xDEADBEEF)
1053                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1054
1055         restore_cpu_complex(flags);
1056
1057         /* for platforms where the core & CPU power requests are
1058          * combined as a single request to the PMU, transition out
1059          * of LP0 state by temporarily enabling both requests
1060          */
1061         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1062                 reg = readl(pmc + PMC_CTRL);
1063                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1064                 pmc_32kwritel(reg, PMC_CTRL);
1065                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1066                 pmc_32kwritel(reg, PMC_CTRL);
1067         }
1068
1069         if (pdata && pdata->board_resume)
1070                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1071
1072         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1073
1074         local_fiq_enable();
1075
1076         tegra_common_resume();
1077
1078 fail:
1079         return err;
1080 }
1081
1082 /*
1083  * Function pointers to optional board specific function
1084  */
1085 void (*tegra_deep_sleep)(int);
1086 EXPORT_SYMBOL(tegra_deep_sleep);
1087
1088 static int tegra_suspend_prepare(void)
1089 {
1090         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1091                 tegra_deep_sleep(1);
1092         return 0;
1093 }
1094
1095 static void tegra_suspend_finish(void)
1096 {
1097         if (pdata && pdata->cpu_resume_boost) {
1098                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1099                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1100                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1101         }
1102
1103         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1104                 tegra_deep_sleep(0);
1105 }
1106
1107 static const struct platform_suspend_ops tegra_suspend_ops = {
1108         .valid          = suspend_valid_only_mem,
1109         .prepare        = tegra_suspend_prepare,
1110         .finish         = tegra_suspend_finish,
1111         .prepare_late   = tegra_suspend_prepare_late,
1112         .wake           = tegra_suspend_wake,
1113         .enter          = tegra_suspend_enter,
1114 };
1115
1116 static ssize_t suspend_mode_show(struct kobject *kobj,
1117                                         struct kobj_attribute *attr, char *buf)
1118 {
1119         char *start = buf;
1120         char *end = buf + PAGE_SIZE;
1121
1122         start += scnprintf(start, end - start, "%s ", \
1123                                 tegra_suspend_name[current_suspend_mode]);
1124         start += scnprintf(start, end - start, "\n");
1125
1126         return start - buf;
1127 }
1128
1129 static ssize_t suspend_mode_store(struct kobject *kobj,
1130                                         struct kobj_attribute *attr,
1131                                         const char *buf, size_t n)
1132 {
1133         int len;
1134         const char *name_ptr;
1135         enum tegra_suspend_mode new_mode;
1136
1137         name_ptr = buf;
1138         while (*name_ptr && !isspace(*name_ptr))
1139                 name_ptr++;
1140         len = name_ptr - buf;
1141         if (!len)
1142                 goto bad_name;
1143         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1144         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1145                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1146                 pr_info("Illegal tegra suspend state: %s\n", buf);
1147                 goto bad_name;
1148         }
1149
1150         for (new_mode = TEGRA_SUSPEND_NONE; \
1151                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1152                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1153                         current_suspend_mode = new_mode;
1154                         break;
1155                 }
1156         }
1157
1158 bad_name:
1159         return n;
1160 }
1161
1162 static struct kobj_attribute suspend_mode_attribute =
1163         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1164
1165 static struct kobject *suspend_kobj;
1166
1167 static int tegra_pm_enter_suspend(void)
1168 {
1169         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1170         suspend_cpu_dfll_mode();
1171         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1172                 tegra_lp0_cpu_mode(true);
1173         return 0;
1174 }
1175
1176 static void tegra_pm_enter_resume(void)
1177 {
1178         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1179                 tegra_lp0_cpu_mode(false);
1180         resume_cpu_dfll_mode();
1181         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1182 }
1183
1184 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1185         .suspend = tegra_pm_enter_suspend,
1186         .resume = tegra_pm_enter_resume,
1187 };
1188
1189 static __init int tegra_pm_enter_syscore_init(void)
1190 {
1191         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1192         return 0;
1193 }
1194 subsys_initcall(tegra_pm_enter_syscore_init);
1195 #endif
1196
1197 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1198 {
1199         u32 reg;
1200         u32 mode;
1201
1202 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1203         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1204         BUG_ON(IS_ERR(tegra_dfll));
1205 #endif
1206         tegra_pclk = clk_get_sys(NULL, "pclk");
1207         BUG_ON(IS_ERR(tegra_pclk));
1208         pdata = plat;
1209         (void)reg;
1210         (void)mode;
1211
1212         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1213                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1214
1215 #ifndef CONFIG_PM_SLEEP
1216         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1217                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1218                            "disabling suspend\n", __func__);
1219                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1220         }
1221 #else
1222         if (create_suspend_pgtable() < 0) {
1223                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1224                                 __func__);
1225                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1226                 goto fail;
1227         }
1228
1229         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1230             (tegra_revision == TEGRA_REVISION_A01) &&
1231             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1232                 /* Tegra 3 A01 supports only LP1 */
1233                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1234                            "-- disabling LP0\n", __func__);
1235                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1236         }
1237         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1238                 tegra_lp0_vec_relocate) {
1239                 unsigned char *reloc_lp0;
1240                 unsigned long tmp;
1241                 void __iomem *orig;
1242                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1243                                         GFP_KERNEL);
1244                 WARN_ON(!reloc_lp0);
1245                 if (!reloc_lp0) {
1246                         pr_err("%s: Failed to allocate reloc_lp0\n",
1247                                 __func__);
1248                         goto out;
1249                 }
1250
1251                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1252                 WARN_ON(!orig);
1253                 if (!orig) {
1254                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1255                                 __func__, tegra_lp0_vec_start);
1256                         kfree(reloc_lp0);
1257                         goto out;
1258                 }
1259
1260                 tmp = (unsigned long) reloc_lp0;
1261                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1262                 reloc_lp0 = (unsigned char *)tmp;
1263                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1264                 iounmap(orig);
1265                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1266         }
1267
1268 out:
1269         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1270                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1271                            "provided by bootlader -- disabling LP0\n",
1272                            __func__);
1273                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1274         }
1275
1276         iram_save_size = tegra_iram_end() - tegra_iram_start();
1277
1278         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1279         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1280                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1281                        "-- LP0/LP1 unavailable\n", __func__);
1282                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1283         }
1284
1285 #ifdef CONFIG_TEGRA_LP1_950
1286         if (pdata->lp1_lowvolt_support) {
1287                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1288                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1289                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1290
1291                 lp1_core_lowvolt = 0;
1292                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1293                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1294
1295                 lp1_core_highvolt = 0;
1296                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1297                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1298         }
1299 #endif
1300         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1301         /* Initialize scratch registers used for CPU LP2 synchronization */
1302         writel(0, pmc + PMC_SCRATCH37);
1303         writel(0, pmc + PMC_SCRATCH38);
1304 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1305         writel(0, pmc + PMC_SCRATCH39);
1306 #endif
1307         writel(0, pmc + PMC_SCRATCH41);
1308
1309         /* Always enable CPU power request; just normal polarity is supported */
1310         reg = readl(pmc + PMC_CTRL);
1311         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1312         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1313         pmc_32kwritel(reg, PMC_CTRL);
1314
1315         /* Configure core power request and system clock control if LP0
1316            is supported */
1317         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1318         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1319
1320         reg = readl(pmc + PMC_CTRL);
1321
1322         if (!pdata->sysclkreq_high)
1323                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1324         else
1325                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1326
1327         if (!pdata->corereq_high)
1328                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1329         else
1330                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1331
1332         /* configure output inverters while the request is tristated */
1333         pmc_32kwritel(reg, PMC_CTRL);
1334
1335         /* now enable requests */
1336         reg |= TEGRA_POWER_SYSCLK_OE;
1337         if (!pdata->combined_req)
1338                 reg |= TEGRA_POWER_PWRREQ_OE;
1339         pmc_32kwritel(reg, PMC_CTRL);
1340
1341         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1342                 tegra_lp0_suspend_init();
1343
1344         suspend_set_ops(&tegra_suspend_ops);
1345
1346         /* Create /sys/power/suspend/type */
1347         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1348         if (suspend_kobj) {
1349                 if (sysfs_create_file(suspend_kobj, \
1350                                                 &suspend_mode_attribute.attr))
1351                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1352                                                                 __func__);
1353         }
1354
1355         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1356         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1357
1358         /* clear io dpd settings before kernel */
1359         tegra_bl_io_dpd_cleanup();
1360
1361 fail:
1362 #endif
1363         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1364                 tegra_pd_in_idle(false);
1365
1366         current_suspend_mode = plat->suspend_mode;
1367 }
1368
1369 unsigned long debug_uart_port_base = 0;
1370 EXPORT_SYMBOL(debug_uart_port_base);
1371
1372 static int tegra_debug_uart_suspend(void)
1373 {
1374         void __iomem *uart;
1375         u32 lcr;
1376
1377         if (!debug_uart_port_base)
1378                 return 0;
1379
1380         uart = IO_ADDRESS(debug_uart_port_base);
1381
1382         lcr = readb(uart + UART_LCR * 4);
1383
1384         tegra_sctx.uart[0] = lcr;
1385         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1386
1387         /* DLAB = 0 */
1388         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1389
1390         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1391
1392         /* DLAB = 1 */
1393         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1394
1395         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1396         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1397
1398         writeb(lcr, uart + UART_LCR * 4);
1399
1400         return 0;
1401 }
1402
1403 static void tegra_debug_uart_resume(void)
1404 {
1405         void __iomem *uart;
1406         u32 lcr;
1407
1408         if (!debug_uart_port_base)
1409                 return;
1410
1411         uart = IO_ADDRESS(debug_uart_port_base);
1412
1413         lcr = tegra_sctx.uart[0];
1414
1415         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1416
1417         /* DLAB = 0 */
1418         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1419
1420         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1421                         uart + UART_FCR * 4);
1422
1423         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1424
1425         /* DLAB = 1 */
1426         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1427
1428         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1429         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1430
1431         writeb(lcr, uart + UART_LCR * 4);
1432 }
1433
1434 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1435         .suspend = tegra_debug_uart_suspend,
1436         .resume = tegra_debug_uart_resume,
1437 };
1438
1439 struct clk *debug_uart_clk = NULL;
1440 EXPORT_SYMBOL(debug_uart_clk);
1441
1442 void tegra_console_uart_suspend(void)
1443 {
1444         if (console_suspend_enabled && debug_uart_clk)
1445                 tegra_clk_disable_unprepare(debug_uart_clk);
1446 }
1447
1448 void tegra_console_uart_resume(void)
1449 {
1450         if (console_suspend_enabled && debug_uart_clk)
1451                 tegra_clk_prepare_enable(debug_uart_clk);
1452 }
1453
1454 static int tegra_debug_uart_syscore_init(void)
1455 {
1456         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1457         return 0;
1458 }
1459 arch_initcall(tegra_debug_uart_syscore_init);