ARM: Tegra: Add CONFIG_TEGRA_USE_SECURE_KERNEL
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2013, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50 #include <trace/events/nvsecurity.h>
51
52 #include <asm/cacheflush.h>
53 #include <asm/idmap.h>
54 #include <asm/localtimer.h>
55 #include <asm/pgalloc.h>
56 #include <asm/pgtable.h>
57 #include <asm/tlbflush.h>
58 #include <asm/suspend.h>
59 #include <asm/smp_plat.h>
60
61 #include <mach/irqs.h>
62 #include <mach/powergate.h>
63 #include <mach/hardware.h>
64
65 #include "board.h"
66 #include "clock.h"
67 #include "common.h"
68 #include "cpuidle.h"
69 #include "fuse.h"
70 #include "gic.h"
71 #include "iomap.h"
72 #include "pm.h"
73 #include "pm-irq.h"
74 #include "reset.h"
75 #include "sleep.h"
76 #include "timer.h"
77 #include "dvfs.h"
78 #include "cpu-tegra.h"
79
80 #define CREATE_TRACE_POINTS
81 #include <trace/events/nvpower.h>
82
83 struct suspend_context {
84         /*
85          * The next 7 values are referenced by offset in __restart_plls
86          * in headsmp-t2.S, and should not be moved
87          */
88         u32 pllx_misc;
89         u32 pllx_base;
90         u32 pllp_misc;
91         u32 pllp_base;
92         u32 pllp_outa;
93         u32 pllp_outb;
94         u32 pll_timeout;
95
96         u32 cpu_burst;
97         u32 clk_csite_src;
98         u32 cclk_divider;
99
100         u32 mc[3];
101         u8 uart[5];
102
103         struct tegra_twd_context twd;
104 };
105
106 #ifdef CONFIG_PM_SLEEP
107 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
108 static pgd_t *tegra_pgd;
109 static DEFINE_SPINLOCK(tegra_lp2_lock);
110 static cpumask_t tegra_in_lp2;
111 static cpumask_t *iram_cpu_lp2_mask;
112 static unsigned long *iram_cpu_lp1_mask;
113 static unsigned long *iram_mc_clk_mask;
114 static u8 *iram_save;
115 static unsigned long iram_save_size;
116 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
117 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
118 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
119 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
120 static void __iomem *tert_ictlr = IO_ADDRESS(TEGRA_TERTIARY_ICTLR_BASE);
121 #endif
122 static int tegra_last_pclk;
123 #endif
124
125 struct suspend_context tegra_sctx;
126
127 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
128 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
129 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
130 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
131 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
132 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
133 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
134 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
135 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
136
137 #define TEGRA_DPAD_ORIDE_SYS_CLK_REQ    (1 << 21)
138
139 #define PMC_CTRL                0x0
140 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
141 #define PMC_WAKE_MASK           0xc
142 #define PMC_WAKE_LEVEL          0x10
143 #define PMC_DPAD_ORIDE          0x1C
144 #define PMC_WAKE_DELAY          0xe0
145 #define PMC_DPD_SAMPLE          0x20
146 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
147 #define PMC_DPD_ENABLE          0x24
148 #endif
149 #define PMC_IO_DPD_REQ          0x1B8
150 #define PMC_IO_DPD2_REQ         0x1C0
151
152
153 #define PMC_WAKE_STATUS         0x14
154 #define PMC_SW_WAKE_STATUS      0x18
155 #define PMC_COREPWRGOOD_TIMER   0x3c
156 #define PMC_CPUPWRGOOD_TIMER    0xc8
157 #define PMC_CPUPWROFF_TIMER     0xcc
158 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
159
160 #define PMC_PWRGATE_TOGGLE      0x30
161 #define PWRGATE_TOGGLE_START    (1 << 8)
162 #define UN_PWRGATE_CPU          \
163         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
164
165 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
166 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
167 #endif
168
169 #define CLK_RESET_CCLK_BURST    0x20
170 #define CLK_RESET_CCLK_DIVIDER  0x24
171 #define CLK_RESET_PLLC_BASE     0x80
172 #define CLK_RESET_PLLM_BASE     0x90
173 #define CLK_RESET_PLLX_BASE     0xe0
174 #define CLK_RESET_PLLX_MISC     0xe4
175 #define CLK_RESET_PLLP_BASE     0xa0
176 #define CLK_RESET_PLLP_OUTA     0xa4
177 #define CLK_RESET_PLLP_OUTB     0xa8
178 #define CLK_RESET_PLLP_MISC     0xac
179
180 #define CLK_RESET_SOURCE_CSITE  0x1d4
181
182 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
183 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
184 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
185 #define CLK_RESET_CCLK_IDLE_POLICY         1
186 #define CLK_RESET_CCLK_RUN_POLICY          2
187 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
188 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
189
190 #define EMC_MRW_0               0x0e8
191 #define EMC_MRW_DEV_SELECTN     30
192 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
193
194 #define MC_SECURITY_START       0x6c
195 #define MC_SECURITY_SIZE        0x70
196 #define MC_SECURITY_CFG2        0x7c
197
198 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
199 static struct clk *tegra_dfll;
200 #endif
201 static struct clk *tegra_pclk;
202 static const struct tegra_suspend_platform_data *pdata;
203 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
204
205 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
206 enum tegra_cluster_switch_time_id {
207         tegra_cluster_switch_time_id_start = 0,
208         tegra_cluster_switch_time_id_prolog,
209         tegra_cluster_switch_time_id_switch,
210         tegra_cluster_switch_time_id_epilog,
211         tegra_cluster_switch_time_id_max
212 };
213
214 static unsigned long
215                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
216 #define tegra_cluster_switch_time(flags, id) \
217         do { \
218                 barrier(); \
219                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
220                         void __iomem *timer_us = \
221                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
222                         if (id < tegra_cluster_switch_time_id_max) \
223                                 tegra_cluster_switch_times[id] = \
224                                                         readl(timer_us); \
225                                 wmb(); \
226                 } \
227                 barrier(); \
228         } while(0)
229 #else
230 #define tegra_cluster_switch_time(flags, id) do {} while(0)
231 #endif
232
233 #ifdef CONFIG_PM_SLEEP
234 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
235         [TEGRA_SUSPEND_NONE]    = "none",
236         [TEGRA_SUSPEND_LP2]     = "lp2",
237         [TEGRA_SUSPEND_LP1]     = "lp1",
238         [TEGRA_SUSPEND_LP0]     = "lp0",
239 };
240
241 unsigned long tegra_cpu_power_good_time(void)
242 {
243         if (WARN_ON_ONCE(!pdata))
244                 return 5000;
245
246         return pdata->cpu_timer;
247 }
248
249 unsigned long tegra_cpu_power_off_time(void)
250 {
251         if (WARN_ON_ONCE(!pdata))
252                 return 5000;
253
254         return pdata->cpu_off_timer;
255 }
256
257 unsigned long tegra_cpu_lp2_min_residency(void)
258 {
259         if (WARN_ON_ONCE(!pdata))
260                 return 2000;
261
262         return pdata->cpu_lp2_min_residency;
263 }
264
265 unsigned long tegra_mc_clk_stop_min_residency(void)
266 {
267         return 20000;
268 }
269
270 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
271 #define TEGRA_MIN_RESIDENCY_VMIN_FMIN   2000
272 #define TEGRA_MIN_RESIDENCY_NCPU_SLOW   2000
273 #define TEGRA_MIN_RESIDENCY_NCPU_FAST   13000
274 #define TEGRA_MIN_RESIDENCY_CRAIL       20000
275
276 unsigned long tegra_min_residency_vmin_fmin(void)
277 {
278         return pdata && pdata->min_residency_vmin_fmin
279                         ? pdata->min_residency_vmin_fmin
280                         : TEGRA_MIN_RESIDENCY_VMIN_FMIN;
281 }
282
283 unsigned long tegra_min_residency_ncpu()
284 {
285         if (is_lp_cluster()) {
286                 return pdata && pdata->min_residency_ncpu_slow
287                         ? pdata->min_residency_ncpu_slow
288                         : TEGRA_MIN_RESIDENCY_NCPU_SLOW;
289         } else
290                 return pdata && pdata->min_residency_ncpu_fast
291                         ? pdata->min_residency_ncpu_fast
292                         : TEGRA_MIN_RESIDENCY_NCPU_FAST;
293 }
294
295 unsigned long tegra_min_residency_crail(void)
296 {
297         return pdata && pdata->min_residency_crail
298                         ? pdata->min_residency_crail
299                         : TEGRA_MIN_RESIDENCY_CRAIL;
300 }
301 #endif
302
303 static void suspend_cpu_dfll_mode(void)
304 {
305 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
306         /* If DFLL is used as CPU clock source go to open loop mode */
307         if (!is_lp_cluster() && tegra_dfll &&
308             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
309                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
310 #endif
311 }
312
313 static void resume_cpu_dfll_mode(void)
314 {
315 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
316         /* If DFLL is used as CPU clock source restore closed loop mode */
317         if (!is_lp_cluster() && tegra_dfll &&
318             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
319                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
320 #endif
321 }
322
323 /*
324  * create_suspend_pgtable
325  *
326  * Creates a page table with identity mappings of physical memory and IRAM
327  * for use when the MMU is off, in addition to all the regular kernel mappings.
328  */
329 static __init int create_suspend_pgtable(void)
330 {
331         tegra_pgd = pgd_alloc(&init_mm);
332         if (!tegra_pgd)
333                 return -ENOMEM;
334
335         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
336         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
337                 high_memory, 0);
338         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
339                 IO_IRAM_VIRT + SECTION_SIZE, 0);
340
341         /* inner/outer write-back/write-allocate, sharable */
342         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
343
344         return 0;
345 }
346
347 /* ensures that sufficient time is passed for a register write to
348  * serialize into the 32KHz domain */
349 static void pmc_32kwritel(u32 val, unsigned long offs)
350 {
351         writel(val, pmc + offs);
352         udelay(130);
353 }
354
355 static void set_power_timers(unsigned long us_on, unsigned long us_off,
356                              long rate)
357 {
358         static unsigned long last_us_off = 0;
359         unsigned long long ticks;
360         unsigned long long pclk;
361
362         if (WARN_ON_ONCE(rate <= 0))
363                 pclk = 100000000;
364         else
365                 pclk = rate;
366
367         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
368                 ticks = (us_on * pclk) + 999999ull;
369                 do_div(ticks, 1000000);
370                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
371
372                 ticks = (us_off * pclk) + 999999ull;
373                 do_div(ticks, 1000000);
374                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
375                 wmb();
376         }
377         tegra_last_pclk = pclk;
378         last_us_off = us_off;
379 }
380
381 /*
382  * restore_cpu_complex
383  *
384  * restores cpu clock setting, clears flow controller
385  *
386  * Always called on CPU 0.
387  */
388 static void restore_cpu_complex(u32 mode)
389 {
390         int cpu = cpu_logical_map(smp_processor_id());
391         unsigned int reg;
392 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
393         unsigned int policy;
394 #endif
395
396 /*
397  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
398  * or restored by common clock suspend/resume procedures. Hence, we don't
399  * need it here.
400  */
401 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
402         /* Is CPU complex already running on PLLX? */
403         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
404         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
405
406         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
407                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
408         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
409                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
410         else
411                 BUG();
412
413         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
414                 /* restore PLLX settings if CPU is on different PLL */
415                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
416                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
417
418                 /* wait for PLL stabilization if PLLX was enabled */
419                 if (tegra_sctx.pllx_base & (1<<30)) {
420 #if USE_PLL_LOCK_BITS
421                         /* Enable lock detector */
422                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
423                         reg |= 1<<18;
424                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
425                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
426                                  (1<<27)))
427                                 cpu_relax();
428
429                         udelay(PLL_POST_LOCK_DELAY);
430 #else
431                         udelay(300);
432 #endif
433                 }
434         }
435
436         /* Restore original burst policy setting for calls resulting from CPU
437            LP2 in idle or system suspend; keep cluster switch prolog setting
438            intact. */
439         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
440                 writel(tegra_sctx.cclk_divider, clk_rst +
441                        CLK_RESET_CCLK_DIVIDER);
442                 writel(tegra_sctx.cpu_burst, clk_rst +
443                        CLK_RESET_CCLK_BURST);
444         }
445 #endif
446         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
447
448         /* Do not power-gate CPU 0 when flow controlled */
449         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
450         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
451         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
452         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
453         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
454         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
455         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
456
457         /* If an immedidate cluster switch is being perfomed, restore the
458            local timer registers. For calls resulting from CPU LP2 in
459            idle or system suspend, the local timer was shut down and
460            timekeeping switched over to the global system timer. In this
461            case keep local timer disabled, and restore only periodic load. */
462 #ifdef CONFIG_HAVE_ARM_TWD
463         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
464                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
465                 tegra_sctx.twd.twd_ctrl = 0;
466         }
467         tegra_twd_resume(&tegra_sctx.twd);
468 #endif
469 }
470
471 /*
472  * suspend_cpu_complex
473  *
474  * saves pll state for use by restart_plls, prepares flow controller for
475  * transition to suspend state
476  *
477  * Must always be called on cpu 0.
478  */
479 static void suspend_cpu_complex(u32 mode)
480 {
481         int cpu = cpu_logical_map(smp_processor_id());
482         unsigned int reg;
483         int i;
484
485         BUG_ON(cpu != 0);
486
487         /* switch coresite to clk_m, save off original source */
488         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
489         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
490
491         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
492         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
493         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
494         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
495         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
496         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
497         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
498         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
499
500 #ifdef CONFIG_HAVE_ARM_TWD
501         tegra_twd_suspend(&tegra_sctx.twd);
502 #endif
503
504         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
505         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
506         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
507         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
508         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
509 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
510         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
511 #else
512         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
513 #endif
514         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
515         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
516
517         for (i = 0; i < num_possible_cpus(); i++) {
518                 if (i == cpu)
519                         continue;
520                 reg = readl(FLOW_CTRL_CPU_CSR(i));
521                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
522                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
523                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
524         }
525
526         tegra_gic_cpu_disable(true);
527 }
528
529 void tegra_clear_cpu_in_pd(int cpu)
530 {
531         spin_lock(&tegra_lp2_lock);
532         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
533         cpumask_clear_cpu(cpu, &tegra_in_lp2);
534
535         /* Update the IRAM copy used by the reset handler. The IRAM copy
536            can't use used directly by cpumask_clear_cpu() because it uses
537            LDREX/STREX which requires the addressed location to be inner
538            cacheable and sharable which IRAM isn't. */
539         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
540         dsb();
541
542         spin_unlock(&tegra_lp2_lock);
543 }
544
545 bool tegra_set_cpu_in_pd(int cpu)
546 {
547         bool last_cpu = false;
548
549         spin_lock(&tegra_lp2_lock);
550         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
551         cpumask_set_cpu(cpu, &tegra_in_lp2);
552
553         /* Update the IRAM copy used by the reset handler. The IRAM copy
554            can't use used directly by cpumask_set_cpu() because it uses
555            LDREX/STREX which requires the addressed location to be inner
556            cacheable and sharable which IRAM isn't. */
557         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
558         dsb();
559
560         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
561                 last_cpu = true;
562 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
563         else if (cpu == 1)
564                 tegra2_cpu_set_resettable_soon();
565 #endif
566
567         spin_unlock(&tegra_lp2_lock);
568         return last_cpu;
569 }
570
571 static void tegra_sleep_core(enum tegra_suspend_mode mode,
572                              unsigned long v2p)
573 {
574 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
575         outer_flush_range(__pa(&tegra_resume_timestamps_start),
576                           __pa(&tegra_resume_timestamps_end));
577
578         if (mode == TEGRA_SUSPEND_LP0) {
579                 trace_smc_sleep_core(NVSEC_SMC_START);
580
581                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
582                                   virt_to_phys(tegra_resume));
583         } else {
584                 trace_smc_sleep_core(NVSEC_SMC_START);
585
586                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
587                                   (TEGRA_RESET_HANDLER_BASE +
588                                    tegra_cpu_reset_handler_offset));
589         }
590
591         trace_smc_sleep_core(NVSEC_SMC_DONE);
592 #endif
593 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
594         cpu_suspend(v2p, tegra2_sleep_core_finish);
595 #else
596         cpu_suspend(v2p, tegra3_sleep_core_finish);
597 #endif
598 }
599
600 static inline void tegra_sleep_cpu(unsigned long v2p)
601 {
602         cpu_suspend(v2p, tegra_sleep_cpu_finish);
603 }
604
605 static inline void tegra_stop_mc_clk(unsigned long v2p)
606 {
607 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
608         outer_flush_range(__pa(&tegra_resume_timestamps_start),
609                           __pa(&tegra_resume_timestamps_end));
610         trace_smc_sleep_core(NVSEC_SMC_START);
611
612         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
613                           (TEGRA_RESET_HANDLER_BASE +
614                            tegra_cpu_reset_handler_offset));
615
616         trace_smc_sleep_core(NVSEC_SMC_DONE);
617 #endif
618         cpu_suspend(v2p, tegra3_stop_mc_clk_finish);
619 }
620
621 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
622                                         unsigned int flags)
623 {
624         u32 reg;
625         unsigned int remain;
626
627         /* Only the last cpu down does the final suspend steps */
628         reg = readl(pmc + PMC_CTRL);
629         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
630         if (pdata->combined_req)
631                 reg &= ~TEGRA_POWER_PWRREQ_OE;
632         else
633                 reg |= TEGRA_POWER_PWRREQ_OE;
634
635         reg &= ~TEGRA_POWER_EFFECT_LP0;
636         writel(reg, pmc + PMC_CTRL);
637
638         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
639
640         /*
641          * We can use clk_get_rate_all_locked() here, because all other cpus
642          * are in LP2 state and irqs are disabled
643          */
644         if (flags & TEGRA_POWER_CLUSTER_MASK) {
645                 trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_START);
646                 set_power_timers(pdata->cpu_timer, 2,
647                         clk_get_rate_all_locked(tegra_pclk));
648                 if (flags & TEGRA_POWER_CLUSTER_G) {
649                         /*
650                          * To reduce the vdd_cpu up latency when LP->G
651                          * transition. Before the transition, enable
652                          * the vdd_cpu rail.
653                          */
654                         if (is_lp_cluster()) {
655 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
656                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
657                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
658                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
659 #else
660                                 writel(UN_PWRGATE_CPU,
661                                        pmc + PMC_PWRGATE_TOGGLE);
662 #endif
663                         }
664                 }
665                 tegra_cluster_switch_prolog(flags);
666         } else {
667                 suspend_cpu_dfll_mode();
668                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
669                         clk_get_rate_all_locked(tegra_pclk));
670 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
671                 reg = readl(FLOW_CTRL_CPU_CSR(0));
672                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
673                 if (is_lp_cluster()) {
674                         /* for LP cluster, there is no option for rail gating */
675                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
676                                                 TEGRA_POWER_CLUSTER_PART_MASK)
677                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
678                         else if (flags)
679                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
680                 }
681                 else {
682                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
683                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
684                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
685                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
686                 }
687                 writel(reg, FLOW_CTRL_CPU_CSR(0));
688 #endif
689         }
690
691         if (sleep_time)
692                 tegra_pd_set_trigger(sleep_time);
693
694         cpu_cluster_pm_enter();
695         suspend_cpu_complex(flags);
696         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
697 #if defined(CONFIG_CACHE_L2X0)
698 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
699         flush_cache_all();
700         outer_disable();
701 #elif !defined(CONFIG_ARCH_TEGRA_14x_SOC)
702         tegra_resume_l2_init = 1;
703         __cpuc_flush_dcache_area(&tegra_resume_l2_init, sizeof(unsigned long));
704         outer_flush_range(__pa(&tegra_resume_l2_init),
705                           __pa(&tegra_resume_l2_init) + sizeof(unsigned long));
706 #endif
707 #endif
708
709         if (flags & TEGRA_POWER_STOP_MC_CLK)
710                 tegra_stop_mc_clk(PHYS_OFFSET - PAGE_OFFSET);
711         else
712                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
713
714 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
715         tegra_init_cache(true);
716 #elif defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
717         tegra_init_cache(false);
718 #endif
719
720 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
721 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
722         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
723         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
724 #endif
725 #endif
726
727         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
728         restore_cpu_complex(flags);
729         cpu_cluster_pm_exit();
730
731         remain = tegra_pd_timer_remain();
732         if (sleep_time)
733                 tegra_pd_set_trigger(0);
734
735         if (flags & TEGRA_POWER_CLUSTER_MASK) {
736                 tegra_cluster_switch_epilog(flags);
737                 if (is_idle_task(current))
738                         trace_nvcpu_cluster_rcuidle(NVPOWER_CPU_CLUSTER_DONE);
739                 else
740                         trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_DONE);
741         } else {
742                 resume_cpu_dfll_mode();
743         }
744         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
745
746 #if INSTRUMENT_CLUSTER_SWITCH
747         if (flags & TEGRA_POWER_CLUSTER_MASK) {
748                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
749                         is_lp_cluster() ? "G=>LP" : "LP=>G",
750                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
751                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
752                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
753                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
754                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
755                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
756                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
757                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
758         }
759 #endif
760         return remain;
761 }
762
763 void tegra_mc_clk_prepare(void)
764 {
765         /* copy the reset vector and SDRAM shutdown code into IRAM */
766         memcpy(iram_save, iram_code, iram_save_size);
767         memcpy(iram_code, tegra_iram_start(), iram_save_size);
768
769         *iram_cpu_lp1_mask = 1;
770         *iram_mc_clk_mask = 1;
771
772         __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
773         wmb();
774 }
775
776 void tegra_mc_clk_finish(void)
777 {
778         /* restore IRAM */
779         memcpy(iram_code, iram_save, iram_save_size);
780         *iram_cpu_lp1_mask = 0;
781         *iram_mc_clk_mask = 0;
782         writel(0, pmc + PMC_SCRATCH41);
783 }
784
785 static int tegra_common_suspend(void)
786 {
787         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
788
789         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
790         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
791         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
792
793         /* copy the reset vector and SDRAM shutdown code into IRAM */
794         memcpy(iram_save, iram_code, iram_save_size);
795         memcpy(iram_code, tegra_iram_start(), iram_save_size);
796
797         return 0;
798 }
799
800 static void tegra_common_resume(void)
801 {
802         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
803 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
804         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
805 #endif
806
807 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
808         /* Clear DPD Enable */
809         writel(0x0, pmc + PMC_DPD_ENABLE);
810 #endif
811
812         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
813         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
814         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
815 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
816         /* trigger emc mode write */
817         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
818         /* clear scratch registers shared by suspend and the reset pen */
819         writel(0x0, pmc + PMC_SCRATCH39);
820 #endif
821         writel(0x0, pmc + PMC_SCRATCH41);
822
823         /* restore IRAM */
824         memcpy(iram_code, iram_save, iram_save_size);
825 }
826
827 static int tegra_suspend_prepare_late(void)
828 {
829 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
830         disable_irq(INT_SYS_STATS_MON);
831 #endif
832         return 0;
833 }
834
835 static void tegra_suspend_wake(void)
836 {
837 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
838         enable_irq(INT_SYS_STATS_MON);
839 #endif
840 }
841
842 static void tegra_pm_set(enum tegra_suspend_mode mode)
843 {
844         u32 reg, boot_flag;
845         unsigned long rate = 32768;
846
847         reg = readl(pmc + PMC_CTRL);
848         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
849         if (pdata->combined_req)
850                 reg &= ~TEGRA_POWER_PWRREQ_OE;
851         else
852                 reg |= TEGRA_POWER_PWRREQ_OE;
853         reg &= ~TEGRA_POWER_EFFECT_LP0;
854
855         switch (mode) {
856         case TEGRA_SUSPEND_LP0:
857 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
858                 rate = clk_get_rate_all_locked(tegra_pclk);
859 #endif
860                 if (pdata->combined_req) {
861                         reg |= TEGRA_POWER_PWRREQ_OE;
862                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
863                 }
864
865 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
866                 /*
867                  * LP0 boots through the AVP, which then resumes the AVP to
868                  * the address in scratch 39, and the cpu to the address in
869                  * scratch 41 to tegra_resume
870                  */
871                 writel(0x0, pmc + PMC_SCRATCH39);
872 #endif
873
874                 /* Enable DPD sample to trigger sampling pads data and direction
875                  * in which pad will be driven during lp0 mode*/
876                 writel(0x1, pmc + PMC_DPD_SAMPLE);
877 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC) &&\
878                                          !defined(CONFIG_ARCH_TEGRA_14x_SOC)
879                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
880                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
881 #endif
882
883 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
884                 /* this is needed only for T11x, not for other chips */
885                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
886 #endif
887
888                 /* Set warmboot flag */
889                 boot_flag = readl(pmc + PMC_SCRATCH0);
890                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
891
892                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
893
894                 reg |= TEGRA_POWER_EFFECT_LP0;
895                 /* No break here. LP0 code falls through to write SCRATCH41 */
896         case TEGRA_SUSPEND_LP1:
897                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
898                 wmb();
899                 break;
900         case TEGRA_SUSPEND_LP2:
901                 rate = clk_get_rate(tegra_pclk);
902                 break;
903         case TEGRA_SUSPEND_NONE:
904                 return;
905         default:
906                 BUG();
907         }
908
909         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
910
911         pmc_32kwritel(reg, PMC_CTRL);
912 }
913
914 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
915         [TEGRA_SUSPEND_NONE] = "none",
916         [TEGRA_SUSPEND_LP2] = "LP2",
917         [TEGRA_SUSPEND_LP1] = "LP1",
918         [TEGRA_SUSPEND_LP0] = "LP0",
919 };
920
921 static int tegra_suspend_enter(suspend_state_t state)
922 {
923         int ret;
924         ktime_t delta;
925         struct timespec ts_entry, ts_exit;
926
927         if (pdata && pdata->board_suspend)
928                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
929
930         read_persistent_clock(&ts_entry);
931
932         ret = tegra_suspend_dram(current_suspend_mode, 0);
933         if (ret) {
934                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
935                 goto abort_suspend;
936         }
937
938         read_persistent_clock(&ts_exit);
939
940         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
941                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
942
943                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
944                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
945                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
946                 else
947                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
948         }
949
950 abort_suspend:
951         if (pdata && pdata->board_resume)
952                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
953
954         return ret;
955 }
956
957 static void tegra_suspend_check_pwr_stats(void)
958 {
959         /* cpus and l2 are powered off later */
960         unsigned long pwrgate_partid_mask =
961 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
962                 (1 << TEGRA_POWERGATE_HEG)      |
963                 (1 << TEGRA_POWERGATE_SATA)     |
964                 (1 << TEGRA_POWERGATE_3D1)      |
965 #endif
966                 (1 << TEGRA_POWERGATE_3D)       |
967                 (1 << TEGRA_POWERGATE_VENC)     |
968                 (1 << TEGRA_POWERGATE_PCIE)     |
969                 (1 << TEGRA_POWERGATE_VDEC)     |
970                 (1 << TEGRA_POWERGATE_MPE);
971
972         int partid;
973
974         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
975                 if ((1 << partid) & pwrgate_partid_mask)
976                         if (tegra_powergate_is_powered(partid))
977                                 pr_warning("partition %s is left on before suspend\n",
978                                         tegra_powergate_get_name(partid));
979
980         return;
981 }
982
983 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
984 /* This is the opposite of the LP1BB related PMC setup that occurs
985  * during suspend.
986  */
987 static void tegra_disable_lp1bb_interrupt(void)
988 {
989         unsigned reg;
990         /* mem_req = 0 was set as an interrupt during LP1BB entry.
991          * It has to be disabled now
992          */
993         reg = readl(pmc + PMC_CTRL2);
994         reg &= ~(PMC_CTRL2_WAKE_DET_EN);
995         pmc_32kwritel(reg, PMC_CTRL2);
996
997         /* Program mem_req NOT to be a wake event */
998         reg = readl(pmc + PMC_WAKE2_MASK);
999         reg &= ~(PMC_WAKE2_BB_MEM_REQ);
1000         pmc_32kwritel(reg, PMC_WAKE2_MASK);
1001
1002         /* Set up the LIC to NOT accept pmc_wake events as interrupts */
1003         reg = TRI_ICTLR_PMC_WAKE_INT;
1004         writel(reg, tert_ictlr + TRI_ICTLR_CPU_IER_CLR);
1005 }
1006 #endif
1007
1008 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
1009 {
1010         int err = 0;
1011         u32 scratch37 = 0xDEADBEEF;
1012         u32 reg;
1013 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1014         u32 enter_state = 0;
1015 #endif
1016
1017         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
1018                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
1019                 err = -ENXIO;
1020                 goto fail;
1021         }
1022
1023         if (tegra_is_voice_call_active()) {
1024                 /* backup the current value of scratch37 */
1025                 scratch37 = readl(pmc + PMC_SCRATCH37);
1026
1027                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
1028                 reg = TEGRA_POWER_LP1_AUDIO;
1029                 pmc_32kwritel(reg, PMC_SCRATCH37);
1030         }
1031
1032         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
1033                 pr_info("LP0 not used due to unsupported wakeup events\n");
1034                 mode = TEGRA_SUSPEND_LP1;
1035         }
1036
1037         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
1038                 tegra_suspend_check_pwr_stats();
1039
1040         tegra_common_suspend();
1041
1042         tegra_pm_set(mode);
1043
1044         if (pdata && pdata->board_suspend)
1045                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
1046
1047         local_fiq_disable();
1048
1049 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1050         tegra_smp_save_power_mask();
1051 #endif
1052
1053         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
1054
1055         if (mode == TEGRA_SUSPEND_LP0) {
1056 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
1057                 reg = readl(pmc + PMC_SCRATCH4);
1058                 if (is_lp_cluster())
1059                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
1060                 else
1061                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
1062                 pmc_32kwritel(reg, PMC_SCRATCH4);
1063 #endif
1064                 tegra_tsc_suspend();
1065                 tegra_lp0_suspend_mc();
1066                 tegra_cpu_reset_handler_save();
1067                 tegra_tsc_wait_for_suspend();
1068 #ifndef CONFIG_TEGRA_SIMULATION_PLATFORM
1069                 tegra_smp_clear_power_mask();
1070 #endif
1071         }
1072
1073 #if !defined(CONFIG_ARCH_TEGRA_14x_SOC)
1074         if (mode == TEGRA_SUSPEND_LP1)
1075 #endif
1076                 *iram_cpu_lp1_mask = 1;
1077
1078         suspend_cpu_complex(flags);
1079
1080 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
1081         /* In case of LP0/1, program external power gating accordinly */
1082         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
1083                 reg = readl(FLOW_CTRL_CPU_CSR(0));
1084                 if (is_lp_cluster())
1085                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
1086                 else
1087                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
1088                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
1089         }
1090 #endif
1091
1092         flush_cache_all();
1093         outer_disable();
1094
1095         if (mode == TEGRA_SUSPEND_LP2)
1096                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
1097         else
1098                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
1099
1100         tegra_init_cache(true);
1101
1102 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1103         reg = readl(pmc + PMC_LP_STATE_SCRATCH_REG);
1104         enter_state = (reg >> PMC_LP_STATE_BIT_OFFSET) & PMC_LP_STATE_BIT_MASK;
1105         /* If we actually had entered in either LP1 or LP1BB,
1106          * restore power mask and disable mem_req interrupt PMC
1107          */
1108         if (enter_state) {
1109                 pr_info("Exited state is LP1/LP1BB\n");
1110                 tegra_disable_lp1bb_interrupt();
1111                 tegra_smp_restore_power_mask();
1112         }
1113 #endif
1114
1115 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
1116 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
1117         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
1118         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
1119 #endif
1120
1121         if (mode == TEGRA_SUSPEND_LP0) {
1122                 trace_secureos_init(tegra_resume_entry_time,
1123                         NVSEC_SUSPEND_EXIT_DONE);
1124         }
1125 #endif
1126
1127         if (mode == TEGRA_SUSPEND_LP0) {
1128
1129                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
1130                 * Otherwise it is creating issue in cluster switch after LP0 *
1131 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1132                 reg = readl(pmc+PMC_CTRL);
1133                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
1134                 pmc_32kwritel(reg, PMC_CTRL);
1135 #endif
1136                 */
1137
1138                 tegra_tsc_resume();
1139                 tegra_cpu_reset_handler_restore();
1140                 tegra_lp0_resume_mc();
1141                 tegra_tsc_wait_for_resume();
1142         }
1143
1144 #if !defined(CONFIG_ARCH_TEGRA_14x_SOC)
1145         if (mode == TEGRA_SUSPEND_LP1)
1146 #endif
1147                 *iram_cpu_lp1_mask = 0;
1148
1149         /* if scratch37 was clobbered during LP1, restore it */
1150         if (scratch37 != 0xDEADBEEF)
1151                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1152
1153         restore_cpu_complex(flags);
1154
1155         /* for platforms where the core & CPU power requests are
1156          * combined as a single request to the PMU, transition out
1157          * of LP0 state by temporarily enabling both requests
1158          */
1159         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1160                 reg = readl(pmc + PMC_CTRL);
1161                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1162                 pmc_32kwritel(reg, PMC_CTRL);
1163                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1164                 pmc_32kwritel(reg, PMC_CTRL);
1165         }
1166
1167         if (pdata && pdata->board_resume)
1168                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1169
1170         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1171
1172         local_fiq_enable();
1173
1174         tegra_common_resume();
1175
1176 fail:
1177         return err;
1178 }
1179
1180 /*
1181  * Function pointers to optional board specific function
1182  */
1183 void (*tegra_deep_sleep)(int);
1184 EXPORT_SYMBOL(tegra_deep_sleep);
1185
1186 static int tegra_suspend_prepare(void)
1187 {
1188         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1189                 tegra_deep_sleep(1);
1190         return 0;
1191 }
1192
1193 static void tegra_suspend_finish(void)
1194 {
1195         if (pdata && pdata->cpu_resume_boost) {
1196                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1197                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1198                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1199         }
1200
1201         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1202                 tegra_deep_sleep(0);
1203 }
1204
1205 static const struct platform_suspend_ops tegra_suspend_ops = {
1206         .valid          = suspend_valid_only_mem,
1207         .prepare        = tegra_suspend_prepare,
1208         .finish         = tegra_suspend_finish,
1209         .prepare_late   = tegra_suspend_prepare_late,
1210         .wake           = tegra_suspend_wake,
1211         .enter          = tegra_suspend_enter,
1212 };
1213
1214 static ssize_t suspend_mode_show(struct kobject *kobj,
1215                                         struct kobj_attribute *attr, char *buf)
1216 {
1217         char *start = buf;
1218         char *end = buf + PAGE_SIZE;
1219
1220         start += scnprintf(start, end - start, "%s ", \
1221                                 tegra_suspend_name[current_suspend_mode]);
1222         start += scnprintf(start, end - start, "\n");
1223
1224         return start - buf;
1225 }
1226
1227 static ssize_t suspend_mode_store(struct kobject *kobj,
1228                                         struct kobj_attribute *attr,
1229                                         const char *buf, size_t n)
1230 {
1231         int len;
1232         const char *name_ptr;
1233         enum tegra_suspend_mode new_mode;
1234
1235         name_ptr = buf;
1236         while (*name_ptr && !isspace(*name_ptr))
1237                 name_ptr++;
1238         len = name_ptr - buf;
1239         if (!len)
1240                 goto bad_name;
1241         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1242         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1243                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1244                 pr_info("Illegal tegra suspend state: %s\n", buf);
1245                 goto bad_name;
1246         }
1247
1248         for (new_mode = TEGRA_SUSPEND_NONE; \
1249                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1250                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1251                         current_suspend_mode = new_mode;
1252                         break;
1253                 }
1254         }
1255
1256 bad_name:
1257         return n;
1258 }
1259
1260 static struct kobj_attribute suspend_mode_attribute =
1261         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1262
1263 static struct kobject *suspend_kobj;
1264
1265 static int tegra_pm_enter_suspend(void)
1266 {
1267         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1268         suspend_cpu_dfll_mode();
1269         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1270                 tegra_lp0_cpu_mode(true);
1271         return 0;
1272 }
1273
1274 static void tegra_pm_enter_resume(void)
1275 {
1276         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1277                 tegra_lp0_cpu_mode(false);
1278         resume_cpu_dfll_mode();
1279         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1280 }
1281
1282 static void tegra_pm_enter_shutdown(void)
1283 {
1284         suspend_cpu_dfll_mode();
1285         pr_info("Shutting down tegra ...\n");
1286 }
1287
1288 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1289         .suspend = tegra_pm_enter_suspend,
1290         .resume = tegra_pm_enter_resume,
1291         .shutdown = tegra_pm_enter_shutdown,
1292 };
1293
1294 static __init int tegra_pm_enter_syscore_init(void)
1295 {
1296         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1297         return 0;
1298 }
1299 subsys_initcall(tegra_pm_enter_syscore_init);
1300 #endif
1301
1302 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1303 {
1304         u32 reg;
1305         u32 mode;
1306
1307 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1308         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1309         BUG_ON(IS_ERR(tegra_dfll));
1310 #endif
1311         tegra_pclk = clk_get_sys(NULL, "pclk");
1312         BUG_ON(IS_ERR(tegra_pclk));
1313         pdata = plat;
1314         (void)reg;
1315         (void)mode;
1316
1317         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1318                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1319
1320 #ifndef CONFIG_PM_SLEEP
1321         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1322                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1323                            "disabling suspend\n", __func__);
1324                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1325         }
1326 #else
1327         if (create_suspend_pgtable() < 0) {
1328                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1329                                 __func__);
1330                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1331                 goto fail;
1332         }
1333
1334         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1335             (tegra_revision == TEGRA_REVISION_A01) &&
1336             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1337                 /* Tegra 3 A01 supports only LP1 */
1338                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1339                            "-- disabling LP0\n", __func__);
1340                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1341         }
1342         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1343                 tegra_lp0_vec_relocate) {
1344                 unsigned char *reloc_lp0;
1345                 unsigned long tmp;
1346                 void __iomem *orig;
1347                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1348                                         GFP_KERNEL);
1349                 WARN_ON(!reloc_lp0);
1350                 if (!reloc_lp0) {
1351                         pr_err("%s: Failed to allocate reloc_lp0\n",
1352                                 __func__);
1353                         goto out;
1354                 }
1355
1356                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1357                 WARN_ON(!orig);
1358                 if (!orig) {
1359                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1360                                 __func__, tegra_lp0_vec_start);
1361                         kfree(reloc_lp0);
1362                         goto out;
1363                 }
1364
1365                 tmp = (unsigned long) reloc_lp0;
1366                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1367                 reloc_lp0 = (unsigned char *)tmp;
1368                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1369                 iounmap(orig);
1370                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1371         }
1372
1373 out:
1374         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1375                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1376                            "provided by bootlader -- disabling LP0\n",
1377                            __func__);
1378                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1379         }
1380
1381         iram_save_size = tegra_iram_end() - tegra_iram_start();
1382
1383         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1384         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1385                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1386                        "-- LP0/LP1 unavailable\n", __func__);
1387                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1388         }
1389
1390 #ifdef CONFIG_TEGRA_LP1_950
1391         if (pdata->lp1_lowvolt_support) {
1392                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1393                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1394                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1395
1396                 lp1_core_lowvolt = 0;
1397                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1398                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1399
1400                 lp1_core_highvolt = 0;
1401                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1402                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1403         }
1404 #endif
1405         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1406         /* Initialize scratch registers used for CPU LP2 synchronization */
1407         writel(0, pmc + PMC_SCRATCH37);
1408         writel(0, pmc + PMC_SCRATCH38);
1409 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1410         writel(0, pmc + PMC_SCRATCH39);
1411 #endif
1412         writel(0, pmc + PMC_SCRATCH41);
1413
1414         /* Always enable CPU power request; just normal polarity is supported */
1415         reg = readl(pmc + PMC_CTRL);
1416         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1417         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1418         pmc_32kwritel(reg, PMC_CTRL);
1419
1420         /* Configure core power request and system clock control if LP0
1421            is supported */
1422         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1423         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1424
1425         reg = readl(pmc + PMC_CTRL);
1426
1427         if (!pdata->sysclkreq_high)
1428                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1429         else
1430                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1431
1432         if (!pdata->corereq_high)
1433                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1434         else
1435                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1436
1437         /* configure output inverters while the request is tristated */
1438         pmc_32kwritel(reg, PMC_CTRL);
1439
1440         /* now enable requests */
1441         reg |= TEGRA_POWER_SYSCLK_OE;
1442         if (!pdata->combined_req)
1443                 reg |= TEGRA_POWER_PWRREQ_OE;
1444         pmc_32kwritel(reg, PMC_CTRL);
1445
1446         if (pdata->sysclkreq_gpio) {
1447                 reg = readl(pmc + PMC_DPAD_ORIDE);
1448                 reg &= ~TEGRA_DPAD_ORIDE_SYS_CLK_REQ;
1449                 pmc_32kwritel(reg, PMC_DPAD_ORIDE);
1450         }
1451
1452         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1453                 tegra_lp0_suspend_init();
1454
1455         suspend_set_ops(&tegra_suspend_ops);
1456
1457         /* Create /sys/power/suspend/type */
1458         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1459         if (suspend_kobj) {
1460                 if (sysfs_create_file(suspend_kobj, \
1461                                                 &suspend_mode_attribute.attr))
1462                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1463                                                                 __func__);
1464         }
1465
1466         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1467         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1468         iram_mc_clk_mask = tegra_mc_clk_mask;
1469
1470         /* clear io dpd settings before kernel */
1471         tegra_bl_io_dpd_cleanup();
1472
1473 fail:
1474 #endif
1475         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1476                 tegra_pd_in_idle(false);
1477
1478         current_suspend_mode = plat->suspend_mode;
1479 }
1480
1481 unsigned long debug_uart_port_base = 0;
1482 EXPORT_SYMBOL(debug_uart_port_base);
1483
1484 static int tegra_debug_uart_suspend(void)
1485 {
1486         void __iomem *uart;
1487         u32 lcr;
1488
1489         if (!debug_uart_port_base)
1490                 return 0;
1491
1492         uart = IO_ADDRESS(debug_uart_port_base);
1493
1494         lcr = readb(uart + UART_LCR * 4);
1495
1496         tegra_sctx.uart[0] = lcr;
1497         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1498
1499         /* DLAB = 0 */
1500         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1501
1502         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1503
1504         /* DLAB = 1 */
1505         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1506
1507         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1508         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1509
1510         writeb(lcr, uart + UART_LCR * 4);
1511
1512         return 0;
1513 }
1514
1515 static void tegra_debug_uart_resume(void)
1516 {
1517         void __iomem *uart;
1518         u32 lcr;
1519
1520         if (!debug_uart_port_base)
1521                 return;
1522
1523         uart = IO_ADDRESS(debug_uart_port_base);
1524
1525         lcr = tegra_sctx.uart[0];
1526
1527         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1528
1529         /* DLAB = 0 */
1530         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1531
1532         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1533                         uart + UART_FCR * 4);
1534
1535         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1536
1537         /* DLAB = 1 */
1538         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1539
1540         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1541         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1542
1543         writeb(lcr, uart + UART_LCR * 4);
1544 }
1545
1546 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1547         .suspend = tegra_debug_uart_suspend,
1548         .resume = tegra_debug_uart_resume,
1549 };
1550
1551 struct clk *debug_uart_clk = NULL;
1552 EXPORT_SYMBOL(debug_uart_clk);
1553
1554 void tegra_console_uart_suspend(void)
1555 {
1556         if (console_suspend_enabled && debug_uart_clk)
1557                 tegra_clk_disable_unprepare(debug_uart_clk);
1558 }
1559
1560 void tegra_console_uart_resume(void)
1561 {
1562         if (console_suspend_enabled && debug_uart_clk)
1563                 tegra_clk_prepare_enable(debug_uart_clk);
1564 }
1565
1566 static int tegra_debug_uart_syscore_init(void)
1567 {
1568         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1569         return 0;
1570 }
1571 arch_initcall(tegra_debug_uart_syscore_init);