ARM: tegra11x: CPUID virtualization support
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58 #include <asm/smp_plat.h>
59
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62 #include <mach/hardware.h>
63
64 #include "board.h"
65 #include "clock.h"
66 #include "common.h"
67 #include "cpuidle.h"
68 #include "fuse.h"
69 #include "gic.h"
70 #include "iomap.h"
71 #include "pm.h"
72 #include "pm-irq.h"
73 #include "reset.h"
74 #include "sleep.h"
75 #include "timer.h"
76 #include "dvfs.h"
77 #include "cpu-tegra.h"
78
79 struct suspend_context {
80         /*
81          * The next 7 values are referenced by offset in __restart_plls
82          * in headsmp-t2.S, and should not be moved
83          */
84         u32 pllx_misc;
85         u32 pllx_base;
86         u32 pllp_misc;
87         u32 pllp_base;
88         u32 pllp_outa;
89         u32 pllp_outb;
90         u32 pll_timeout;
91
92         u32 cpu_burst;
93         u32 clk_csite_src;
94         u32 cclk_divider;
95
96         u32 mc[3];
97         u8 uart[5];
98
99         struct tegra_twd_context twd;
100 #ifdef CONFIG_ARM_ARCH_TIMER
101         struct arch_timer_context arch_timer;
102 #endif
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
107 static pgd_t *tegra_pgd;
108 static DEFINE_SPINLOCK(tegra_lp2_lock);
109 static cpumask_t tegra_in_lp2;
110 static cpumask_t *iram_cpu_lp2_mask;
111 static unsigned long *iram_cpu_lp1_mask;
112 static u8 *iram_save;
113 static unsigned long iram_save_size;
114 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
115 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
116 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
117 static int tegra_last_pclk;
118 #endif
119
120 struct suspend_context tegra_sctx;
121
122 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
123 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
124 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
125 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
126 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
127 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
128 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
129 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
130 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
131
132 #define PMC_CTRL                0x0
133 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
134 #define PMC_WAKE_MASK           0xc
135 #define PMC_WAKE_LEVEL          0x10
136 #define PMC_DPAD_ORIDE          0x1C
137 #define PMC_WAKE_DELAY          0xe0
138 #define PMC_DPD_SAMPLE          0x20
139
140 #define PMC_WAKE_STATUS         0x14
141 #define PMC_SW_WAKE_STATUS      0x18
142 #define PMC_COREPWRGOOD_TIMER   0x3c
143 #define PMC_CPUPWRGOOD_TIMER    0xc8
144 #define PMC_CPUPWROFF_TIMER     0xcc
145 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
146
147 #define PMC_PWRGATE_TOGGLE      0x30
148 #define PWRGATE_TOGGLE_START    (1 << 8)
149 #define UN_PWRGATE_CPU          \
150         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
151
152 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
153 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
154 #endif
155
156 #define CLK_RESET_CCLK_BURST    0x20
157 #define CLK_RESET_CCLK_DIVIDER  0x24
158 #define CLK_RESET_PLLC_BASE     0x80
159 #define CLK_RESET_PLLM_BASE     0x90
160 #define CLK_RESET_PLLX_BASE     0xe0
161 #define CLK_RESET_PLLX_MISC     0xe4
162 #define CLK_RESET_PLLP_BASE     0xa0
163 #define CLK_RESET_PLLP_OUTA     0xa4
164 #define CLK_RESET_PLLP_OUTB     0xa8
165 #define CLK_RESET_PLLP_MISC     0xac
166
167 #define CLK_RESET_SOURCE_CSITE  0x1d4
168
169 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
170 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
171 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
172 #define CLK_RESET_CCLK_IDLE_POLICY         1
173 #define CLK_RESET_CCLK_RUN_POLICY          2
174 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
175 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
176
177 #define EMC_MRW_0               0x0e8
178 #define EMC_MRW_DEV_SELECTN     30
179 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
180
181 #define MC_SECURITY_START       0x6c
182 #define MC_SECURITY_SIZE        0x70
183 #define MC_SECURITY_CFG2        0x7c
184
185 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
186 static struct clk *tegra_dfll;
187 #endif
188 static struct clk *tegra_pclk;
189 static const struct tegra_suspend_platform_data *pdata;
190 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
191
192 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
193 enum tegra_cluster_switch_time_id {
194         tegra_cluster_switch_time_id_start = 0,
195         tegra_cluster_switch_time_id_prolog,
196         tegra_cluster_switch_time_id_switch,
197         tegra_cluster_switch_time_id_epilog,
198         tegra_cluster_switch_time_id_max
199 };
200
201 static unsigned long
202                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
203 #define tegra_cluster_switch_time(flags, id) \
204         do { \
205                 barrier(); \
206                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
207                         void __iomem *timer_us = \
208                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
209                         if (id < tegra_cluster_switch_time_id_max) \
210                                 tegra_cluster_switch_times[id] = \
211                                                         readl(timer_us); \
212                                 wmb(); \
213                 } \
214                 barrier(); \
215         } while(0)
216 #else
217 #define tegra_cluster_switch_time(flags, id) do {} while(0)
218 #endif
219
220 #ifdef CONFIG_PM_SLEEP
221 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
222         [TEGRA_SUSPEND_NONE]    = "none",
223         [TEGRA_SUSPEND_LP2]     = "lp2",
224         [TEGRA_SUSPEND_LP1]     = "lp1",
225         [TEGRA_SUSPEND_LP0]     = "lp0",
226 };
227
228 unsigned long tegra_cpu_power_good_time(void)
229 {
230         if (WARN_ON_ONCE(!pdata))
231                 return 5000;
232
233         return pdata->cpu_timer;
234 }
235
236 unsigned long tegra_cpu_power_off_time(void)
237 {
238         if (WARN_ON_ONCE(!pdata))
239                 return 5000;
240
241         return pdata->cpu_off_timer;
242 }
243
244 unsigned long tegra_cpu_lp2_min_residency(void)
245 {
246         if (WARN_ON_ONCE(!pdata))
247                 return 2000;
248
249         return pdata->cpu_lp2_min_residency;
250 }
251
252 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
253 unsigned long tegra_min_residency_noncpu(void)
254 {
255         return pdata->min_residency_noncpu;
256 }
257
258 unsigned long tegra_min_residency_crail(void)
259 {
260         return pdata->min_residency_crail;
261 }
262 #endif
263
264 static void suspend_cpu_dfll_mode(void)
265 {
266 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
267         /* If DFLL is used as CPU clock source go to open loop mode */
268         if (!is_lp_cluster() && tegra_dfll &&
269             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
270                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
271 #endif
272 }
273
274 static void resume_cpu_dfll_mode(void)
275 {
276 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
277         /* If DFLL is used as CPU clock source restore closed loop mode */
278         if (!is_lp_cluster() && tegra_dfll &&
279             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
280                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
281 #endif
282 }
283
284 /*
285  * create_suspend_pgtable
286  *
287  * Creates a page table with identity mappings of physical memory and IRAM
288  * for use when the MMU is off, in addition to all the regular kernel mappings.
289  */
290 static __init int create_suspend_pgtable(void)
291 {
292         tegra_pgd = pgd_alloc(&init_mm);
293         if (!tegra_pgd)
294                 return -ENOMEM;
295
296         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
297         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
298                 high_memory, 0);
299         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
300                 IO_IRAM_VIRT + SECTION_SIZE, 0);
301
302         /* inner/outer write-back/write-allocate, sharable */
303         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
304
305         return 0;
306 }
307
308 /* ensures that sufficient time is passed for a register write to
309  * serialize into the 32KHz domain */
310 static void pmc_32kwritel(u32 val, unsigned long offs)
311 {
312         writel(val, pmc + offs);
313         udelay(130);
314 }
315
316 static void set_power_timers(unsigned long us_on, unsigned long us_off,
317                              long rate)
318 {
319         static unsigned long last_us_off = 0;
320         unsigned long long ticks;
321         unsigned long long pclk;
322
323         if (WARN_ON_ONCE(rate <= 0))
324                 pclk = 100000000;
325         else
326                 pclk = rate;
327
328         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
329                 ticks = (us_on * pclk) + 999999ull;
330                 do_div(ticks, 1000000);
331                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
332
333                 ticks = (us_off * pclk) + 999999ull;
334                 do_div(ticks, 1000000);
335                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
336                 wmb();
337         }
338         tegra_last_pclk = pclk;
339         last_us_off = us_off;
340 }
341
342 /*
343  * restore_cpu_complex
344  *
345  * restores cpu clock setting, clears flow controller
346  *
347  * Always called on CPU 0.
348  */
349 static void restore_cpu_complex(u32 mode)
350 {
351         int cpu = cpu_logical_map(smp_processor_id());
352         unsigned int reg;
353 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
354         unsigned int policy;
355 #endif
356
357 /*
358  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
359  * or restored by common clock suspend/resume procedures. Hence, we don't
360  * need it here.
361  */
362 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
363         /* Is CPU complex already running on PLLX? */
364         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
365         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
366
367         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
368                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
369         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
370                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
371         else
372                 BUG();
373
374         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
375                 /* restore PLLX settings if CPU is on different PLL */
376                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
377                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
378
379                 /* wait for PLL stabilization if PLLX was enabled */
380                 if (tegra_sctx.pllx_base & (1<<30)) {
381 #if USE_PLL_LOCK_BITS
382                         /* Enable lock detector */
383                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
384                         reg |= 1<<18;
385                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
386                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
387                                  (1<<27)))
388                                 cpu_relax();
389
390                         udelay(PLL_POST_LOCK_DELAY);
391 #else
392                         udelay(300);
393 #endif
394                 }
395         }
396
397         /* Restore original burst policy setting for calls resulting from CPU
398            LP2 in idle or system suspend; keep cluster switch prolog setting
399            intact. */
400         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
401                 writel(tegra_sctx.cclk_divider, clk_rst +
402                        CLK_RESET_CCLK_DIVIDER);
403                 writel(tegra_sctx.cpu_burst, clk_rst +
404                        CLK_RESET_CCLK_BURST);
405         }
406 #endif
407         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
408
409         /* Do not power-gate CPU 0 when flow controlled */
410         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
411         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
412         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
413         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
414         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
415         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
416         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
417
418         /* If an immedidate cluster switch is being perfomed, restore the
419            local timer registers. For calls resulting from CPU LP2 in
420            idle or system suspend, the local timer was shut down and
421            timekeeping switched over to the global system timer. In this
422            case keep local timer disabled, and restore only periodic load. */
423         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
424                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
425 #ifdef CONFIG_ARM_ARCH_TIMER
426                 tegra_sctx.arch_timer.cntp_ctl = 0;
427 #endif
428 #ifdef CONFIG_HAVE_ARM_TWD
429                 tegra_sctx.twd.twd_ctrl = 0;
430 #endif
431         }
432 #ifdef CONFIG_ARM_ARCH_TIMER
433         arch_timer_resume(&tegra_sctx.arch_timer);
434 #endif
435 #ifdef CONFIG_HAVE_ARM_TWD
436         tegra_twd_resume(&tegra_sctx.twd);
437 #endif
438 }
439
440 /*
441  * suspend_cpu_complex
442  *
443  * saves pll state for use by restart_plls, prepares flow controller for
444  * transition to suspend state
445  *
446  * Must always be called on cpu 0.
447  */
448 static void suspend_cpu_complex(u32 mode)
449 {
450         int cpu = cpu_logical_map(smp_processor_id());
451         unsigned int reg;
452         int i;
453
454         BUG_ON(cpu != 0);
455
456         /* switch coresite to clk_m, save off original source */
457         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
458         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
459
460         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
461         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
462         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
463         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
464         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
465         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
466         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
467         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
468
469 #ifdef CONFIG_HAVE_ARM_TWD
470         tegra_twd_suspend(&tegra_sctx.twd);
471 #endif
472 #ifdef CONFIG_ARM_ARCH_TIMER
473         arch_timer_suspend(&tegra_sctx.arch_timer);
474 #endif
475
476         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
477         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
478         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
479         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
480         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
481 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
482         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
483 #else
484         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
485 #endif
486         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
487         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
488
489         for (i = 0; i < num_possible_cpus(); i++) {
490                 if (i == cpu)
491                         continue;
492                 reg = readl(FLOW_CTRL_CPU_CSR(i));
493                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
494                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
495                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
496         }
497
498         tegra_gic_cpu_disable(true);
499 }
500
501 void tegra_clear_cpu_in_lp2(int cpu)
502 {
503         spin_lock(&tegra_lp2_lock);
504         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
505         cpumask_clear_cpu(cpu, &tegra_in_lp2);
506
507         /* Update the IRAM copy used by the reset handler. The IRAM copy
508            can't use used directly by cpumask_clear_cpu() because it uses
509            LDREX/STREX which requires the addressed location to be inner
510            cacheable and sharable which IRAM isn't. */
511         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
512         dsb();
513
514         spin_unlock(&tegra_lp2_lock);
515 }
516
517 bool tegra_set_cpu_in_lp2(int cpu)
518 {
519         bool last_cpu = false;
520
521         spin_lock(&tegra_lp2_lock);
522         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
523         cpumask_set_cpu(cpu, &tegra_in_lp2);
524
525         /* Update the IRAM copy used by the reset handler. The IRAM copy
526            can't use used directly by cpumask_set_cpu() because it uses
527            LDREX/STREX which requires the addressed location to be inner
528            cacheable and sharable which IRAM isn't. */
529         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
530         dsb();
531
532         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
533                 last_cpu = true;
534 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
535         else if (cpu == 1)
536                 tegra2_cpu_set_resettable_soon();
537 #endif
538
539         spin_unlock(&tegra_lp2_lock);
540         return last_cpu;
541 }
542
543 static void tegra_sleep_core(enum tegra_suspend_mode mode,
544                              unsigned long v2p)
545 {
546 #ifdef CONFIG_TRUSTED_FOUNDATIONS
547         if (mode == TEGRA_SUSPEND_LP0) {
548                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
549                                   virt_to_phys(tegra_resume));
550         } else {
551                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
552                                   (TEGRA_RESET_HANDLER_BASE +
553                                    tegra_cpu_reset_handler_offset));
554         }
555 #endif
556 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
557         cpu_suspend(v2p, tegra2_sleep_core_finish);
558 #else
559         cpu_suspend(v2p, tegra3_sleep_core_finish);
560 #endif
561 }
562
563 static inline void tegra_sleep_cpu(unsigned long v2p)
564 {
565 #ifdef CONFIG_TRUSTED_FOUNDATIONS
566         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
567                           (TEGRA_RESET_HANDLER_BASE +
568                            tegra_cpu_reset_handler_offset));
569 #endif
570         cpu_suspend(v2p, tegra_sleep_cpu_finish);
571 }
572
573 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
574 {
575         u32 reg;
576         unsigned int remain;
577 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
578         pgd_t *pgd;
579 #endif
580
581         /* Only the last cpu down does the final suspend steps */
582         reg = readl(pmc + PMC_CTRL);
583         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
584         if (pdata->combined_req)
585                 reg &= ~TEGRA_POWER_PWRREQ_OE;
586         else
587                 reg |= TEGRA_POWER_PWRREQ_OE;
588
589         reg &= ~TEGRA_POWER_EFFECT_LP0;
590         writel(reg, pmc + PMC_CTRL);
591
592         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
593
594         /*
595          * We can use clk_get_rate_all_locked() here, because all other cpus
596          * are in LP2 state and irqs are disabled
597          */
598         if (flags & TEGRA_POWER_CLUSTER_MASK) {
599                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
600                 set_power_timers(pdata->cpu_timer, 2,
601                         clk_get_rate_all_locked(tegra_pclk));
602                 if (flags & TEGRA_POWER_CLUSTER_G) {
603                         /*
604                          * To reduce the vdd_cpu up latency when LP->G
605                          * transition. Before the transition, enable
606                          * the vdd_cpu rail.
607                          */
608                         if (is_lp_cluster()) {
609 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
610                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
611                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
612                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
613 #else
614                                 writel(UN_PWRGATE_CPU,
615                                        pmc + PMC_PWRGATE_TOGGLE);
616 #endif
617                         }
618                 }
619                 tegra_cluster_switch_prolog(flags);
620         } else {
621                 suspend_cpu_dfll_mode();
622                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
623                         clk_get_rate_all_locked(tegra_pclk));
624 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
625                 reg = readl(FLOW_CTRL_CPU_CSR(0));
626                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
627                 if (is_lp_cluster()) {
628                         /* for LP cluster, there is no option for rail gating */
629                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
630                                                 TEGRA_POWER_CLUSTER_PART_MASK)
631                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
632                         else if (flags)
633                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
634                 }
635                 else {
636                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
637                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
638                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
639                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
640                 }
641                 writel(reg, FLOW_CTRL_CPU_CSR(0));
642 #endif
643         }
644
645         if (sleep_time)
646                 tegra_lp2_set_trigger(sleep_time);
647
648         cpu_cluster_pm_enter();
649         suspend_cpu_complex(flags);
650         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
651 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
652         flush_cache_all();
653         /*
654          * No need to flush complete L2. Cleaning kernel and IO mappings
655          * is enough for the LP code sequence that has L2 disabled but
656          * MMU on.
657          */
658         pgd = cpu_get_pgd();
659         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
660                           __pa(pgd + PTRS_PER_PGD));
661         outer_disable();
662 #endif
663         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
664
665         tegra_init_cache(false);
666         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
667         restore_cpu_complex(flags);
668         cpu_cluster_pm_exit();
669
670         remain = tegra_lp2_timer_remain();
671         if (sleep_time)
672                 tegra_lp2_set_trigger(0);
673
674         if (flags & TEGRA_POWER_CLUSTER_MASK) {
675                 tegra_cluster_switch_epilog(flags);
676                 if (is_idle_task(current))
677                         trace_cpu_cluster_rcuidle(POWER_CPU_CLUSTER_DONE);
678                 else
679                         trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
680         } else {
681                 resume_cpu_dfll_mode();
682         }
683         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
684
685 #if INSTRUMENT_CLUSTER_SWITCH
686         if (flags & TEGRA_POWER_CLUSTER_MASK) {
687                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
688                         is_lp_cluster() ? "G=>LP" : "LP=>G",
689                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
690                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
691                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
692                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
693                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
694                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
695                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
696                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
697         }
698 #endif
699         return remain;
700 }
701
702 static int tegra_common_suspend(void)
703 {
704         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
705
706         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
707         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
708         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
709
710         /* copy the reset vector and SDRAM shutdown code into IRAM */
711         memcpy(iram_save, iram_code, iram_save_size);
712         memcpy(iram_code, tegra_iram_start(), iram_save_size);
713
714         return 0;
715 }
716
717 static void tegra_common_resume(void)
718 {
719         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
720 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
721         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
722 #endif
723
724         /* Clear DPD sample */
725         writel(0x0, pmc + PMC_DPD_SAMPLE);
726
727         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
728         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
729         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
730 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
731         /* trigger emc mode write */
732         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
733 #endif
734         /* clear scratch registers shared by suspend and the reset pen */
735         writel(0x0, pmc + PMC_SCRATCH39);
736         writel(0x0, pmc + PMC_SCRATCH41);
737
738         /* restore IRAM */
739         memcpy(iram_code, iram_save, iram_save_size);
740 }
741
742 static int tegra_suspend_prepare_late(void)
743 {
744 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
745         disable_irq(INT_SYS_STATS_MON);
746 #endif
747         return 0;
748 }
749
750 static void tegra_suspend_wake(void)
751 {
752 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
753         enable_irq(INT_SYS_STATS_MON);
754 #endif
755 }
756
757 static void tegra_pm_set(enum tegra_suspend_mode mode)
758 {
759         u32 reg, boot_flag;
760         unsigned long rate = 32768;
761
762         reg = readl(pmc + PMC_CTRL);
763         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
764         if (pdata->combined_req)
765                 reg &= ~TEGRA_POWER_PWRREQ_OE;
766         else
767                 reg |= TEGRA_POWER_PWRREQ_OE;
768         reg &= ~TEGRA_POWER_EFFECT_LP0;
769
770         switch (mode) {
771         case TEGRA_SUSPEND_LP0:
772 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
773                 rate = clk_get_rate_all_locked(tegra_pclk);
774 #endif
775                 if (pdata->combined_req) {
776                         reg |= TEGRA_POWER_PWRREQ_OE;
777                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
778                 }
779
780                 /*
781                  * LP0 boots through the AVP, which then resumes the AVP to
782                  * the address in scratch 39, and the cpu to the address in
783                  * scratch 41 to tegra_resume
784                  */
785                 writel(0x0, pmc + PMC_SCRATCH39);
786
787                 /* Enable DPD sample to trigger sampling pads data and direction
788                  * in which pad will be driven during lp0 mode*/
789                 writel(0x1, pmc + PMC_DPD_SAMPLE);
790 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
791                 /* this is needed only for T11x, not for other chips */
792                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
793 #endif
794
795                 /* Set warmboot flag */
796                 boot_flag = readl(pmc + PMC_SCRATCH0);
797                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
798
799                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
800
801                 reg |= TEGRA_POWER_EFFECT_LP0;
802                 /* No break here. LP0 code falls through to write SCRATCH41 */
803         case TEGRA_SUSPEND_LP1:
804                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
805                 wmb();
806                 break;
807         case TEGRA_SUSPEND_LP2:
808                 rate = clk_get_rate(tegra_pclk);
809                 break;
810         case TEGRA_SUSPEND_NONE:
811                 return;
812         default:
813                 BUG();
814         }
815
816         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
817
818         pmc_32kwritel(reg, PMC_CTRL);
819 }
820
821 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
822         [TEGRA_SUSPEND_NONE] = "none",
823         [TEGRA_SUSPEND_LP2] = "LP2",
824         [TEGRA_SUSPEND_LP1] = "LP1",
825         [TEGRA_SUSPEND_LP0] = "LP0",
826 };
827
828 static int tegra_suspend_enter(suspend_state_t state)
829 {
830         int ret;
831         ktime_t delta;
832         struct timespec ts_entry, ts_exit;
833
834         if (pdata && pdata->board_suspend)
835                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
836
837         read_persistent_clock(&ts_entry);
838
839         ret = tegra_suspend_dram(current_suspend_mode, 0);
840         if (ret) {
841                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
842                 goto abort_suspend;
843         }
844
845         read_persistent_clock(&ts_exit);
846
847         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
848                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
849
850                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
851                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
852                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
853                 else
854                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
855         }
856
857 abort_suspend:
858         if (pdata && pdata->board_resume)
859                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
860
861         return ret;
862 }
863
864 static void tegra_suspend_check_pwr_stats(void)
865 {
866         /* cpus and l2 are powered off later */
867         unsigned long pwrgate_partid_mask =
868 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
869                 (1 << TEGRA_POWERGATE_HEG)      |
870                 (1 << TEGRA_POWERGATE_SATA)     |
871                 (1 << TEGRA_POWERGATE_3D1)      |
872 #endif
873                 (1 << TEGRA_POWERGATE_3D)       |
874                 (1 << TEGRA_POWERGATE_VENC)     |
875                 (1 << TEGRA_POWERGATE_PCIE)     |
876                 (1 << TEGRA_POWERGATE_VDEC)     |
877                 (1 << TEGRA_POWERGATE_MPE);
878
879         int partid;
880
881         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
882                 if ((1 << partid) & pwrgate_partid_mask)
883                         if (tegra_powergate_is_powered(partid))
884                                 pr_warning("partition %s is left on before suspend\n",
885                                         tegra_powergate_get_name(partid));
886
887         return;
888 }
889
890 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
891 {
892         int err = 0;
893         u32 scratch37 = 0xDEADBEEF;
894         u32 reg;
895
896         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
897                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
898                 err = -ENXIO;
899                 goto fail;
900         }
901
902         if (tegra_is_voice_call_active()) {
903                 /* backup the current value of scratch37 */
904                 scratch37 = readl(pmc + PMC_SCRATCH37);
905
906                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
907                 reg = TEGRA_POWER_LP1_AUDIO;
908                 pmc_32kwritel(reg, PMC_SCRATCH37);
909         }
910
911         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
912                 pr_info("LP0 not used due to unsupported wakeup events\n");
913                 mode = TEGRA_SUSPEND_LP1;
914         }
915
916         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
917                 tegra_suspend_check_pwr_stats();
918
919         tegra_common_suspend();
920
921         tegra_pm_set(mode);
922
923         if (pdata && pdata->board_suspend)
924                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
925
926         local_fiq_disable();
927
928         trace_cpu_suspend(CPU_SUSPEND_START);
929
930         if (mode == TEGRA_SUSPEND_LP0) {
931 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
932                 reg = readl(pmc + PMC_SCRATCH4);
933                 if (is_lp_cluster())
934                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
935                 else
936                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
937                 pmc_32kwritel(reg, PMC_SCRATCH4);
938 #endif
939                 tegra_tsc_suspend();
940                 tegra_lp0_suspend_mc();
941                 tegra_cpu_reset_handler_save();
942                 tegra_tsc_wait_for_suspend();
943                 tegra_smp_clear_power_mask();
944         }
945         else if (mode == TEGRA_SUSPEND_LP1)
946                 *iram_cpu_lp1_mask = 1;
947
948         suspend_cpu_complex(flags);
949
950 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
951         /* In case of LP0/1, program external power gating accordinly */
952         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
953                 reg = readl(FLOW_CTRL_CPU_CSR(0));
954                 if (is_lp_cluster())
955                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
956                 else
957                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
958                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
959         }
960 #endif
961
962         flush_cache_all();
963         outer_flush_all();
964         outer_disable();
965
966         if (mode == TEGRA_SUSPEND_LP2)
967                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
968         else
969                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
970
971         tegra_init_cache(true);
972
973         if (mode == TEGRA_SUSPEND_LP0) {
974
975                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
976                 * Otherwise it is creating issue in cluster switch after LP0 *
977 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
978                 reg = readl(pmc+PMC_CTRL);
979                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
980                 pmc_32kwritel(reg, PMC_CTRL);
981 #endif
982                 */
983
984                 tegra_tsc_resume();
985                 tegra_cpu_reset_handler_restore();
986                 tegra_lp0_resume_mc();
987                 tegra_tsc_wait_for_resume();
988         } else if (mode == TEGRA_SUSPEND_LP1)
989                 *iram_cpu_lp1_mask = 0;
990
991         /* if scratch37 was clobbered during LP1, restore it */
992         if (scratch37 != 0xDEADBEEF)
993                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
994
995         restore_cpu_complex(flags);
996
997         /* for platforms where the core & CPU power requests are
998          * combined as a single request to the PMU, transition out
999          * of LP0 state by temporarily enabling both requests
1000          */
1001         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1002                 reg = readl(pmc + PMC_CTRL);
1003                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1004                 pmc_32kwritel(reg, PMC_CTRL);
1005                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1006                 pmc_32kwritel(reg, PMC_CTRL);
1007         }
1008
1009         if (pdata && pdata->board_resume)
1010                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1011
1012         trace_cpu_suspend(CPU_SUSPEND_DONE);
1013
1014         local_fiq_enable();
1015
1016         tegra_common_resume();
1017
1018 fail:
1019         return err;
1020 }
1021
1022 /*
1023  * Function pointers to optional board specific function
1024  */
1025 void (*tegra_deep_sleep)(int);
1026 EXPORT_SYMBOL(tegra_deep_sleep);
1027
1028 static int tegra_suspend_prepare(void)
1029 {
1030         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1031                 tegra_deep_sleep(1);
1032         return 0;
1033 }
1034
1035 static void tegra_suspend_finish(void)
1036 {
1037         if (pdata && pdata->cpu_resume_boost) {
1038                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1039                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1040                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1041         }
1042
1043         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1044                 tegra_deep_sleep(0);
1045 }
1046
1047 static const struct platform_suspend_ops tegra_suspend_ops = {
1048         .valid          = suspend_valid_only_mem,
1049         .prepare        = tegra_suspend_prepare,
1050         .finish         = tegra_suspend_finish,
1051         .prepare_late   = tegra_suspend_prepare_late,
1052         .wake           = tegra_suspend_wake,
1053         .enter          = tegra_suspend_enter,
1054 };
1055
1056 static ssize_t suspend_mode_show(struct kobject *kobj,
1057                                         struct kobj_attribute *attr, char *buf)
1058 {
1059         char *start = buf;
1060         char *end = buf + PAGE_SIZE;
1061
1062         start += scnprintf(start, end - start, "%s ", \
1063                                 tegra_suspend_name[current_suspend_mode]);
1064         start += scnprintf(start, end - start, "\n");
1065
1066         return start - buf;
1067 }
1068
1069 static ssize_t suspend_mode_store(struct kobject *kobj,
1070                                         struct kobj_attribute *attr,
1071                                         const char *buf, size_t n)
1072 {
1073         int len;
1074         const char *name_ptr;
1075         enum tegra_suspend_mode new_mode;
1076
1077         name_ptr = buf;
1078         while (*name_ptr && !isspace(*name_ptr))
1079                 name_ptr++;
1080         len = name_ptr - buf;
1081         if (!len)
1082                 goto bad_name;
1083         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1084         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1085                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1086                 pr_info("Illegal tegra suspend state: %s\n", buf);
1087                 goto bad_name;
1088         }
1089
1090         for (new_mode = TEGRA_SUSPEND_NONE; \
1091                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1092                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1093                         current_suspend_mode = new_mode;
1094                         break;
1095                 }
1096         }
1097
1098 bad_name:
1099         return n;
1100 }
1101
1102 static struct kobj_attribute suspend_mode_attribute =
1103         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1104
1105 static struct kobject *suspend_kobj;
1106
1107 static int tegra_pm_enter_suspend(void)
1108 {
1109         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1110         suspend_cpu_dfll_mode();
1111         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1112                 tegra_lp0_cpu_mode(true);
1113         return 0;
1114 }
1115
1116 static void tegra_pm_enter_resume(void)
1117 {
1118         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1119                 tegra_lp0_cpu_mode(false);
1120         resume_cpu_dfll_mode();
1121         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1122 }
1123
1124 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1125         .suspend = tegra_pm_enter_suspend,
1126         .resume = tegra_pm_enter_resume,
1127 };
1128
1129 static __init int tegra_pm_enter_syscore_init(void)
1130 {
1131         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1132         return 0;
1133 }
1134 subsys_initcall(tegra_pm_enter_syscore_init);
1135 #endif
1136
1137 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1138 {
1139         u32 reg;
1140         u32 mode;
1141
1142 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1143         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1144         BUG_ON(IS_ERR(tegra_dfll));
1145 #endif
1146         tegra_pclk = clk_get_sys(NULL, "pclk");
1147         BUG_ON(IS_ERR(tegra_pclk));
1148         pdata = plat;
1149         (void)reg;
1150         (void)mode;
1151
1152         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1153                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1154
1155 #ifndef CONFIG_PM_SLEEP
1156         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1157                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1158                            "disabling suspend\n", __func__);
1159                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1160         }
1161 #else
1162         if (create_suspend_pgtable() < 0) {
1163                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1164                                 __func__);
1165                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1166                 goto fail;
1167         }
1168
1169         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1170             (tegra_revision == TEGRA_REVISION_A01) &&
1171             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1172                 /* Tegra 3 A01 supports only LP1 */
1173                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1174                            "-- disabling LP0\n", __func__);
1175                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1176         }
1177         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1178                 tegra_lp0_vec_relocate) {
1179                 unsigned char *reloc_lp0;
1180                 unsigned long tmp;
1181                 void __iomem *orig;
1182                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1183                                         GFP_KERNEL);
1184                 WARN_ON(!reloc_lp0);
1185                 if (!reloc_lp0) {
1186                         pr_err("%s: Failed to allocate reloc_lp0\n",
1187                                 __func__);
1188                         goto out;
1189                 }
1190
1191                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1192                 WARN_ON(!orig);
1193                 if (!orig) {
1194                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1195                                 __func__, tegra_lp0_vec_start);
1196                         kfree(reloc_lp0);
1197                         goto out;
1198                 }
1199
1200                 tmp = (unsigned long) reloc_lp0;
1201                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1202                 reloc_lp0 = (unsigned char *)tmp;
1203                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1204                 iounmap(orig);
1205                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1206         }
1207
1208 out:
1209         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1210                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1211                            "provided by bootlader -- disabling LP0\n",
1212                            __func__);
1213                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1214         }
1215
1216         iram_save_size = tegra_iram_end() - tegra_iram_start();
1217
1218         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1219         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1220                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1221                        "-- LP0/LP1 unavailable\n", __func__);
1222                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1223         }
1224
1225 #ifdef CONFIG_TEGRA_LP1_950
1226         if (pdata->lp1_lowvolt_support) {
1227                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1228                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1229                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1230
1231                 lp1_core_lowvolt = 0;
1232                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1233                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1234
1235                 lp1_core_highvolt = 0;
1236                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1237                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1238         }
1239 #endif
1240         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1241         /* Initialize scratch registers used for CPU LP2 synchronization */
1242         writel(0, pmc + PMC_SCRATCH37);
1243         writel(0, pmc + PMC_SCRATCH38);
1244         writel(0, pmc + PMC_SCRATCH39);
1245         writel(0, pmc + PMC_SCRATCH41);
1246
1247         /* Always enable CPU power request; just normal polarity is supported */
1248         reg = readl(pmc + PMC_CTRL);
1249         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1250         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1251         pmc_32kwritel(reg, PMC_CTRL);
1252
1253         /* Configure core power request and system clock control if LP0
1254            is supported */
1255         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1256         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1257
1258         reg = readl(pmc + PMC_CTRL);
1259
1260         if (!pdata->sysclkreq_high)
1261                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1262         else
1263                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1264
1265         if (!pdata->corereq_high)
1266                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1267         else
1268                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1269
1270         /* configure output inverters while the request is tristated */
1271         pmc_32kwritel(reg, PMC_CTRL);
1272
1273         /* now enable requests */
1274         reg |= TEGRA_POWER_SYSCLK_OE;
1275         if (!pdata->combined_req)
1276                 reg |= TEGRA_POWER_PWRREQ_OE;
1277         pmc_32kwritel(reg, PMC_CTRL);
1278
1279         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1280                 tegra_lp0_suspend_init();
1281
1282         suspend_set_ops(&tegra_suspend_ops);
1283
1284         /* Create /sys/power/suspend/type */
1285         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1286         if (suspend_kobj) {
1287                 if (sysfs_create_file(suspend_kobj, \
1288                                                 &suspend_mode_attribute.attr))
1289                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1290                                                                 __func__);
1291         }
1292
1293         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1294         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1295
1296         /* clear io dpd settings before kernel */
1297         tegra_bl_io_dpd_cleanup();
1298
1299 fail:
1300 #endif
1301         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1302                 tegra_lp2_in_idle(false);
1303
1304         current_suspend_mode = plat->suspend_mode;
1305 }
1306
1307 unsigned long debug_uart_port_base = 0;
1308 EXPORT_SYMBOL(debug_uart_port_base);
1309
1310 static int tegra_debug_uart_suspend(void)
1311 {
1312         void __iomem *uart;
1313         u32 lcr;
1314
1315         if (!debug_uart_port_base)
1316                 return 0;
1317
1318         uart = IO_ADDRESS(debug_uart_port_base);
1319
1320         lcr = readb(uart + UART_LCR * 4);
1321
1322         tegra_sctx.uart[0] = lcr;
1323         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1324
1325         /* DLAB = 0 */
1326         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1327
1328         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1329
1330         /* DLAB = 1 */
1331         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1332
1333         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1334         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1335
1336         writeb(lcr, uart + UART_LCR * 4);
1337
1338         return 0;
1339 }
1340
1341 static void tegra_debug_uart_resume(void)
1342 {
1343         void __iomem *uart;
1344         u32 lcr;
1345
1346         if (!debug_uart_port_base)
1347                 return;
1348
1349         uart = IO_ADDRESS(debug_uart_port_base);
1350
1351         lcr = tegra_sctx.uart[0];
1352
1353         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1354
1355         /* DLAB = 0 */
1356         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1357
1358         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1359                         uart + UART_FCR * 4);
1360
1361         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1362
1363         /* DLAB = 1 */
1364         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1365
1366         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1367         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1368
1369         writeb(lcr, uart + UART_LCR * 4);
1370 }
1371
1372 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1373         .suspend = tegra_debug_uart_suspend,
1374         .resume = tegra_debug_uart_resume,
1375 };
1376
1377 struct clk *debug_uart_clk = NULL;
1378 EXPORT_SYMBOL(debug_uart_clk);
1379
1380 void tegra_console_uart_suspend(void)
1381 {
1382         if (console_suspend_enabled && debug_uart_clk)
1383                 clk_disable(debug_uart_clk);
1384 }
1385
1386 void tegra_console_uart_resume(void)
1387 {
1388         if (console_suspend_enabled && debug_uart_clk)
1389                 clk_enable(debug_uart_clk);
1390 }
1391
1392 static int tegra_debug_uart_syscore_init(void)
1393 {
1394         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1395         return 0;
1396 }
1397 arch_initcall(tegra_debug_uart_syscore_init);