ARM: T11x: core power off during LP0 suspend
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation. All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58 #include <asm/smp_plat.h>
59
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62 #include <mach/hardware.h>
63
64 #include "board.h"
65 #include "clock.h"
66 #include "common.h"
67 #include "cpuidle.h"
68 #include "fuse.h"
69 #include "gic.h"
70 #include "iomap.h"
71 #include "pm.h"
72 #include "pm-irq.h"
73 #include "reset.h"
74 #include "sleep.h"
75 #include "timer.h"
76 #include "dvfs.h"
77 #include "cpu-tegra.h"
78
79 struct suspend_context {
80         /*
81          * The next 7 values are referenced by offset in __restart_plls
82          * in headsmp-t2.S, and should not be moved
83          */
84         u32 pllx_misc;
85         u32 pllx_base;
86         u32 pllp_misc;
87         u32 pllp_base;
88         u32 pllp_outa;
89         u32 pllp_outb;
90         u32 pll_timeout;
91
92         u32 cpu_burst;
93         u32 clk_csite_src;
94         u32 cclk_divider;
95
96         u32 mc[3];
97         u8 uart[5];
98
99         struct tegra_twd_context twd;
100 #ifdef CONFIG_ARM_ARCH_TIMER
101         struct arch_timer_context arch_timer;
102 #endif
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 void *tegra_cpu_context;        /* non-cacheable page for CPU context */
107 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
108 static pgd_t *tegra_pgd;
109 static DEFINE_SPINLOCK(tegra_lp2_lock);
110 static cpumask_t tegra_in_lp2;
111 static cpumask_t *iram_cpu_lp2_mask;
112 static unsigned long *iram_cpu_lp1_mask;
113 static u8 *iram_save;
114 static unsigned long iram_save_size;
115 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
116 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
117 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
118 static int tegra_last_pclk;
119 #endif
120
121 struct suspend_context tegra_sctx;
122
123 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
124 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
125 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
126 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
127 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
128 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
129 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
130 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
131 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
132
133 #define PMC_CTRL                0x0
134 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
135 #define PMC_WAKE_MASK           0xc
136 #define PMC_WAKE_LEVEL          0x10
137 #define PMC_DPAD_ORIDE          0x1C
138 #define PMC_WAKE_DELAY          0xe0
139 #define PMC_DPD_SAMPLE          0x20
140 #define PMC_IO_DPD_REQ_0        0x1b8
141 #define PMC_IO_DPD2_REQ_0       0X1C0
142
143 #define PMC_WAKE_STATUS         0x14
144 #define PMC_SW_WAKE_STATUS      0x18
145 #define PMC_COREPWRGOOD_TIMER   0x3c
146 #define PMC_CPUPWRGOOD_TIMER    0xc8
147 #define PMC_CPUPWROFF_TIMER     0xcc
148 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
149
150 #define PMC_PWRGATE_TOGGLE      0x30
151 #define PWRGATE_TOGGLE_START    (1 << 8)
152 #define UN_PWRGATE_CPU          \
153         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
154
155 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
156 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
157 #endif
158
159 #define CLK_RESET_CCLK_BURST    0x20
160 #define CLK_RESET_CCLK_DIVIDER  0x24
161 #define CLK_RESET_PLLC_BASE     0x80
162 #define CLK_RESET_PLLM_BASE     0x90
163 #define CLK_RESET_PLLX_BASE     0xe0
164 #define CLK_RESET_PLLX_MISC     0xe4
165 #define CLK_RESET_PLLP_BASE     0xa0
166 #define CLK_RESET_PLLP_OUTA     0xa4
167 #define CLK_RESET_PLLP_OUTB     0xa8
168 #define CLK_RESET_PLLP_MISC     0xac
169
170 #define CLK_RESET_SOURCE_CSITE  0x1d4
171
172 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
173 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
174 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
175 #define CLK_RESET_CCLK_IDLE_POLICY         1
176 #define CLK_RESET_CCLK_RUN_POLICY          2
177 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
178 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
179
180 #define EMC_MRW_0               0x0e8
181 #define EMC_MRW_DEV_SELECTN     30
182 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
183
184 #define MC_SECURITY_START       0x6c
185 #define MC_SECURITY_SIZE        0x70
186 #define MC_SECURITY_CFG2        0x7c
187
188 static struct clk *tegra_pclk;
189 static const struct tegra_suspend_platform_data *pdata;
190 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
191
192 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
193 enum tegra_cluster_switch_time_id {
194         tegra_cluster_switch_time_id_start = 0,
195         tegra_cluster_switch_time_id_prolog,
196         tegra_cluster_switch_time_id_switch,
197         tegra_cluster_switch_time_id_epilog,
198         tegra_cluster_switch_time_id_max
199 };
200
201 static unsigned long
202                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
203 #define tegra_cluster_switch_time(flags, id) \
204         do { \
205                 barrier(); \
206                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
207                         void __iomem *timer_us = \
208                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
209                         if (id < tegra_cluster_switch_time_id_max) \
210                                 tegra_cluster_switch_times[id] = \
211                                                         readl(timer_us); \
212                                 wmb(); \
213                 } \
214                 barrier(); \
215         } while(0)
216 #else
217 #define tegra_cluster_switch_time(flags, id) do {} while(0)
218 #endif
219
220 #ifdef CONFIG_PM_SLEEP
221 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
222         [TEGRA_SUSPEND_NONE]    = "none",
223         [TEGRA_SUSPEND_LP2]     = "lp2",
224         [TEGRA_SUSPEND_LP1]     = "lp1",
225         [TEGRA_SUSPEND_LP0]     = "lp0",
226 };
227
228 unsigned long tegra_cpu_power_good_time(void)
229 {
230         if (WARN_ON_ONCE(!pdata))
231                 return 5000;
232
233         return pdata->cpu_timer;
234 }
235
236 unsigned long tegra_cpu_power_off_time(void)
237 {
238         if (WARN_ON_ONCE(!pdata))
239                 return 5000;
240
241         return pdata->cpu_off_timer;
242 }
243
244 unsigned long tegra_cpu_lp2_min_residency(void)
245 {
246         if (WARN_ON_ONCE(!pdata))
247                 return 2000;
248
249         return pdata->cpu_lp2_min_residency;
250 }
251
252 /*
253  * create_suspend_pgtable
254  *
255  * Creates a page table with identity mappings of physical memory and IRAM
256  * for use when the MMU is off, in addition to all the regular kernel mappings.
257  */
258 static __init int create_suspend_pgtable(void)
259 {
260         tegra_pgd = pgd_alloc(&init_mm);
261         if (!tegra_pgd)
262                 return -ENOMEM;
263
264         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
265         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
266                 high_memory, 0);
267         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
268                 IO_IRAM_VIRT + SECTION_SIZE, 0);
269
270         /* inner/outer write-back/write-allocate, sharable */
271         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
272
273         return 0;
274 }
275 /*
276  * alloc_suspend_context
277  *
278  * Allocate a non-cacheable page to hold the CPU contexts.
279  * The standard ARM CPU context save functions don't work if there's
280  * an external L2 cache controller (like a PL310) in system.
281  */
282 static __init int alloc_suspend_context(void)
283 {
284         pgprot_t prot = __pgprot_modify(pgprot_kernel, L_PTE_MT_MASK,
285                                         L_PTE_MT_BUFFERABLE | L_PTE_XN);
286         struct page *ctx_page;
287
288         ctx_page = alloc_pages(GFP_KERNEL, 0);
289         if (IS_ERR_OR_NULL(ctx_page))
290                 goto fail;
291
292         tegra_cpu_context = vm_map_ram(&ctx_page, 1, -1, prot);
293         if (IS_ERR_OR_NULL(tegra_cpu_context))
294                 goto fail;
295
296         return 0;
297
298 fail:
299         if (!IS_ERR(ctx_page) && ctx_page)
300                 __free_page(ctx_page);
301         if (!IS_ERR(tegra_cpu_context) && tegra_cpu_context)
302                 vm_unmap_ram((void *)tegra_cpu_context, 1);
303         tegra_cpu_context = NULL;
304         return -ENOMEM;
305 }
306
307 /* ensures that sufficient time is passed for a register write to
308  * serialize into the 32KHz domain */
309 static void pmc_32kwritel(u32 val, unsigned long offs)
310 {
311         writel(val, pmc + offs);
312         udelay(130);
313 }
314
315 static void set_power_timers(unsigned long us_on, unsigned long us_off,
316                              long rate)
317 {
318         static unsigned long last_us_off = 0;
319         unsigned long long ticks;
320         unsigned long long pclk;
321
322         if (WARN_ON_ONCE(rate <= 0))
323                 pclk = 100000000;
324         else
325                 pclk = rate;
326
327         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
328                 ticks = (us_on * pclk) + 999999ull;
329                 do_div(ticks, 1000000);
330                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
331
332                 ticks = (us_off * pclk) + 999999ull;
333                 do_div(ticks, 1000000);
334                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
335                 wmb();
336         }
337         tegra_last_pclk = pclk;
338         last_us_off = us_off;
339 }
340
341 /*
342  * restore_cpu_complex
343  *
344  * restores cpu clock setting, clears flow controller
345  *
346  * Always called on CPU 0.
347  */
348 static void restore_cpu_complex(u32 mode)
349 {
350         int cpu = smp_processor_id();
351         unsigned int reg;
352 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
353         unsigned int policy;
354 #endif
355
356         BUG_ON(cpu != 0);
357
358 #ifdef CONFIG_SMP
359         cpu = cpu_logical_map(cpu);
360 #endif
361
362 /*
363  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
364  * or restored by common clock suspend/resume procedures. Hence, we don't
365  * need it here.
366  */
367 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
368         /* Is CPU complex already running on PLLX? */
369         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
370         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
371
372         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
373                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
374         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
375                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
376         else
377                 BUG();
378
379         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
380                 /* restore PLLX settings if CPU is on different PLL */
381                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
382                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
383
384                 /* wait for PLL stabilization if PLLX was enabled */
385                 if (tegra_sctx.pllx_base & (1<<30)) {
386 #if USE_PLL_LOCK_BITS
387                         /* Enable lock detector */
388                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
389                         reg |= 1<<18;
390                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
391                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
392                                  (1<<27)))
393                                 cpu_relax();
394
395                         udelay(PLL_POST_LOCK_DELAY);
396 #else
397                         udelay(300);
398 #endif
399                 }
400         }
401
402         /* Restore original burst policy setting for calls resulting from CPU
403            LP2 in idle or system suspend; keep cluster switch prolog setting
404            intact. */
405         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
406                 writel(tegra_sctx.cclk_divider, clk_rst +
407                        CLK_RESET_CCLK_DIVIDER);
408                 writel(tegra_sctx.cpu_burst, clk_rst +
409                        CLK_RESET_CCLK_BURST);
410         }
411 #endif
412         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
413
414         /* Do not power-gate CPU 0 when flow controlled */
415         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
416         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
417         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
418         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
419         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
420         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
421         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
422
423         /* If an immedidate cluster switch is being perfomed, restore the
424            local timer registers. For calls resulting from CPU LP2 in
425            idle or system suspend, the local timer was shut down and
426            timekeeping switched over to the global system timer. In this
427            case keep local timer disabled, and restore only periodic load. */
428         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
429                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
430 #ifdef CONFIG_ARM_ARCH_TIMER
431                 tegra_sctx.arch_timer.cntp_ctl = 0;
432 #endif
433 #ifdef CONFIG_HAVE_ARM_TWD
434                 tegra_sctx.twd.twd_ctrl = 0;
435 #endif
436         }
437 #ifdef CONFIG_ARM_ARCH_TIMER
438         arch_timer_resume(&tegra_sctx.arch_timer);
439 #endif
440 #ifdef CONFIG_HAVE_ARM_TWD
441         tegra_twd_resume(&tegra_sctx.twd);
442 #endif
443 }
444
445 /*
446  * suspend_cpu_complex
447  *
448  * saves pll state for use by restart_plls, prepares flow controller for
449  * transition to suspend state
450  *
451  * Must always be called on cpu 0.
452  */
453 static void suspend_cpu_complex(u32 mode)
454 {
455         int cpu = smp_processor_id();
456         unsigned int reg;
457         int i;
458
459         BUG_ON(cpu != 0);
460
461 #ifdef CONFIG_SMP
462         cpu = cpu_logical_map(cpu);
463 #endif
464         /* switch coresite to clk_m, save off original source */
465         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
466         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
467
468         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
469         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
470         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
471         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
472         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
473         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
474         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
475         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
476
477 #ifdef CONFIG_HAVE_ARM_TWD
478         tegra_twd_suspend(&tegra_sctx.twd);
479 #endif
480 #ifdef CONFIG_ARM_ARCH_TIMER
481         arch_timer_suspend(&tegra_sctx.arch_timer);
482 #endif
483
484         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
485         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
486         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
487         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
488         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
489 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
490         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
491 #else
492         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
493 #endif
494         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
495         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
496
497         for (i = 0; i < num_possible_cpus(); i++) {
498                 if (i == cpu)
499                         continue;
500                 reg = readl(FLOW_CTRL_CPU_CSR(i));
501                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
502                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
503                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
504         }
505
506         tegra_gic_cpu_disable(true);
507 }
508
509 void tegra_clear_cpu_in_lp2(int cpu)
510 {
511         spin_lock(&tegra_lp2_lock);
512         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
513         cpumask_clear_cpu(cpu, &tegra_in_lp2);
514
515         /* Update the IRAM copy used by the reset handler. The IRAM copy
516            can't use used directly by cpumask_clear_cpu() because it uses
517            LDREX/STREX which requires the addressed location to be inner
518            cacheable and sharable which IRAM isn't. */
519         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
520         dsb();
521
522         spin_unlock(&tegra_lp2_lock);
523 }
524
525 bool tegra_set_cpu_in_lp2(int cpu)
526 {
527         bool last_cpu = false;
528
529         spin_lock(&tegra_lp2_lock);
530         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
531         cpumask_set_cpu(cpu, &tegra_in_lp2);
532
533         /* Update the IRAM copy used by the reset handler. The IRAM copy
534            can't use used directly by cpumask_set_cpu() because it uses
535            LDREX/STREX which requires the addressed location to be inner
536            cacheable and sharable which IRAM isn't. */
537         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
538         dsb();
539
540         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
541                 last_cpu = true;
542 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
543         else if (cpu == 1)
544                 tegra2_cpu_set_resettable_soon();
545 #endif
546
547         spin_unlock(&tegra_lp2_lock);
548         return last_cpu;
549 }
550
551 bool tegra_is_cpu_in_lp2(int cpu)
552 {
553         bool in_lp2;
554
555         spin_lock(&tegra_lp2_lock);
556         in_lp2 = cpumask_test_cpu(cpu, &tegra_in_lp2);
557         spin_unlock(&tegra_lp2_lock);
558         return in_lp2;
559 }
560
561 static void tegra_sleep_core(enum tegra_suspend_mode mode,
562                              unsigned long v2p)
563 {
564 #ifdef CONFIG_TRUSTED_FOUNDATIONS
565         if (mode == TEGRA_SUSPEND_LP0) {
566                 tegra_generic_smc_uncached(0xFFFFFFFC, 0xFFFFFFE3,
567                                   virt_to_phys(tegra_resume));
568         } else {
569                 tegra_generic_smc_uncached(0xFFFFFFFC, 0xFFFFFFE6,
570                                   (TEGRA_RESET_HANDLER_BASE +
571                                    tegra_cpu_reset_handler_offset));
572         }
573 #endif
574 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
575         cpu_suspend(v2p, tegra2_sleep_core_finish);
576 #else
577         cpu_suspend(v2p, tegra3_sleep_core_finish);
578 #endif
579 }
580
581 static inline void tegra_sleep_cpu(unsigned long v2p)
582 {
583 #ifdef CONFIG_TRUSTED_FOUNDATIONS
584         tegra_generic_smc_uncached(0xFFFFFFFC, 0xFFFFFFE4,
585                           (TEGRA_RESET_HANDLER_BASE +
586                            tegra_cpu_reset_handler_offset));
587 #endif
588         cpu_suspend(v2p, tegra_sleep_cpu_finish);
589 }
590
591 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
592 {
593         u32 reg;
594         unsigned int remain;
595         pgd_t *pgd;
596
597         /* Only the last cpu down does the final suspend steps */
598         reg = readl(pmc + PMC_CTRL);
599         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
600         if (pdata->combined_req)
601                 reg &= ~TEGRA_POWER_PWRREQ_OE;
602         else
603                 reg |= TEGRA_POWER_PWRREQ_OE;
604
605         reg &= ~TEGRA_POWER_EFFECT_LP0;
606         writel(reg, pmc + PMC_CTRL);
607
608         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
609
610         /*
611          * We can use clk_get_rate_all_locked() here, because all other cpus
612          * are in LP2 state and irqs are disabled
613          */
614         if (flags & TEGRA_POWER_CLUSTER_MASK) {
615                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
616                 set_power_timers(pdata->cpu_timer, 2,
617                         clk_get_rate_all_locked(tegra_pclk));
618                 if (flags & TEGRA_POWER_CLUSTER_G) {
619                         /*
620                          * To reduce the vdd_cpu up latency when LP->G
621                          * transition. Before the transition, enable
622                          * the vdd_cpu rail.
623                          */
624                         if (is_lp_cluster())
625                                 writel(UN_PWRGATE_CPU,
626                                        pmc + PMC_PWRGATE_TOGGLE);
627                 }
628                 tegra_cluster_switch_prolog(flags);
629         } else {
630                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
631                         clk_get_rate_all_locked(tegra_pclk));
632 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
633                 reg = readl(FLOW_CTRL_CPU_CSR(0));
634                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
635                 if (is_lp_cluster()) {
636                         /* for LP cluster, there is no option for rail gating */
637                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
638                                                 TEGRA_POWER_CLUSTER_PART_MASK)
639                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
640                         else if (flags)
641                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
642                 }
643                 else {
644                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
645                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
646                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
647                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
648                 }
649                 writel(reg, FLOW_CTRL_CPU_CSR(0));
650 #endif
651         }
652
653         if (sleep_time)
654                 tegra_lp2_set_trigger(sleep_time);
655
656         cpu_cluster_pm_enter();
657         suspend_cpu_complex(flags);
658         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
659         flush_cache_all();
660         /*
661          * No need to flush complete L2. Cleaning kernel and IO mappings
662          * is enough for the LP code sequence that has L2 disabled but
663          * MMU on.
664          */
665         pgd = cpu_get_pgd();
666         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
667                           __pa(pgd + PTRS_PER_PGD));
668         outer_disable();
669
670         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
671
672         tegra_init_cache(false);
673         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
674         restore_cpu_complex(flags);
675         cpu_cluster_pm_exit();
676
677         remain = tegra_lp2_timer_remain();
678         if (sleep_time)
679                 tegra_lp2_set_trigger(0);
680
681         if (flags & TEGRA_POWER_CLUSTER_MASK) {
682                 tegra_cluster_switch_epilog(flags);
683                 if (is_idle_task(current))
684                         trace_cpu_cluster_rcuidle(POWER_CPU_CLUSTER_DONE);
685                 else
686                         trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
687         }
688         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
689
690 #if INSTRUMENT_CLUSTER_SWITCH
691         if (flags & TEGRA_POWER_CLUSTER_MASK) {
692                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
693                         is_lp_cluster() ? "G=>LP" : "LP=>G",
694                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
695                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
696                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
697                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
698                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
699                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
700                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
701                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
702         }
703 #endif
704         return remain;
705 }
706
707 static int tegra_common_suspend(void)
708 {
709         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
710
711         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
712         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
713         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
714
715         /* copy the reset vector and SDRAM shutdown code into IRAM */
716         memcpy(iram_save, iram_code, iram_save_size);
717         memcpy(iram_code, tegra_iram_start(), iram_save_size);
718
719         return 0;
720 }
721
722 static void tegra_common_resume(void)
723 {
724         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
725 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
726         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
727 #endif
728
729         /* Clear DPD sample */
730         writel(0x0, pmc + PMC_DPD_SAMPLE);
731
732         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
733         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
734         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
735 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
736         /* trigger emc mode write */
737         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
738 #endif
739         /* clear scratch registers shared by suspend and the reset pen */
740         writel(0x0, pmc + PMC_SCRATCH39);
741         writel(0x0, pmc + PMC_SCRATCH41);
742
743         /* restore IRAM */
744         memcpy(iram_code, iram_save, iram_save_size);
745 }
746
747 static int tegra_suspend_prepare_late(void)
748 {
749 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
750         disable_irq(INT_SYS_STATS_MON);
751 #endif
752         return 0;
753 }
754
755 static void tegra_suspend_wake(void)
756 {
757 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
758         enable_irq(INT_SYS_STATS_MON);
759 #endif
760 }
761
762 static void tegra_pm_set(enum tegra_suspend_mode mode)
763 {
764         u32 reg, boot_flag;
765         unsigned long rate = 32768;
766
767         reg = readl(pmc + PMC_CTRL);
768         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
769         if (pdata->combined_req)
770                 reg &= ~TEGRA_POWER_PWRREQ_OE;
771         else
772                 reg |= TEGRA_POWER_PWRREQ_OE;
773         reg &= ~TEGRA_POWER_EFFECT_LP0;
774
775         switch (mode) {
776         case TEGRA_SUSPEND_LP0:
777 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
778                 rate = clk_get_rate_all_locked(tegra_pclk);
779 #endif
780                 if (pdata->combined_req) {
781                         reg |= TEGRA_POWER_PWRREQ_OE;
782                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
783                 }
784
785                 /*
786                  * LP0 boots through the AVP, which then resumes the AVP to
787                  * the address in scratch 39, and the cpu to the address in
788                  * scratch 41 to tegra_resume
789                  */
790                 writel(0x0, pmc + PMC_SCRATCH39);
791
792                 /* Enable DPD sample to trigger sampling pads data and direction
793                  * in which pad will be driven during lp0 mode*/
794                 writel(0x1, pmc + PMC_DPD_SAMPLE);
795 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
796                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ_0);
797                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ_0);
798 #endif
799 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
800                 /* this is needed only for T11x, not for other chips */
801                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
802 #endif
803
804                 /* Set warmboot flag */
805                 boot_flag = readl(pmc + PMC_SCRATCH0);
806                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
807
808                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
809
810                 reg |= TEGRA_POWER_EFFECT_LP0;
811                 /* No break here. LP0 code falls through to write SCRATCH41 */
812         case TEGRA_SUSPEND_LP1:
813                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
814                 wmb();
815                 break;
816         case TEGRA_SUSPEND_LP2:
817                 rate = clk_get_rate(tegra_pclk);
818                 break;
819         case TEGRA_SUSPEND_NONE:
820                 return;
821         default:
822                 BUG();
823         }
824
825         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
826
827         pmc_32kwritel(reg, PMC_CTRL);
828 }
829
830 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
831         [TEGRA_SUSPEND_NONE] = "none",
832         [TEGRA_SUSPEND_LP2] = "LP2",
833         [TEGRA_SUSPEND_LP1] = "LP1",
834         [TEGRA_SUSPEND_LP0] = "LP0",
835 };
836
837 static int tegra_suspend_enter(suspend_state_t state)
838 {
839         int ret;
840         ktime_t delta;
841         struct timespec ts_entry, ts_exit;
842
843         if (pdata && pdata->board_suspend)
844                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
845
846         read_persistent_clock(&ts_entry);
847
848         ret = tegra_suspend_dram(current_suspend_mode, 0);
849         if (ret) {
850                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
851                 goto abort_suspend;
852         }
853
854         read_persistent_clock(&ts_exit);
855
856         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
857                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
858
859                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
860                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
861                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
862                 else
863                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
864         }
865
866 abort_suspend:
867         if (pdata && pdata->board_resume)
868                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
869
870         return ret;
871 }
872
873 static void tegra_suspend_check_pwr_stats(void)
874 {
875         /* cpus and l2 are powered off later */
876         unsigned long pwrgate_partid_mask =
877 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
878                 (1 << TEGRA_POWERGATE_HEG)      |
879                 (1 << TEGRA_POWERGATE_SATA)     |
880                 (1 << TEGRA_POWERGATE_3D1)      |
881 #endif
882                 (1 << TEGRA_POWERGATE_3D)       |
883                 (1 << TEGRA_POWERGATE_VENC)     |
884                 (1 << TEGRA_POWERGATE_PCIE)     |
885                 (1 << TEGRA_POWERGATE_VDEC)     |
886                 (1 << TEGRA_POWERGATE_MPE);
887
888         int partid;
889
890         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
891                 if ((1 << partid) & pwrgate_partid_mask)
892                         if (tegra_powergate_is_powered(partid))
893                                 pr_warning("partition %s is left on before suspend\n",
894                                         tegra_powergate_get_name(partid));
895
896         return;
897 }
898
899 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
900 {
901         int err = 0;
902         u32 scratch37 = 0xDEADBEEF;
903         u32 reg;
904
905         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
906                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
907                 err = -ENXIO;
908                 goto fail;
909         }
910
911         if (tegra_is_voice_call_active()) {
912                 /* backup the current value of scratch37 */
913                 scratch37 = readl(pmc + PMC_SCRATCH37);
914
915                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
916                 reg = TEGRA_POWER_LP1_AUDIO;
917                 pmc_32kwritel(reg, PMC_SCRATCH37);
918         }
919
920         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
921                 pr_info("LP0 not used due to unsupported wakeup events\n");
922                 mode = TEGRA_SUSPEND_LP1;
923         }
924
925         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
926                 tegra_suspend_check_pwr_stats();
927
928         tegra_common_suspend();
929
930         tegra_pm_set(mode);
931
932         if (pdata && pdata->board_suspend)
933                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
934
935         local_fiq_disable();
936
937         trace_cpu_suspend(CPU_SUSPEND_START);
938
939         if (mode == TEGRA_SUSPEND_LP0) {
940 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
941                 reg = readl(pmc + PMC_SCRATCH4);
942                 if (is_lp_cluster())
943                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
944                 else
945                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
946                 pmc_32kwritel(reg, PMC_SCRATCH4);
947 #endif
948                 tegra_tsc_suspend();
949                 tegra_lp0_suspend_mc();
950                 tegra_cpu_reset_handler_save();
951                 tegra_tsc_wait_for_suspend();
952         }
953         else if (mode == TEGRA_SUSPEND_LP1)
954                 *iram_cpu_lp1_mask = 1;
955
956         suspend_cpu_complex(flags);
957
958 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
959         /* In case of LP0, program external power gating accordinly */
960         if (mode == TEGRA_SUSPEND_LP0) {
961                 reg = readl(FLOW_CTRL_CPU_CSR(0));
962                 if (is_lp_cluster())
963                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
964                 else
965                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
966                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
967         }
968 #endif
969
970         flush_cache_all();
971         outer_flush_all();
972         outer_disable();
973
974         if (mode == TEGRA_SUSPEND_LP2)
975                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
976         else
977                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
978
979         tegra_init_cache(true);
980
981         if (mode == TEGRA_SUSPEND_LP0) {
982 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
983                 reg = readl(pmc+PMC_CTRL);
984                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
985                 pmc_32kwritel(reg, PMC_CTRL);
986 #endif
987                 tegra_tsc_resume();
988                 tegra_cpu_reset_handler_restore();
989                 tegra_lp0_resume_mc();
990                 tegra_tsc_wait_for_resume();
991         } else if (mode == TEGRA_SUSPEND_LP1)
992                 *iram_cpu_lp1_mask = 0;
993
994         /* if scratch37 was clobbered during LP1, restore it */
995         if (scratch37 != 0xDEADBEEF)
996                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
997
998         restore_cpu_complex(flags);
999
1000         /* for platforms where the core & CPU power requests are
1001          * combined as a single request to the PMU, transition out
1002          * of LP0 state by temporarily enabling both requests
1003          */
1004         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1005                 reg = readl(pmc + PMC_CTRL);
1006                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1007                 pmc_32kwritel(reg, PMC_CTRL);
1008                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1009                 pmc_32kwritel(reg, PMC_CTRL);
1010         }
1011
1012         if (pdata && pdata->board_resume)
1013                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1014
1015         trace_cpu_suspend(CPU_SUSPEND_DONE);
1016
1017         local_fiq_enable();
1018
1019         tegra_common_resume();
1020
1021 fail:
1022         return err;
1023 }
1024
1025 /*
1026  * Function pointers to optional board specific function
1027  */
1028 void (*tegra_deep_sleep)(int);
1029 EXPORT_SYMBOL(tegra_deep_sleep);
1030
1031 static int tegra_suspend_prepare(void)
1032 {
1033         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1034                 tegra_deep_sleep(1);
1035         return 0;
1036 }
1037
1038 static void tegra_suspend_finish(void)
1039 {
1040         if (pdata && pdata->cpu_resume_boost) {
1041                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1042                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1043                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1044         }
1045
1046         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1047                 tegra_deep_sleep(0);
1048 }
1049
1050 static const struct platform_suspend_ops tegra_suspend_ops = {
1051         .valid          = suspend_valid_only_mem,
1052         .prepare        = tegra_suspend_prepare,
1053         .finish         = tegra_suspend_finish,
1054         .prepare_late   = tegra_suspend_prepare_late,
1055         .wake           = tegra_suspend_wake,
1056         .enter          = tegra_suspend_enter,
1057 };
1058
1059 static ssize_t suspend_mode_show(struct kobject *kobj,
1060                                         struct kobj_attribute *attr, char *buf)
1061 {
1062         char *start = buf;
1063         char *end = buf + PAGE_SIZE;
1064
1065         start += scnprintf(start, end - start, "%s ", \
1066                                 tegra_suspend_name[current_suspend_mode]);
1067         start += scnprintf(start, end - start, "\n");
1068
1069         return start - buf;
1070 }
1071
1072 static ssize_t suspend_mode_store(struct kobject *kobj,
1073                                         struct kobj_attribute *attr,
1074                                         const char *buf, size_t n)
1075 {
1076         int len;
1077         const char *name_ptr;
1078         enum tegra_suspend_mode new_mode;
1079
1080         name_ptr = buf;
1081         while (*name_ptr && !isspace(*name_ptr))
1082                 name_ptr++;
1083         len = name_ptr - buf;
1084         if (!len)
1085                 goto bad_name;
1086         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1087         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1088                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1089                 pr_info("Illegal tegra suspend state: %s\n", buf);
1090                 goto bad_name;
1091         }
1092
1093         for (new_mode = TEGRA_SUSPEND_NONE; \
1094                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1095                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1096                         current_suspend_mode = new_mode;
1097                         break;
1098                 }
1099         }
1100
1101 bad_name:
1102         return n;
1103 }
1104
1105 static struct kobj_attribute suspend_mode_attribute =
1106         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1107
1108 static struct kobject *suspend_kobj;
1109
1110 static int tegra_pm_enter_suspend(void)
1111 {
1112         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1113         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1114                 tegra_lp0_cpu_mode(true);
1115         return 0;
1116 }
1117
1118 static void tegra_pm_enter_resume(void)
1119 {
1120         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1121                 tegra_lp0_cpu_mode(false);
1122         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1123 }
1124
1125 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1126         .suspend = tegra_pm_enter_suspend,
1127         .resume = tegra_pm_enter_resume,
1128 };
1129
1130 static __init int tegra_pm_enter_syscore_init(void)
1131 {
1132         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1133         return 0;
1134 }
1135 subsys_initcall(tegra_pm_enter_syscore_init);
1136 #endif
1137
1138 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1139 {
1140         u32 reg;
1141         u32 mode;
1142
1143         tegra_pclk = clk_get_sys(NULL, "pclk");
1144         BUG_ON(IS_ERR(tegra_pclk));
1145         pdata = plat;
1146         (void)reg;
1147         (void)mode;
1148
1149         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1150                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1151
1152 #ifndef CONFIG_PM_SLEEP
1153         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1154                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1155                            "disabling suspend\n", __func__);
1156                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1157         }
1158 #else
1159         if (create_suspend_pgtable() < 0) {
1160                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1161                                 __func__);
1162                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1163                 goto fail;
1164         }
1165
1166         if (alloc_suspend_context() < 0) {
1167                 pr_err("%s: alloc_suspend_context failed -- LP0/LP1/LP2 unavailable\n",
1168                                 __func__);
1169                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1170                 goto fail;
1171         }
1172
1173         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1174             (tegra_revision == TEGRA_REVISION_A01) &&
1175             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1176                 /* Tegra 3 A01 supports only LP1 */
1177                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1178                            "-- disabling LP0\n", __func__);
1179                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1180         }
1181         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1182                 tegra_lp0_vec_relocate) {
1183                 unsigned char *reloc_lp0;
1184                 unsigned long tmp;
1185                 void __iomem *orig;
1186                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1187                                         GFP_KERNEL);
1188                 WARN_ON(!reloc_lp0);
1189                 if (!reloc_lp0) {
1190                         pr_err("%s: Failed to allocate reloc_lp0\n",
1191                                 __func__);
1192                         goto out;
1193                 }
1194
1195                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1196                 WARN_ON(!orig);
1197                 if (!orig) {
1198                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1199                                 __func__, tegra_lp0_vec_start);
1200                         kfree(reloc_lp0);
1201                         goto out;
1202                 }
1203
1204                 tmp = (unsigned long) reloc_lp0;
1205                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1206                 reloc_lp0 = (unsigned char *)tmp;
1207                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1208                 iounmap(orig);
1209                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1210         }
1211
1212 out:
1213         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1214                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1215                            "provided by bootlader -- disabling LP0\n",
1216                            __func__);
1217                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1218         }
1219
1220         iram_save_size = tegra_iram_end() - tegra_iram_start();
1221
1222         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1223         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1224                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1225                        "-- LP0/LP1 unavailable\n", __func__);
1226                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1227         }
1228
1229 #ifdef CONFIG_TEGRA_LP1_950
1230         if (pdata->lp1_lowvolt_support) {
1231                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1232                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1233                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1234
1235                 lp1_core_lowvolt = 0;
1236                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1237                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1238
1239                 lp1_core_highvolt = 0;
1240                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1241                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1242         }
1243 #endif
1244         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1245         /* Initialize scratch registers used for CPU LP2 synchronization */
1246         writel(0, pmc + PMC_SCRATCH37);
1247         writel(0, pmc + PMC_SCRATCH38);
1248         writel(0, pmc + PMC_SCRATCH39);
1249         writel(0, pmc + PMC_SCRATCH41);
1250
1251         /* Always enable CPU power request; just normal polarity is supported */
1252         reg = readl(pmc + PMC_CTRL);
1253         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1254         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1255         pmc_32kwritel(reg, PMC_CTRL);
1256
1257         /* Configure core power request and system clock control if LP0
1258            is supported */
1259         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1260         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1261
1262         reg = readl(pmc + PMC_CTRL);
1263
1264         if (!pdata->sysclkreq_high)
1265                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1266         else
1267                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1268
1269         if (!pdata->corereq_high)
1270                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1271         else
1272                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1273
1274         /* configure output inverters while the request is tristated */
1275         pmc_32kwritel(reg, PMC_CTRL);
1276
1277         /* now enable requests */
1278         reg |= TEGRA_POWER_SYSCLK_OE;
1279         if (!pdata->combined_req)
1280                 reg |= TEGRA_POWER_PWRREQ_OE;
1281         pmc_32kwritel(reg, PMC_CTRL);
1282
1283         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1284                 tegra_lp0_suspend_init();
1285
1286         suspend_set_ops(&tegra_suspend_ops);
1287
1288         /* Create /sys/power/suspend/type */
1289         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1290         if (suspend_kobj) {
1291                 if (sysfs_create_file(suspend_kobj, \
1292                                                 &suspend_mode_attribute.attr))
1293                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1294                                                                 __func__);
1295         }
1296
1297         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1298         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1299
1300         /* clear io dpd settings before kernel */
1301         tegra_bl_io_dpd_cleanup();
1302
1303 fail:
1304 #endif
1305         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1306                 tegra_lp2_in_idle(false);
1307
1308         current_suspend_mode = plat->suspend_mode;
1309 }
1310
1311 unsigned long debug_uart_port_base = 0;
1312 EXPORT_SYMBOL(debug_uart_port_base);
1313
1314 static int tegra_debug_uart_suspend(void)
1315 {
1316         void __iomem *uart;
1317         u32 lcr;
1318
1319         if (!debug_uart_port_base)
1320                 return 0;
1321
1322         uart = IO_ADDRESS(debug_uart_port_base);
1323
1324         lcr = readb(uart + UART_LCR * 4);
1325
1326         tegra_sctx.uart[0] = lcr;
1327         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1328
1329         /* DLAB = 0 */
1330         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1331
1332         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1333
1334         /* DLAB = 1 */
1335         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1336
1337         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1338         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1339
1340         writeb(lcr, uart + UART_LCR * 4);
1341
1342         return 0;
1343 }
1344
1345 static void tegra_debug_uart_resume(void)
1346 {
1347         void __iomem *uart;
1348         u32 lcr;
1349
1350         if (!debug_uart_port_base)
1351                 return;
1352
1353         uart = IO_ADDRESS(debug_uart_port_base);
1354
1355         lcr = tegra_sctx.uart[0];
1356
1357         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1358
1359         /* DLAB = 0 */
1360         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1361
1362         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1363                         uart + UART_FCR * 4);
1364
1365         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1366
1367         /* DLAB = 1 */
1368         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1369
1370         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1371         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1372
1373         writeb(lcr, uart + UART_LCR * 4);
1374 }
1375
1376 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1377         .suspend = tegra_debug_uart_suspend,
1378         .resume = tegra_debug_uart_resume,
1379 };
1380
1381 struct clk *debug_uart_clk = NULL;
1382 EXPORT_SYMBOL(debug_uart_clk);
1383
1384 void tegra_console_uart_suspend(void)
1385 {
1386         if (console_suspend_enabled && debug_uart_clk)
1387                 clk_disable(debug_uart_clk);
1388 }
1389
1390 void tegra_console_uart_resume(void)
1391 {
1392         if (console_suspend_enabled && debug_uart_clk)
1393                 clk_enable(debug_uart_clk);
1394 }
1395
1396 static int tegra_debug_uart_syscore_init(void)
1397 {
1398         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1399         return 0;
1400 }
1401 arch_initcall(tegra_debug_uart_syscore_init);