751766879f6b71297540bef4e8363862c4db174d
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58 #include <asm/smp_plat.h>
59
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62 #include <mach/hardware.h>
63
64 #include "board.h"
65 #include "clock.h"
66 #include "common.h"
67 #include "cpuidle.h"
68 #include "fuse.h"
69 #include "gic.h"
70 #include "iomap.h"
71 #include "pm.h"
72 #include "pm-irq.h"
73 #include "reset.h"
74 #include "sleep.h"
75 #include "timer.h"
76 #include "dvfs.h"
77 #include "cpu-tegra.h"
78
79 struct suspend_context {
80         /*
81          * The next 7 values are referenced by offset in __restart_plls
82          * in headsmp-t2.S, and should not be moved
83          */
84         u32 pllx_misc;
85         u32 pllx_base;
86         u32 pllp_misc;
87         u32 pllp_base;
88         u32 pllp_outa;
89         u32 pllp_outb;
90         u32 pll_timeout;
91
92         u32 cpu_burst;
93         u32 clk_csite_src;
94         u32 cclk_divider;
95
96         u32 mc[3];
97         u8 uart[5];
98
99         struct tegra_twd_context twd;
100 #ifdef CONFIG_ARM_ARCH_TIMER
101         struct arch_timer_context arch_timer;
102 #endif
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
107 static pgd_t *tegra_pgd;
108 static DEFINE_SPINLOCK(tegra_lp2_lock);
109 static cpumask_t tegra_in_lp2;
110 static cpumask_t *iram_cpu_lp2_mask;
111 static unsigned long *iram_cpu_lp1_mask;
112 static u8 *iram_save;
113 static unsigned long iram_save_size;
114 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
115 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
116 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
117 static int tegra_last_pclk;
118 #endif
119
120 struct suspend_context tegra_sctx;
121
122 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
123 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
124 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
125 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
126 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
127 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
128 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
129 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
130 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
131
132 #define PMC_CTRL                0x0
133 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
134 #define PMC_WAKE_MASK           0xc
135 #define PMC_WAKE_LEVEL          0x10
136 #define PMC_DPAD_ORIDE          0x1C
137 #define PMC_WAKE_DELAY          0xe0
138 #define PMC_DPD_SAMPLE          0x20
139
140 #define PMC_WAKE_STATUS         0x14
141 #define PMC_SW_WAKE_STATUS      0x18
142 #define PMC_COREPWRGOOD_TIMER   0x3c
143 #define PMC_CPUPWRGOOD_TIMER    0xc8
144 #define PMC_CPUPWROFF_TIMER     0xcc
145 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
146
147 #define PMC_PWRGATE_TOGGLE      0x30
148 #define PWRGATE_TOGGLE_START    (1 << 8)
149 #define UN_PWRGATE_CPU          \
150         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
151
152 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
153 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
154 #endif
155
156 #define CLK_RESET_CCLK_BURST    0x20
157 #define CLK_RESET_CCLK_DIVIDER  0x24
158 #define CLK_RESET_PLLC_BASE     0x80
159 #define CLK_RESET_PLLM_BASE     0x90
160 #define CLK_RESET_PLLX_BASE     0xe0
161 #define CLK_RESET_PLLX_MISC     0xe4
162 #define CLK_RESET_PLLP_BASE     0xa0
163 #define CLK_RESET_PLLP_OUTA     0xa4
164 #define CLK_RESET_PLLP_OUTB     0xa8
165 #define CLK_RESET_PLLP_MISC     0xac
166
167 #define CLK_RESET_SOURCE_CSITE  0x1d4
168
169 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
170 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
171 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
172 #define CLK_RESET_CCLK_IDLE_POLICY         1
173 #define CLK_RESET_CCLK_RUN_POLICY          2
174 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
175 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
176
177 #define EMC_MRW_0               0x0e8
178 #define EMC_MRW_DEV_SELECTN     30
179 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
180
181 #define MC_SECURITY_START       0x6c
182 #define MC_SECURITY_SIZE        0x70
183 #define MC_SECURITY_CFG2        0x7c
184
185 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
186 static struct clk *tegra_dfll;
187 #endif
188 static struct clk *tegra_pclk;
189 static const struct tegra_suspend_platform_data *pdata;
190 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
191
192 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
193 enum tegra_cluster_switch_time_id {
194         tegra_cluster_switch_time_id_start = 0,
195         tegra_cluster_switch_time_id_prolog,
196         tegra_cluster_switch_time_id_switch,
197         tegra_cluster_switch_time_id_epilog,
198         tegra_cluster_switch_time_id_max
199 };
200
201 static unsigned long
202                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
203 #define tegra_cluster_switch_time(flags, id) \
204         do { \
205                 barrier(); \
206                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
207                         void __iomem *timer_us = \
208                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
209                         if (id < tegra_cluster_switch_time_id_max) \
210                                 tegra_cluster_switch_times[id] = \
211                                                         readl(timer_us); \
212                                 wmb(); \
213                 } \
214                 barrier(); \
215         } while(0)
216 #else
217 #define tegra_cluster_switch_time(flags, id) do {} while(0)
218 #endif
219
220 #ifdef CONFIG_PM_SLEEP
221 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
222         [TEGRA_SUSPEND_NONE]    = "none",
223         [TEGRA_SUSPEND_LP2]     = "lp2",
224         [TEGRA_SUSPEND_LP1]     = "lp1",
225         [TEGRA_SUSPEND_LP0]     = "lp0",
226 };
227
228 unsigned long tegra_cpu_power_good_time(void)
229 {
230         if (WARN_ON_ONCE(!pdata))
231                 return 5000;
232
233         return pdata->cpu_timer;
234 }
235
236 unsigned long tegra_cpu_power_off_time(void)
237 {
238         if (WARN_ON_ONCE(!pdata))
239                 return 5000;
240
241         return pdata->cpu_off_timer;
242 }
243
244 unsigned long tegra_cpu_lp2_min_residency(void)
245 {
246         if (WARN_ON_ONCE(!pdata))
247                 return 2000;
248
249         return pdata->cpu_lp2_min_residency;
250 }
251
252 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
253 unsigned long tegra_min_residency_noncpu(void)
254 {
255         return pdata->min_residency_noncpu;
256 }
257
258 unsigned long tegra_min_residency_crail(void)
259 {
260         return pdata->min_residency_crail;
261 }
262 #endif
263
264 static void suspend_cpu_dfll_mode(void)
265 {
266 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
267         /* If DFLL is used as CPU clock source go to open loop mode */
268         if (!is_lp_cluster() && tegra_dfll &&
269             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
270                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
271 #endif
272 }
273
274 static void resume_cpu_dfll_mode(void)
275 {
276 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
277         /* If DFLL is used as CPU clock source restore closed loop mode */
278         if (!is_lp_cluster() && tegra_dfll &&
279             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
280                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
281 #endif
282 }
283
284 /*
285  * create_suspend_pgtable
286  *
287  * Creates a page table with identity mappings of physical memory and IRAM
288  * for use when the MMU is off, in addition to all the regular kernel mappings.
289  */
290 static __init int create_suspend_pgtable(void)
291 {
292         tegra_pgd = pgd_alloc(&init_mm);
293         if (!tegra_pgd)
294                 return -ENOMEM;
295
296         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
297         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
298                 high_memory, 0);
299         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
300                 IO_IRAM_VIRT + SECTION_SIZE, 0);
301
302         /* inner/outer write-back/write-allocate, sharable */
303         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
304
305         return 0;
306 }
307
308 /* ensures that sufficient time is passed for a register write to
309  * serialize into the 32KHz domain */
310 static void pmc_32kwritel(u32 val, unsigned long offs)
311 {
312         writel(val, pmc + offs);
313         udelay(130);
314 }
315
316 static void set_power_timers(unsigned long us_on, unsigned long us_off,
317                              long rate)
318 {
319         static unsigned long last_us_off = 0;
320         unsigned long long ticks;
321         unsigned long long pclk;
322
323         if (WARN_ON_ONCE(rate <= 0))
324                 pclk = 100000000;
325         else
326                 pclk = rate;
327
328         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
329                 ticks = (us_on * pclk) + 999999ull;
330                 do_div(ticks, 1000000);
331                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
332
333                 ticks = (us_off * pclk) + 999999ull;
334                 do_div(ticks, 1000000);
335                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
336                 wmb();
337         }
338         tegra_last_pclk = pclk;
339         last_us_off = us_off;
340 }
341
342 /*
343  * restore_cpu_complex
344  *
345  * restores cpu clock setting, clears flow controller
346  *
347  * Always called on CPU 0.
348  */
349 static void restore_cpu_complex(u32 mode)
350 {
351         int cpu = smp_processor_id();
352         unsigned int reg;
353 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
354         unsigned int policy;
355 #endif
356
357         BUG_ON(cpu != 0);
358
359 #ifdef CONFIG_SMP
360         cpu = cpu_logical_map(cpu);
361 #endif
362
363 /*
364  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
365  * or restored by common clock suspend/resume procedures. Hence, we don't
366  * need it here.
367  */
368 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
369         /* Is CPU complex already running on PLLX? */
370         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
371         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
372
373         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
374                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
375         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
376                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
377         else
378                 BUG();
379
380         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
381                 /* restore PLLX settings if CPU is on different PLL */
382                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
383                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
384
385                 /* wait for PLL stabilization if PLLX was enabled */
386                 if (tegra_sctx.pllx_base & (1<<30)) {
387 #if USE_PLL_LOCK_BITS
388                         /* Enable lock detector */
389                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
390                         reg |= 1<<18;
391                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
392                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
393                                  (1<<27)))
394                                 cpu_relax();
395
396                         udelay(PLL_POST_LOCK_DELAY);
397 #else
398                         udelay(300);
399 #endif
400                 }
401         }
402
403         /* Restore original burst policy setting for calls resulting from CPU
404            LP2 in idle or system suspend; keep cluster switch prolog setting
405            intact. */
406         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
407                 writel(tegra_sctx.cclk_divider, clk_rst +
408                        CLK_RESET_CCLK_DIVIDER);
409                 writel(tegra_sctx.cpu_burst, clk_rst +
410                        CLK_RESET_CCLK_BURST);
411         }
412 #endif
413         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
414
415         /* Do not power-gate CPU 0 when flow controlled */
416         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
417         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
418         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
419         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
420         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
421         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
422         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
423
424         /* If an immedidate cluster switch is being perfomed, restore the
425            local timer registers. For calls resulting from CPU LP2 in
426            idle or system suspend, the local timer was shut down and
427            timekeeping switched over to the global system timer. In this
428            case keep local timer disabled, and restore only periodic load. */
429         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
430                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
431 #ifdef CONFIG_ARM_ARCH_TIMER
432                 tegra_sctx.arch_timer.cntp_ctl = 0;
433 #endif
434 #ifdef CONFIG_HAVE_ARM_TWD
435                 tegra_sctx.twd.twd_ctrl = 0;
436 #endif
437         }
438 #ifdef CONFIG_ARM_ARCH_TIMER
439         arch_timer_resume(&tegra_sctx.arch_timer);
440 #endif
441 #ifdef CONFIG_HAVE_ARM_TWD
442         tegra_twd_resume(&tegra_sctx.twd);
443 #endif
444 }
445
446 /*
447  * suspend_cpu_complex
448  *
449  * saves pll state for use by restart_plls, prepares flow controller for
450  * transition to suspend state
451  *
452  * Must always be called on cpu 0.
453  */
454 static void suspend_cpu_complex(u32 mode)
455 {
456         int cpu = smp_processor_id();
457         unsigned int reg;
458         int i;
459
460         BUG_ON(cpu != 0);
461
462 #ifdef CONFIG_SMP
463         cpu = cpu_logical_map(cpu);
464 #endif
465         /* switch coresite to clk_m, save off original source */
466         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
467         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
468
469         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
470         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
471         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
472         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
473         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
474         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
475         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
476         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
477
478 #ifdef CONFIG_HAVE_ARM_TWD
479         tegra_twd_suspend(&tegra_sctx.twd);
480 #endif
481 #ifdef CONFIG_ARM_ARCH_TIMER
482         arch_timer_suspend(&tegra_sctx.arch_timer);
483 #endif
484
485         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
486         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
487         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
488         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
489         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
490 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
491         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
492 #else
493         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
494 #endif
495         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
496         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
497
498         for (i = 0; i < num_possible_cpus(); i++) {
499                 if (i == cpu)
500                         continue;
501                 reg = readl(FLOW_CTRL_CPU_CSR(i));
502                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
503                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
504                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
505         }
506
507         tegra_gic_cpu_disable(true);
508 }
509
510 void tegra_clear_cpu_in_lp2(int cpu)
511 {
512         spin_lock(&tegra_lp2_lock);
513         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
514         cpumask_clear_cpu(cpu, &tegra_in_lp2);
515
516         /* Update the IRAM copy used by the reset handler. The IRAM copy
517            can't use used directly by cpumask_clear_cpu() because it uses
518            LDREX/STREX which requires the addressed location to be inner
519            cacheable and sharable which IRAM isn't. */
520         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
521         dsb();
522
523         spin_unlock(&tegra_lp2_lock);
524 }
525
526 bool tegra_set_cpu_in_lp2(int cpu)
527 {
528         bool last_cpu = false;
529
530         spin_lock(&tegra_lp2_lock);
531         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
532         cpumask_set_cpu(cpu, &tegra_in_lp2);
533
534         /* Update the IRAM copy used by the reset handler. The IRAM copy
535            can't use used directly by cpumask_set_cpu() because it uses
536            LDREX/STREX which requires the addressed location to be inner
537            cacheable and sharable which IRAM isn't. */
538         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
539         dsb();
540
541         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
542                 last_cpu = true;
543 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
544         else if (cpu == 1)
545                 tegra2_cpu_set_resettable_soon();
546 #endif
547
548         spin_unlock(&tegra_lp2_lock);
549         return last_cpu;
550 }
551
552 static void tegra_sleep_core(enum tegra_suspend_mode mode,
553                              unsigned long v2p)
554 {
555 #ifdef CONFIG_TRUSTED_FOUNDATIONS
556         if (mode == TEGRA_SUSPEND_LP0) {
557                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
558                                   virt_to_phys(tegra_resume));
559         } else {
560                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
561                                   (TEGRA_RESET_HANDLER_BASE +
562                                    tegra_cpu_reset_handler_offset));
563         }
564 #endif
565 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
566         cpu_suspend(v2p, tegra2_sleep_core_finish);
567 #else
568         cpu_suspend(v2p, tegra3_sleep_core_finish);
569 #endif
570 }
571
572 static inline void tegra_sleep_cpu(unsigned long v2p)
573 {
574 #ifdef CONFIG_TRUSTED_FOUNDATIONS
575         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
576                           (TEGRA_RESET_HANDLER_BASE +
577                            tegra_cpu_reset_handler_offset));
578 #endif
579         cpu_suspend(v2p, tegra_sleep_cpu_finish);
580 }
581
582 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
583 {
584         u32 reg;
585         unsigned int remain;
586 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
587         pgd_t *pgd;
588 #endif
589
590         /* Only the last cpu down does the final suspend steps */
591         reg = readl(pmc + PMC_CTRL);
592         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
593         if (pdata->combined_req)
594                 reg &= ~TEGRA_POWER_PWRREQ_OE;
595         else
596                 reg |= TEGRA_POWER_PWRREQ_OE;
597
598         reg &= ~TEGRA_POWER_EFFECT_LP0;
599         writel(reg, pmc + PMC_CTRL);
600
601         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
602
603         /*
604          * We can use clk_get_rate_all_locked() here, because all other cpus
605          * are in LP2 state and irqs are disabled
606          */
607         if (flags & TEGRA_POWER_CLUSTER_MASK) {
608                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
609                 set_power_timers(pdata->cpu_timer, 2,
610                         clk_get_rate_all_locked(tegra_pclk));
611                 if (flags & TEGRA_POWER_CLUSTER_G) {
612                         /*
613                          * To reduce the vdd_cpu up latency when LP->G
614                          * transition. Before the transition, enable
615                          * the vdd_cpu rail.
616                          */
617                         if (is_lp_cluster()) {
618 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
619                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
620                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
621                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
622 #else
623                                 writel(UN_PWRGATE_CPU,
624                                        pmc + PMC_PWRGATE_TOGGLE);
625 #endif
626                         }
627                 }
628                 tegra_cluster_switch_prolog(flags);
629         } else {
630                 suspend_cpu_dfll_mode();
631                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
632                         clk_get_rate_all_locked(tegra_pclk));
633 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
634                 reg = readl(FLOW_CTRL_CPU_CSR(0));
635                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
636                 if (is_lp_cluster()) {
637                         /* for LP cluster, there is no option for rail gating */
638                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
639                                                 TEGRA_POWER_CLUSTER_PART_MASK)
640                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
641                         else if (flags)
642                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
643                 }
644                 else {
645                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
646                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
647                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
648                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
649                 }
650                 writel(reg, FLOW_CTRL_CPU_CSR(0));
651 #endif
652         }
653
654         if (sleep_time)
655                 tegra_lp2_set_trigger(sleep_time);
656
657         cpu_cluster_pm_enter();
658         suspend_cpu_complex(flags);
659         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
660 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
661         flush_cache_all();
662         /*
663          * No need to flush complete L2. Cleaning kernel and IO mappings
664          * is enough for the LP code sequence that has L2 disabled but
665          * MMU on.
666          */
667         pgd = cpu_get_pgd();
668         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
669                           __pa(pgd + PTRS_PER_PGD));
670         outer_disable();
671 #endif
672         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
673
674         tegra_init_cache(false);
675         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
676         restore_cpu_complex(flags);
677         cpu_cluster_pm_exit();
678
679         remain = tegra_lp2_timer_remain();
680         if (sleep_time)
681                 tegra_lp2_set_trigger(0);
682
683         if (flags & TEGRA_POWER_CLUSTER_MASK) {
684                 tegra_cluster_switch_epilog(flags);
685                 if (is_idle_task(current))
686                         trace_cpu_cluster_rcuidle(POWER_CPU_CLUSTER_DONE);
687                 else
688                         trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
689         } else {
690                 resume_cpu_dfll_mode();
691         }
692         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
693
694 #if INSTRUMENT_CLUSTER_SWITCH
695         if (flags & TEGRA_POWER_CLUSTER_MASK) {
696                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
697                         is_lp_cluster() ? "G=>LP" : "LP=>G",
698                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
699                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
700                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
701                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
702                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
703                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
704                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
705                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
706         }
707 #endif
708         return remain;
709 }
710
711 static int tegra_common_suspend(void)
712 {
713         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
714
715         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
716         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
717         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
718
719         /* copy the reset vector and SDRAM shutdown code into IRAM */
720         memcpy(iram_save, iram_code, iram_save_size);
721         memcpy(iram_code, tegra_iram_start(), iram_save_size);
722
723         return 0;
724 }
725
726 static void tegra_common_resume(void)
727 {
728         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
729 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
730         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
731 #endif
732
733         /* Clear DPD sample */
734         writel(0x0, pmc + PMC_DPD_SAMPLE);
735
736         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
737         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
738         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
739 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
740         /* trigger emc mode write */
741         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
742 #endif
743         /* clear scratch registers shared by suspend and the reset pen */
744         writel(0x0, pmc + PMC_SCRATCH39);
745         writel(0x0, pmc + PMC_SCRATCH41);
746
747         /* restore IRAM */
748         memcpy(iram_code, iram_save, iram_save_size);
749 }
750
751 static int tegra_suspend_prepare_late(void)
752 {
753 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
754         disable_irq(INT_SYS_STATS_MON);
755 #endif
756         return 0;
757 }
758
759 static void tegra_suspend_wake(void)
760 {
761 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
762         enable_irq(INT_SYS_STATS_MON);
763 #endif
764 }
765
766 static void tegra_pm_set(enum tegra_suspend_mode mode)
767 {
768         u32 reg, boot_flag;
769         unsigned long rate = 32768;
770
771         reg = readl(pmc + PMC_CTRL);
772         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
773         if (pdata->combined_req)
774                 reg &= ~TEGRA_POWER_PWRREQ_OE;
775         else
776                 reg |= TEGRA_POWER_PWRREQ_OE;
777         reg &= ~TEGRA_POWER_EFFECT_LP0;
778
779         switch (mode) {
780         case TEGRA_SUSPEND_LP0:
781 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
782                 rate = clk_get_rate_all_locked(tegra_pclk);
783 #endif
784                 if (pdata->combined_req) {
785                         reg |= TEGRA_POWER_PWRREQ_OE;
786                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
787                 }
788
789                 /*
790                  * LP0 boots through the AVP, which then resumes the AVP to
791                  * the address in scratch 39, and the cpu to the address in
792                  * scratch 41 to tegra_resume
793                  */
794                 writel(0x0, pmc + PMC_SCRATCH39);
795
796                 /* Enable DPD sample to trigger sampling pads data and direction
797                  * in which pad will be driven during lp0 mode*/
798                 writel(0x1, pmc + PMC_DPD_SAMPLE);
799 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
800                 /* this is needed only for T11x, not for other chips */
801                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
802 #endif
803
804                 /* Set warmboot flag */
805                 boot_flag = readl(pmc + PMC_SCRATCH0);
806                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
807
808                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
809
810                 reg |= TEGRA_POWER_EFFECT_LP0;
811                 /* No break here. LP0 code falls through to write SCRATCH41 */
812         case TEGRA_SUSPEND_LP1:
813                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
814                 wmb();
815                 break;
816         case TEGRA_SUSPEND_LP2:
817                 rate = clk_get_rate(tegra_pclk);
818                 break;
819         case TEGRA_SUSPEND_NONE:
820                 return;
821         default:
822                 BUG();
823         }
824
825         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
826
827         pmc_32kwritel(reg, PMC_CTRL);
828 }
829
830 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
831         [TEGRA_SUSPEND_NONE] = "none",
832         [TEGRA_SUSPEND_LP2] = "LP2",
833         [TEGRA_SUSPEND_LP1] = "LP1",
834         [TEGRA_SUSPEND_LP0] = "LP0",
835 };
836
837 static int tegra_suspend_enter(suspend_state_t state)
838 {
839         int ret;
840         ktime_t delta;
841         struct timespec ts_entry, ts_exit;
842
843         if (pdata && pdata->board_suspend)
844                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
845
846         read_persistent_clock(&ts_entry);
847
848         ret = tegra_suspend_dram(current_suspend_mode, 0);
849         if (ret) {
850                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
851                 goto abort_suspend;
852         }
853
854         read_persistent_clock(&ts_exit);
855
856         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
857                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
858
859                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
860                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
861                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
862                 else
863                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
864         }
865
866 abort_suspend:
867         if (pdata && pdata->board_resume)
868                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
869
870         return ret;
871 }
872
873 static void tegra_suspend_check_pwr_stats(void)
874 {
875         /* cpus and l2 are powered off later */
876         unsigned long pwrgate_partid_mask =
877 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
878                 (1 << TEGRA_POWERGATE_HEG)      |
879                 (1 << TEGRA_POWERGATE_SATA)     |
880                 (1 << TEGRA_POWERGATE_3D1)      |
881 #endif
882                 (1 << TEGRA_POWERGATE_3D)       |
883                 (1 << TEGRA_POWERGATE_VENC)     |
884                 (1 << TEGRA_POWERGATE_PCIE)     |
885                 (1 << TEGRA_POWERGATE_VDEC)     |
886                 (1 << TEGRA_POWERGATE_MPE);
887
888         int partid;
889
890         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
891                 if ((1 << partid) & pwrgate_partid_mask)
892                         if (tegra_powergate_is_powered(partid))
893                                 pr_warning("partition %s is left on before suspend\n",
894                                         tegra_powergate_get_name(partid));
895
896         return;
897 }
898
899 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
900 {
901         int err = 0;
902         u32 scratch37 = 0xDEADBEEF;
903         u32 reg;
904
905         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
906                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
907                 err = -ENXIO;
908                 goto fail;
909         }
910
911         if (tegra_is_voice_call_active()) {
912                 /* backup the current value of scratch37 */
913                 scratch37 = readl(pmc + PMC_SCRATCH37);
914
915                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
916                 reg = TEGRA_POWER_LP1_AUDIO;
917                 pmc_32kwritel(reg, PMC_SCRATCH37);
918         }
919
920         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
921                 pr_info("LP0 not used due to unsupported wakeup events\n");
922                 mode = TEGRA_SUSPEND_LP1;
923         }
924
925         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
926                 tegra_suspend_check_pwr_stats();
927
928         tegra_common_suspend();
929
930         tegra_pm_set(mode);
931
932         if (pdata && pdata->board_suspend)
933                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
934
935         local_fiq_disable();
936
937         trace_cpu_suspend(CPU_SUSPEND_START);
938
939         if (mode == TEGRA_SUSPEND_LP0) {
940 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
941                 reg = readl(pmc + PMC_SCRATCH4);
942                 if (is_lp_cluster())
943                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
944                 else
945                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
946                 pmc_32kwritel(reg, PMC_SCRATCH4);
947 #endif
948                 tegra_tsc_suspend();
949                 tegra_lp0_suspend_mc();
950                 tegra_cpu_reset_handler_save();
951                 tegra_tsc_wait_for_suspend();
952                 tegra_smp_clear_power_mask();
953         }
954         else if (mode == TEGRA_SUSPEND_LP1)
955                 *iram_cpu_lp1_mask = 1;
956
957         suspend_cpu_complex(flags);
958
959 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
960         /* In case of LP0/1, program external power gating accordinly */
961         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
962                 reg = readl(FLOW_CTRL_CPU_CSR(0));
963                 if (is_lp_cluster())
964                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
965                 else
966                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
967                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
968         }
969 #endif
970
971         flush_cache_all();
972         outer_flush_all();
973         outer_disable();
974
975         if (mode == TEGRA_SUSPEND_LP2)
976                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
977         else
978                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
979
980         tegra_init_cache(true);
981
982         if (mode == TEGRA_SUSPEND_LP0) {
983
984                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
985                 * Otherwise it is creating issue in cluster switch after LP0 *
986 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
987                 reg = readl(pmc+PMC_CTRL);
988                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
989                 pmc_32kwritel(reg, PMC_CTRL);
990 #endif
991                 */
992
993                 tegra_tsc_resume();
994                 tegra_cpu_reset_handler_restore();
995                 tegra_lp0_resume_mc();
996                 tegra_tsc_wait_for_resume();
997         } else if (mode == TEGRA_SUSPEND_LP1)
998                 *iram_cpu_lp1_mask = 0;
999
1000         /* if scratch37 was clobbered during LP1, restore it */
1001         if (scratch37 != 0xDEADBEEF)
1002                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1003
1004         restore_cpu_complex(flags);
1005
1006         /* for platforms where the core & CPU power requests are
1007          * combined as a single request to the PMU, transition out
1008          * of LP0 state by temporarily enabling both requests
1009          */
1010         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1011                 reg = readl(pmc + PMC_CTRL);
1012                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1013                 pmc_32kwritel(reg, PMC_CTRL);
1014                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1015                 pmc_32kwritel(reg, PMC_CTRL);
1016         }
1017
1018         if (pdata && pdata->board_resume)
1019                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1020
1021         trace_cpu_suspend(CPU_SUSPEND_DONE);
1022
1023         local_fiq_enable();
1024
1025         tegra_common_resume();
1026
1027 fail:
1028         return err;
1029 }
1030
1031 /*
1032  * Function pointers to optional board specific function
1033  */
1034 void (*tegra_deep_sleep)(int);
1035 EXPORT_SYMBOL(tegra_deep_sleep);
1036
1037 static int tegra_suspend_prepare(void)
1038 {
1039         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1040                 tegra_deep_sleep(1);
1041         return 0;
1042 }
1043
1044 static void tegra_suspend_finish(void)
1045 {
1046         if (pdata && pdata->cpu_resume_boost) {
1047                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1048                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1049                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1050         }
1051
1052         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1053                 tegra_deep_sleep(0);
1054 }
1055
1056 static const struct platform_suspend_ops tegra_suspend_ops = {
1057         .valid          = suspend_valid_only_mem,
1058         .prepare        = tegra_suspend_prepare,
1059         .finish         = tegra_suspend_finish,
1060         .prepare_late   = tegra_suspend_prepare_late,
1061         .wake           = tegra_suspend_wake,
1062         .enter          = tegra_suspend_enter,
1063 };
1064
1065 static ssize_t suspend_mode_show(struct kobject *kobj,
1066                                         struct kobj_attribute *attr, char *buf)
1067 {
1068         char *start = buf;
1069         char *end = buf + PAGE_SIZE;
1070
1071         start += scnprintf(start, end - start, "%s ", \
1072                                 tegra_suspend_name[current_suspend_mode]);
1073         start += scnprintf(start, end - start, "\n");
1074
1075         return start - buf;
1076 }
1077
1078 static ssize_t suspend_mode_store(struct kobject *kobj,
1079                                         struct kobj_attribute *attr,
1080                                         const char *buf, size_t n)
1081 {
1082         int len;
1083         const char *name_ptr;
1084         enum tegra_suspend_mode new_mode;
1085
1086         name_ptr = buf;
1087         while (*name_ptr && !isspace(*name_ptr))
1088                 name_ptr++;
1089         len = name_ptr - buf;
1090         if (!len)
1091                 goto bad_name;
1092         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1093         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1094                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1095                 pr_info("Illegal tegra suspend state: %s\n", buf);
1096                 goto bad_name;
1097         }
1098
1099         for (new_mode = TEGRA_SUSPEND_NONE; \
1100                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1101                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1102                         current_suspend_mode = new_mode;
1103                         break;
1104                 }
1105         }
1106
1107 bad_name:
1108         return n;
1109 }
1110
1111 static struct kobj_attribute suspend_mode_attribute =
1112         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1113
1114 static struct kobject *suspend_kobj;
1115
1116 static int tegra_pm_enter_suspend(void)
1117 {
1118         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1119         suspend_cpu_dfll_mode();
1120         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1121                 tegra_lp0_cpu_mode(true);
1122         return 0;
1123 }
1124
1125 static void tegra_pm_enter_resume(void)
1126 {
1127         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1128                 tegra_lp0_cpu_mode(false);
1129         resume_cpu_dfll_mode();
1130         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1131 }
1132
1133 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1134         .suspend = tegra_pm_enter_suspend,
1135         .resume = tegra_pm_enter_resume,
1136 };
1137
1138 static __init int tegra_pm_enter_syscore_init(void)
1139 {
1140         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1141         return 0;
1142 }
1143 subsys_initcall(tegra_pm_enter_syscore_init);
1144 #endif
1145
1146 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1147 {
1148         u32 reg;
1149         u32 mode;
1150
1151 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1152         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1153         BUG_ON(IS_ERR(tegra_dfll));
1154 #endif
1155         tegra_pclk = clk_get_sys(NULL, "pclk");
1156         BUG_ON(IS_ERR(tegra_pclk));
1157         pdata = plat;
1158         (void)reg;
1159         (void)mode;
1160
1161         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1162                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1163
1164 #ifndef CONFIG_PM_SLEEP
1165         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1166                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1167                            "disabling suspend\n", __func__);
1168                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1169         }
1170 #else
1171         if (create_suspend_pgtable() < 0) {
1172                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1173                                 __func__);
1174                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1175                 goto fail;
1176         }
1177
1178         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1179             (tegra_revision == TEGRA_REVISION_A01) &&
1180             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1181                 /* Tegra 3 A01 supports only LP1 */
1182                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1183                            "-- disabling LP0\n", __func__);
1184                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1185         }
1186         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1187                 tegra_lp0_vec_relocate) {
1188                 unsigned char *reloc_lp0;
1189                 unsigned long tmp;
1190                 void __iomem *orig;
1191                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1192                                         GFP_KERNEL);
1193                 WARN_ON(!reloc_lp0);
1194                 if (!reloc_lp0) {
1195                         pr_err("%s: Failed to allocate reloc_lp0\n",
1196                                 __func__);
1197                         goto out;
1198                 }
1199
1200                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1201                 WARN_ON(!orig);
1202                 if (!orig) {
1203                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1204                                 __func__, tegra_lp0_vec_start);
1205                         kfree(reloc_lp0);
1206                         goto out;
1207                 }
1208
1209                 tmp = (unsigned long) reloc_lp0;
1210                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1211                 reloc_lp0 = (unsigned char *)tmp;
1212                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1213                 iounmap(orig);
1214                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1215         }
1216
1217 out:
1218         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1219                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1220                            "provided by bootlader -- disabling LP0\n",
1221                            __func__);
1222                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1223         }
1224
1225         iram_save_size = tegra_iram_end() - tegra_iram_start();
1226
1227         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1228         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1229                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1230                        "-- LP0/LP1 unavailable\n", __func__);
1231                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1232         }
1233
1234 #ifdef CONFIG_TEGRA_LP1_950
1235         if (pdata->lp1_lowvolt_support) {
1236                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1237                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1238                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1239
1240                 lp1_core_lowvolt = 0;
1241                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1242                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1243
1244                 lp1_core_highvolt = 0;
1245                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1246                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1247         }
1248 #endif
1249         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1250         /* Initialize scratch registers used for CPU LP2 synchronization */
1251         writel(0, pmc + PMC_SCRATCH37);
1252         writel(0, pmc + PMC_SCRATCH38);
1253         writel(0, pmc + PMC_SCRATCH39);
1254         writel(0, pmc + PMC_SCRATCH41);
1255
1256         /* Always enable CPU power request; just normal polarity is supported */
1257         reg = readl(pmc + PMC_CTRL);
1258         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1259         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1260         pmc_32kwritel(reg, PMC_CTRL);
1261
1262         /* Configure core power request and system clock control if LP0
1263            is supported */
1264         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1265         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1266
1267         reg = readl(pmc + PMC_CTRL);
1268
1269         if (!pdata->sysclkreq_high)
1270                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1271         else
1272                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1273
1274         if (!pdata->corereq_high)
1275                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1276         else
1277                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1278
1279         /* configure output inverters while the request is tristated */
1280         pmc_32kwritel(reg, PMC_CTRL);
1281
1282         /* now enable requests */
1283         reg |= TEGRA_POWER_SYSCLK_OE;
1284         if (!pdata->combined_req)
1285                 reg |= TEGRA_POWER_PWRREQ_OE;
1286         pmc_32kwritel(reg, PMC_CTRL);
1287
1288         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1289                 tegra_lp0_suspend_init();
1290
1291         suspend_set_ops(&tegra_suspend_ops);
1292
1293         /* Create /sys/power/suspend/type */
1294         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1295         if (suspend_kobj) {
1296                 if (sysfs_create_file(suspend_kobj, \
1297                                                 &suspend_mode_attribute.attr))
1298                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1299                                                                 __func__);
1300         }
1301
1302         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1303         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1304
1305         /* clear io dpd settings before kernel */
1306         tegra_bl_io_dpd_cleanup();
1307
1308 fail:
1309 #endif
1310         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1311                 tegra_lp2_in_idle(false);
1312
1313         current_suspend_mode = plat->suspend_mode;
1314 }
1315
1316 unsigned long debug_uart_port_base = 0;
1317 EXPORT_SYMBOL(debug_uart_port_base);
1318
1319 static int tegra_debug_uart_suspend(void)
1320 {
1321         void __iomem *uart;
1322         u32 lcr;
1323
1324         if (!debug_uart_port_base)
1325                 return 0;
1326
1327         uart = IO_ADDRESS(debug_uart_port_base);
1328
1329         lcr = readb(uart + UART_LCR * 4);
1330
1331         tegra_sctx.uart[0] = lcr;
1332         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1333
1334         /* DLAB = 0 */
1335         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1336
1337         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1338
1339         /* DLAB = 1 */
1340         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1341
1342         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1343         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1344
1345         writeb(lcr, uart + UART_LCR * 4);
1346
1347         return 0;
1348 }
1349
1350 static void tegra_debug_uart_resume(void)
1351 {
1352         void __iomem *uart;
1353         u32 lcr;
1354
1355         if (!debug_uart_port_base)
1356                 return;
1357
1358         uart = IO_ADDRESS(debug_uart_port_base);
1359
1360         lcr = tegra_sctx.uart[0];
1361
1362         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1363
1364         /* DLAB = 0 */
1365         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1366
1367         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1368                         uart + UART_FCR * 4);
1369
1370         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1371
1372         /* DLAB = 1 */
1373         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1374
1375         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1376         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1377
1378         writeb(lcr, uart + UART_LCR * 4);
1379 }
1380
1381 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1382         .suspend = tegra_debug_uart_suspend,
1383         .resume = tegra_debug_uart_resume,
1384 };
1385
1386 struct clk *debug_uart_clk = NULL;
1387 EXPORT_SYMBOL(debug_uart_clk);
1388
1389 void tegra_console_uart_suspend(void)
1390 {
1391         if (console_suspend_enabled && debug_uart_clk)
1392                 clk_disable(debug_uart_clk);
1393 }
1394
1395 void tegra_console_uart_resume(void)
1396 {
1397         if (console_suspend_enabled && debug_uart_clk)
1398                 clk_enable(debug_uart_clk);
1399 }
1400
1401 static int tegra_debug_uart_syscore_init(void)
1402 {
1403         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1404         return 0;
1405 }
1406 arch_initcall(tegra_debug_uart_syscore_init);