ARM: tegra: power: Suspend/resume cpu dfll mode
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58 #include <asm/smp_plat.h>
59
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62 #include <mach/hardware.h>
63
64 #include "board.h"
65 #include "clock.h"
66 #include "common.h"
67 #include "cpuidle.h"
68 #include "fuse.h"
69 #include "gic.h"
70 #include "iomap.h"
71 #include "pm.h"
72 #include "pm-irq.h"
73 #include "reset.h"
74 #include "sleep.h"
75 #include "timer.h"
76 #include "dvfs.h"
77 #include "cpu-tegra.h"
78
79 struct suspend_context {
80         /*
81          * The next 7 values are referenced by offset in __restart_plls
82          * in headsmp-t2.S, and should not be moved
83          */
84         u32 pllx_misc;
85         u32 pllx_base;
86         u32 pllp_misc;
87         u32 pllp_base;
88         u32 pllp_outa;
89         u32 pllp_outb;
90         u32 pll_timeout;
91
92         u32 cpu_burst;
93         u32 clk_csite_src;
94         u32 cclk_divider;
95
96         u32 mc[3];
97         u8 uart[5];
98
99         struct tegra_twd_context twd;
100 #ifdef CONFIG_ARM_ARCH_TIMER
101         struct arch_timer_context arch_timer;
102 #endif
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
107 static pgd_t *tegra_pgd;
108 static DEFINE_SPINLOCK(tegra_lp2_lock);
109 static cpumask_t tegra_in_lp2;
110 static cpumask_t *iram_cpu_lp2_mask;
111 static unsigned long *iram_cpu_lp1_mask;
112 static u8 *iram_save;
113 static unsigned long iram_save_size;
114 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
115 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
116 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
117 static int tegra_last_pclk;
118 #endif
119
120 struct suspend_context tegra_sctx;
121
122 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
123 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
124 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
125 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
126 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
127 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
128 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
129 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
130 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
131
132 #define PMC_CTRL                0x0
133 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
134 #define PMC_WAKE_MASK           0xc
135 #define PMC_WAKE_LEVEL          0x10
136 #define PMC_DPAD_ORIDE          0x1C
137 #define PMC_WAKE_DELAY          0xe0
138 #define PMC_DPD_SAMPLE          0x20
139 #define PMC_IO_DPD_REQ_0        0x1b8
140 #define PMC_IO_DPD2_REQ_0       0X1C0
141
142 #define PMC_WAKE_STATUS         0x14
143 #define PMC_SW_WAKE_STATUS      0x18
144 #define PMC_COREPWRGOOD_TIMER   0x3c
145 #define PMC_CPUPWRGOOD_TIMER    0xc8
146 #define PMC_CPUPWROFF_TIMER     0xcc
147 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
148
149 #define PMC_PWRGATE_TOGGLE      0x30
150 #define PWRGATE_TOGGLE_START    (1 << 8)
151 #define UN_PWRGATE_CPU          \
152         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
153
154 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
155 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
156 #endif
157
158 #define CLK_RESET_CCLK_BURST    0x20
159 #define CLK_RESET_CCLK_DIVIDER  0x24
160 #define CLK_RESET_PLLC_BASE     0x80
161 #define CLK_RESET_PLLM_BASE     0x90
162 #define CLK_RESET_PLLX_BASE     0xe0
163 #define CLK_RESET_PLLX_MISC     0xe4
164 #define CLK_RESET_PLLP_BASE     0xa0
165 #define CLK_RESET_PLLP_OUTA     0xa4
166 #define CLK_RESET_PLLP_OUTB     0xa8
167 #define CLK_RESET_PLLP_MISC     0xac
168
169 #define CLK_RESET_SOURCE_CSITE  0x1d4
170
171 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
172 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
173 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
174 #define CLK_RESET_CCLK_IDLE_POLICY         1
175 #define CLK_RESET_CCLK_RUN_POLICY          2
176 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
177 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
178
179 #define EMC_MRW_0               0x0e8
180 #define EMC_MRW_DEV_SELECTN     30
181 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
182
183 #define MC_SECURITY_START       0x6c
184 #define MC_SECURITY_SIZE        0x70
185 #define MC_SECURITY_CFG2        0x7c
186
187 static struct clk *tegra_dfll;
188 static struct clk *tegra_pclk;
189 static const struct tegra_suspend_platform_data *pdata;
190 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
191
192 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
193 enum tegra_cluster_switch_time_id {
194         tegra_cluster_switch_time_id_start = 0,
195         tegra_cluster_switch_time_id_prolog,
196         tegra_cluster_switch_time_id_switch,
197         tegra_cluster_switch_time_id_epilog,
198         tegra_cluster_switch_time_id_max
199 };
200
201 static unsigned long
202                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
203 #define tegra_cluster_switch_time(flags, id) \
204         do { \
205                 barrier(); \
206                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
207                         void __iomem *timer_us = \
208                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
209                         if (id < tegra_cluster_switch_time_id_max) \
210                                 tegra_cluster_switch_times[id] = \
211                                                         readl(timer_us); \
212                                 wmb(); \
213                 } \
214                 barrier(); \
215         } while(0)
216 #else
217 #define tegra_cluster_switch_time(flags, id) do {} while(0)
218 #endif
219
220 #ifdef CONFIG_PM_SLEEP
221 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
222         [TEGRA_SUSPEND_NONE]    = "none",
223         [TEGRA_SUSPEND_LP2]     = "lp2",
224         [TEGRA_SUSPEND_LP1]     = "lp1",
225         [TEGRA_SUSPEND_LP0]     = "lp0",
226 };
227
228 unsigned long tegra_cpu_power_good_time(void)
229 {
230         if (WARN_ON_ONCE(!pdata))
231                 return 5000;
232
233         return pdata->cpu_timer;
234 }
235
236 unsigned long tegra_cpu_power_off_time(void)
237 {
238         if (WARN_ON_ONCE(!pdata))
239                 return 5000;
240
241         return pdata->cpu_off_timer;
242 }
243
244 unsigned long tegra_cpu_lp2_min_residency(void)
245 {
246         if (WARN_ON_ONCE(!pdata))
247                 return 2000;
248
249         return pdata->cpu_lp2_min_residency;
250 }
251
252 static void suspend_cpu_dfll_mode(void)
253 {
254         /* If DFLL is used as CPU clock source go to open loop mode */
255         if (!is_lp_cluster() && tegra_dfll &&
256             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
257                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
258 }
259
260 static void resume_cpu_dfll_mode(void)
261 {
262         /* If DFLL is used as CPU clock source restore closed loop mode */
263         if (!is_lp_cluster() && tegra_dfll &&
264             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
265                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
266 }
267
268 /*
269  * create_suspend_pgtable
270  *
271  * Creates a page table with identity mappings of physical memory and IRAM
272  * for use when the MMU is off, in addition to all the regular kernel mappings.
273  */
274 static __init int create_suspend_pgtable(void)
275 {
276         tegra_pgd = pgd_alloc(&init_mm);
277         if (!tegra_pgd)
278                 return -ENOMEM;
279
280         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
281         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
282                 high_memory, 0);
283         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
284                 IO_IRAM_VIRT + SECTION_SIZE, 0);
285
286         /* inner/outer write-back/write-allocate, sharable */
287         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
288
289         return 0;
290 }
291
292 /* ensures that sufficient time is passed for a register write to
293  * serialize into the 32KHz domain */
294 static void pmc_32kwritel(u32 val, unsigned long offs)
295 {
296         writel(val, pmc + offs);
297         udelay(130);
298 }
299
300 static void set_power_timers(unsigned long us_on, unsigned long us_off,
301                              long rate)
302 {
303         static unsigned long last_us_off = 0;
304         unsigned long long ticks;
305         unsigned long long pclk;
306
307         if (WARN_ON_ONCE(rate <= 0))
308                 pclk = 100000000;
309         else
310                 pclk = rate;
311
312         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
313                 ticks = (us_on * pclk) + 999999ull;
314                 do_div(ticks, 1000000);
315                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
316
317                 ticks = (us_off * pclk) + 999999ull;
318                 do_div(ticks, 1000000);
319                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
320                 wmb();
321         }
322         tegra_last_pclk = pclk;
323         last_us_off = us_off;
324 }
325
326 /*
327  * restore_cpu_complex
328  *
329  * restores cpu clock setting, clears flow controller
330  *
331  * Always called on CPU 0.
332  */
333 static void restore_cpu_complex(u32 mode)
334 {
335         int cpu = smp_processor_id();
336         unsigned int reg;
337 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
338         unsigned int policy;
339 #endif
340
341         BUG_ON(cpu != 0);
342
343 #ifdef CONFIG_SMP
344         cpu = cpu_logical_map(cpu);
345 #endif
346
347 /*
348  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
349  * or restored by common clock suspend/resume procedures. Hence, we don't
350  * need it here.
351  */
352 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
353         /* Is CPU complex already running on PLLX? */
354         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
355         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
356
357         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
358                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
359         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
360                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
361         else
362                 BUG();
363
364         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
365                 /* restore PLLX settings if CPU is on different PLL */
366                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
367                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
368
369                 /* wait for PLL stabilization if PLLX was enabled */
370                 if (tegra_sctx.pllx_base & (1<<30)) {
371 #if USE_PLL_LOCK_BITS
372                         /* Enable lock detector */
373                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
374                         reg |= 1<<18;
375                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
376                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
377                                  (1<<27)))
378                                 cpu_relax();
379
380                         udelay(PLL_POST_LOCK_DELAY);
381 #else
382                         udelay(300);
383 #endif
384                 }
385         }
386
387         /* Restore original burst policy setting for calls resulting from CPU
388            LP2 in idle or system suspend; keep cluster switch prolog setting
389            intact. */
390         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
391                 writel(tegra_sctx.cclk_divider, clk_rst +
392                        CLK_RESET_CCLK_DIVIDER);
393                 writel(tegra_sctx.cpu_burst, clk_rst +
394                        CLK_RESET_CCLK_BURST);
395         }
396 #endif
397         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
398
399         /* Do not power-gate CPU 0 when flow controlled */
400         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
401         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
402         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
403         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
404         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
405         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
406         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
407
408         /* If an immedidate cluster switch is being perfomed, restore the
409            local timer registers. For calls resulting from CPU LP2 in
410            idle or system suspend, the local timer was shut down and
411            timekeeping switched over to the global system timer. In this
412            case keep local timer disabled, and restore only periodic load. */
413         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
414                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
415 #ifdef CONFIG_ARM_ARCH_TIMER
416                 tegra_sctx.arch_timer.cntp_ctl = 0;
417 #endif
418 #ifdef CONFIG_HAVE_ARM_TWD
419                 tegra_sctx.twd.twd_ctrl = 0;
420 #endif
421         }
422 #ifdef CONFIG_ARM_ARCH_TIMER
423         arch_timer_resume(&tegra_sctx.arch_timer);
424 #endif
425 #ifdef CONFIG_HAVE_ARM_TWD
426         tegra_twd_resume(&tegra_sctx.twd);
427 #endif
428 }
429
430 /*
431  * suspend_cpu_complex
432  *
433  * saves pll state for use by restart_plls, prepares flow controller for
434  * transition to suspend state
435  *
436  * Must always be called on cpu 0.
437  */
438 static void suspend_cpu_complex(u32 mode)
439 {
440         int cpu = smp_processor_id();
441         unsigned int reg;
442         int i;
443
444         BUG_ON(cpu != 0);
445
446 #ifdef CONFIG_SMP
447         cpu = cpu_logical_map(cpu);
448 #endif
449         /* switch coresite to clk_m, save off original source */
450         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
451         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
452
453         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
454         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
455         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
456         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
457         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
458         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
459         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
460         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
461
462 #ifdef CONFIG_HAVE_ARM_TWD
463         tegra_twd_suspend(&tegra_sctx.twd);
464 #endif
465 #ifdef CONFIG_ARM_ARCH_TIMER
466         arch_timer_suspend(&tegra_sctx.arch_timer);
467 #endif
468
469         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
470         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
471         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
472         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
473         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
474 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
475         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
476 #else
477         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
478 #endif
479         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
480         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
481
482         for (i = 0; i < num_possible_cpus(); i++) {
483                 if (i == cpu)
484                         continue;
485                 reg = readl(FLOW_CTRL_CPU_CSR(i));
486                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
487                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
488                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
489         }
490
491         tegra_gic_cpu_disable(true);
492 }
493
494 void tegra_clear_cpu_in_lp2(int cpu)
495 {
496         spin_lock(&tegra_lp2_lock);
497         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
498         cpumask_clear_cpu(cpu, &tegra_in_lp2);
499
500         /* Update the IRAM copy used by the reset handler. The IRAM copy
501            can't use used directly by cpumask_clear_cpu() because it uses
502            LDREX/STREX which requires the addressed location to be inner
503            cacheable and sharable which IRAM isn't. */
504         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
505         dsb();
506
507         spin_unlock(&tegra_lp2_lock);
508 }
509
510 bool tegra_set_cpu_in_lp2(int cpu)
511 {
512         bool last_cpu = false;
513
514         spin_lock(&tegra_lp2_lock);
515         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
516         cpumask_set_cpu(cpu, &tegra_in_lp2);
517
518         /* Update the IRAM copy used by the reset handler. The IRAM copy
519            can't use used directly by cpumask_set_cpu() because it uses
520            LDREX/STREX which requires the addressed location to be inner
521            cacheable and sharable which IRAM isn't. */
522         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
523         dsb();
524
525         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
526                 last_cpu = true;
527 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
528         else if (cpu == 1)
529                 tegra2_cpu_set_resettable_soon();
530 #endif
531
532         spin_unlock(&tegra_lp2_lock);
533         return last_cpu;
534 }
535
536 static void tegra_sleep_core(enum tegra_suspend_mode mode,
537                              unsigned long v2p)
538 {
539 #ifdef CONFIG_TRUSTED_FOUNDATIONS
540         if (mode == TEGRA_SUSPEND_LP0) {
541                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
542                                   virt_to_phys(tegra_resume));
543         } else {
544                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
545                                   (TEGRA_RESET_HANDLER_BASE +
546                                    tegra_cpu_reset_handler_offset));
547         }
548 #endif
549 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
550         cpu_suspend(v2p, tegra2_sleep_core_finish);
551 #else
552         cpu_suspend(v2p, tegra3_sleep_core_finish);
553 #endif
554 }
555
556 static inline void tegra_sleep_cpu(unsigned long v2p)
557 {
558 #ifdef CONFIG_TRUSTED_FOUNDATIONS
559         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
560                           (TEGRA_RESET_HANDLER_BASE +
561                            tegra_cpu_reset_handler_offset));
562 #endif
563         cpu_suspend(v2p, tegra_sleep_cpu_finish);
564 }
565
566 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
567 {
568         u32 reg;
569         unsigned int remain;
570 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
571         pgd_t *pgd;
572 #endif
573
574         /* Only the last cpu down does the final suspend steps */
575         reg = readl(pmc + PMC_CTRL);
576         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
577         if (pdata->combined_req)
578                 reg &= ~TEGRA_POWER_PWRREQ_OE;
579         else
580                 reg |= TEGRA_POWER_PWRREQ_OE;
581
582         reg &= ~TEGRA_POWER_EFFECT_LP0;
583         writel(reg, pmc + PMC_CTRL);
584
585         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
586
587         /*
588          * We can use clk_get_rate_all_locked() here, because all other cpus
589          * are in LP2 state and irqs are disabled
590          */
591         if (flags & TEGRA_POWER_CLUSTER_MASK) {
592                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
593                 set_power_timers(pdata->cpu_timer, 2,
594                         clk_get_rate_all_locked(tegra_pclk));
595                 if (flags & TEGRA_POWER_CLUSTER_G) {
596                         /*
597                          * To reduce the vdd_cpu up latency when LP->G
598                          * transition. Before the transition, enable
599                          * the vdd_cpu rail.
600                          */
601                         if (is_lp_cluster())
602                                 writel(UN_PWRGATE_CPU,
603                                        pmc + PMC_PWRGATE_TOGGLE);
604                 }
605                 tegra_cluster_switch_prolog(flags);
606         } else {
607                 suspend_cpu_dfll_mode();
608                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
609                         clk_get_rate_all_locked(tegra_pclk));
610 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
611                 reg = readl(FLOW_CTRL_CPU_CSR(0));
612                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
613                 if (is_lp_cluster()) {
614                         /* for LP cluster, there is no option for rail gating */
615                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
616                                                 TEGRA_POWER_CLUSTER_PART_MASK)
617                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
618                         else if (flags)
619                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
620                 }
621                 else {
622                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
623                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
624                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
625                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
626                 }
627                 writel(reg, FLOW_CTRL_CPU_CSR(0));
628 #endif
629         }
630
631         if (sleep_time)
632                 tegra_lp2_set_trigger(sleep_time);
633
634         cpu_cluster_pm_enter();
635         suspend_cpu_complex(flags);
636         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
637 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
638         flush_cache_all();
639         /*
640          * No need to flush complete L2. Cleaning kernel and IO mappings
641          * is enough for the LP code sequence that has L2 disabled but
642          * MMU on.
643          */
644         pgd = cpu_get_pgd();
645         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
646                           __pa(pgd + PTRS_PER_PGD));
647         outer_disable();
648 #endif
649         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
650
651         tegra_init_cache(false);
652         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
653         restore_cpu_complex(flags);
654         cpu_cluster_pm_exit();
655
656         remain = tegra_lp2_timer_remain();
657         if (sleep_time)
658                 tegra_lp2_set_trigger(0);
659
660         if (flags & TEGRA_POWER_CLUSTER_MASK) {
661                 tegra_cluster_switch_epilog(flags);
662                 if (is_idle_task(current))
663                         trace_cpu_cluster_rcuidle(POWER_CPU_CLUSTER_DONE);
664                 else
665                         trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
666         } else {
667                 resume_cpu_dfll_mode();
668         }
669         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
670
671 #if INSTRUMENT_CLUSTER_SWITCH
672         if (flags & TEGRA_POWER_CLUSTER_MASK) {
673                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
674                         is_lp_cluster() ? "G=>LP" : "LP=>G",
675                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
676                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
677                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
678                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
679                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
680                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
681                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
682                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
683         }
684 #endif
685         return remain;
686 }
687
688 static int tegra_common_suspend(void)
689 {
690         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
691
692         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
693         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
694         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
695
696         /* copy the reset vector and SDRAM shutdown code into IRAM */
697         memcpy(iram_save, iram_code, iram_save_size);
698         memcpy(iram_code, tegra_iram_start(), iram_save_size);
699
700         return 0;
701 }
702
703 static void tegra_common_resume(void)
704 {
705         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
706 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
707         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
708 #endif
709
710         /* Clear DPD sample */
711         writel(0x0, pmc + PMC_DPD_SAMPLE);
712
713         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
714         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
715         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
716 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
717         /* trigger emc mode write */
718         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
719 #endif
720         /* clear scratch registers shared by suspend and the reset pen */
721         writel(0x0, pmc + PMC_SCRATCH39);
722         writel(0x0, pmc + PMC_SCRATCH41);
723
724         /* restore IRAM */
725         memcpy(iram_code, iram_save, iram_save_size);
726 }
727
728 static int tegra_suspend_prepare_late(void)
729 {
730 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
731         disable_irq(INT_SYS_STATS_MON);
732 #endif
733         return 0;
734 }
735
736 static void tegra_suspend_wake(void)
737 {
738 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
739         enable_irq(INT_SYS_STATS_MON);
740 #endif
741 }
742
743 static void tegra_pm_set(enum tegra_suspend_mode mode)
744 {
745         u32 reg, boot_flag;
746         unsigned long rate = 32768;
747
748         reg = readl(pmc + PMC_CTRL);
749         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
750         if (pdata->combined_req)
751                 reg &= ~TEGRA_POWER_PWRREQ_OE;
752         else
753                 reg |= TEGRA_POWER_PWRREQ_OE;
754         reg &= ~TEGRA_POWER_EFFECT_LP0;
755
756         switch (mode) {
757         case TEGRA_SUSPEND_LP0:
758 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
759                 rate = clk_get_rate_all_locked(tegra_pclk);
760 #endif
761                 if (pdata->combined_req) {
762                         reg |= TEGRA_POWER_PWRREQ_OE;
763                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
764                 }
765
766                 /*
767                  * LP0 boots through the AVP, which then resumes the AVP to
768                  * the address in scratch 39, and the cpu to the address in
769                  * scratch 41 to tegra_resume
770                  */
771                 writel(0x0, pmc + PMC_SCRATCH39);
772
773                 /* Enable DPD sample to trigger sampling pads data and direction
774                  * in which pad will be driven during lp0 mode*/
775                 writel(0x1, pmc + PMC_DPD_SAMPLE);
776 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
777                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ_0);
778                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ_0);
779 #endif
780 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
781                 /* this is needed only for T11x, not for other chips */
782                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
783 #endif
784
785                 /* Set warmboot flag */
786                 boot_flag = readl(pmc + PMC_SCRATCH0);
787                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
788
789                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
790
791                 reg |= TEGRA_POWER_EFFECT_LP0;
792                 /* No break here. LP0 code falls through to write SCRATCH41 */
793         case TEGRA_SUSPEND_LP1:
794                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
795                 wmb();
796                 break;
797         case TEGRA_SUSPEND_LP2:
798                 rate = clk_get_rate(tegra_pclk);
799                 break;
800         case TEGRA_SUSPEND_NONE:
801                 return;
802         default:
803                 BUG();
804         }
805
806         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
807
808         pmc_32kwritel(reg, PMC_CTRL);
809 }
810
811 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
812         [TEGRA_SUSPEND_NONE] = "none",
813         [TEGRA_SUSPEND_LP2] = "LP2",
814         [TEGRA_SUSPEND_LP1] = "LP1",
815         [TEGRA_SUSPEND_LP0] = "LP0",
816 };
817
818 static int tegra_suspend_enter(suspend_state_t state)
819 {
820         int ret;
821         ktime_t delta;
822         struct timespec ts_entry, ts_exit;
823
824         if (pdata && pdata->board_suspend)
825                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
826
827         read_persistent_clock(&ts_entry);
828
829         ret = tegra_suspend_dram(current_suspend_mode, 0);
830         if (ret) {
831                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
832                 goto abort_suspend;
833         }
834
835         read_persistent_clock(&ts_exit);
836
837         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
838                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
839
840                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
841                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
842                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
843                 else
844                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
845         }
846
847 abort_suspend:
848         if (pdata && pdata->board_resume)
849                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
850
851         return ret;
852 }
853
854 static void tegra_suspend_check_pwr_stats(void)
855 {
856         /* cpus and l2 are powered off later */
857         unsigned long pwrgate_partid_mask =
858 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
859                 (1 << TEGRA_POWERGATE_HEG)      |
860                 (1 << TEGRA_POWERGATE_SATA)     |
861                 (1 << TEGRA_POWERGATE_3D1)      |
862 #endif
863                 (1 << TEGRA_POWERGATE_3D)       |
864                 (1 << TEGRA_POWERGATE_VENC)     |
865                 (1 << TEGRA_POWERGATE_PCIE)     |
866                 (1 << TEGRA_POWERGATE_VDEC)     |
867                 (1 << TEGRA_POWERGATE_MPE);
868
869         int partid;
870
871         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
872                 if ((1 << partid) & pwrgate_partid_mask)
873                         if (tegra_powergate_is_powered(partid))
874                                 pr_warning("partition %s is left on before suspend\n",
875                                         tegra_powergate_get_name(partid));
876
877         return;
878 }
879
880 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
881 {
882         int err = 0;
883         u32 scratch37 = 0xDEADBEEF;
884         u32 reg;
885
886         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
887                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
888                 err = -ENXIO;
889                 goto fail;
890         }
891
892         if (tegra_is_voice_call_active()) {
893                 /* backup the current value of scratch37 */
894                 scratch37 = readl(pmc + PMC_SCRATCH37);
895
896                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
897                 reg = TEGRA_POWER_LP1_AUDIO;
898                 pmc_32kwritel(reg, PMC_SCRATCH37);
899         }
900
901         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
902                 pr_info("LP0 not used due to unsupported wakeup events\n");
903                 mode = TEGRA_SUSPEND_LP1;
904         }
905
906         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
907                 tegra_suspend_check_pwr_stats();
908
909         tegra_common_suspend();
910
911         tegra_pm_set(mode);
912
913         if (pdata && pdata->board_suspend)
914                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
915
916         local_fiq_disable();
917
918         trace_cpu_suspend(CPU_SUSPEND_START);
919
920         if (mode == TEGRA_SUSPEND_LP0) {
921 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
922                 reg = readl(pmc + PMC_SCRATCH4);
923                 if (is_lp_cluster())
924                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
925                 else
926                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
927                 pmc_32kwritel(reg, PMC_SCRATCH4);
928 #endif
929                 tegra_tsc_suspend();
930                 tegra_lp0_suspend_mc();
931                 tegra_cpu_reset_handler_save();
932                 tegra_tsc_wait_for_suspend();
933         }
934         else if (mode == TEGRA_SUSPEND_LP1)
935                 *iram_cpu_lp1_mask = 1;
936
937         suspend_cpu_complex(flags);
938
939 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
940         /* In case of LP0, program external power gating accordinly */
941         if (mode == TEGRA_SUSPEND_LP0) {
942                 reg = readl(FLOW_CTRL_CPU_CSR(0));
943                 if (is_lp_cluster())
944                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
945                 else
946                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
947                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
948         }
949 #endif
950
951         flush_cache_all();
952         outer_flush_all();
953         outer_disable();
954
955         if (mode == TEGRA_SUSPEND_LP2)
956                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
957         else
958                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
959
960         tegra_init_cache(true);
961
962         if (mode == TEGRA_SUSPEND_LP0) {
963 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
964                 reg = readl(pmc+PMC_CTRL);
965                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
966                 pmc_32kwritel(reg, PMC_CTRL);
967 #endif
968                 tegra_tsc_resume();
969                 tegra_cpu_reset_handler_restore();
970                 tegra_lp0_resume_mc();
971                 tegra_tsc_wait_for_resume();
972         } else if (mode == TEGRA_SUSPEND_LP1)
973                 *iram_cpu_lp1_mask = 0;
974
975         /* if scratch37 was clobbered during LP1, restore it */
976         if (scratch37 != 0xDEADBEEF)
977                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
978
979         restore_cpu_complex(flags);
980
981         /* for platforms where the core & CPU power requests are
982          * combined as a single request to the PMU, transition out
983          * of LP0 state by temporarily enabling both requests
984          */
985         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
986                 reg = readl(pmc + PMC_CTRL);
987                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
988                 pmc_32kwritel(reg, PMC_CTRL);
989                 reg &= ~TEGRA_POWER_PWRREQ_OE;
990                 pmc_32kwritel(reg, PMC_CTRL);
991         }
992
993         if (pdata && pdata->board_resume)
994                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
995
996         trace_cpu_suspend(CPU_SUSPEND_DONE);
997
998         local_fiq_enable();
999
1000         tegra_common_resume();
1001
1002 fail:
1003         return err;
1004 }
1005
1006 /*
1007  * Function pointers to optional board specific function
1008  */
1009 void (*tegra_deep_sleep)(int);
1010 EXPORT_SYMBOL(tegra_deep_sleep);
1011
1012 static int tegra_suspend_prepare(void)
1013 {
1014         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1015                 tegra_deep_sleep(1);
1016         return 0;
1017 }
1018
1019 static void tegra_suspend_finish(void)
1020 {
1021         if (pdata && pdata->cpu_resume_boost) {
1022                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1023                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1024                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1025         }
1026
1027         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1028                 tegra_deep_sleep(0);
1029 }
1030
1031 static const struct platform_suspend_ops tegra_suspend_ops = {
1032         .valid          = suspend_valid_only_mem,
1033         .prepare        = tegra_suspend_prepare,
1034         .finish         = tegra_suspend_finish,
1035         .prepare_late   = tegra_suspend_prepare_late,
1036         .wake           = tegra_suspend_wake,
1037         .enter          = tegra_suspend_enter,
1038 };
1039
1040 static ssize_t suspend_mode_show(struct kobject *kobj,
1041                                         struct kobj_attribute *attr, char *buf)
1042 {
1043         char *start = buf;
1044         char *end = buf + PAGE_SIZE;
1045
1046         start += scnprintf(start, end - start, "%s ", \
1047                                 tegra_suspend_name[current_suspend_mode]);
1048         start += scnprintf(start, end - start, "\n");
1049
1050         return start - buf;
1051 }
1052
1053 static ssize_t suspend_mode_store(struct kobject *kobj,
1054                                         struct kobj_attribute *attr,
1055                                         const char *buf, size_t n)
1056 {
1057         int len;
1058         const char *name_ptr;
1059         enum tegra_suspend_mode new_mode;
1060
1061         name_ptr = buf;
1062         while (*name_ptr && !isspace(*name_ptr))
1063                 name_ptr++;
1064         len = name_ptr - buf;
1065         if (!len)
1066                 goto bad_name;
1067         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1068         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1069                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1070                 pr_info("Illegal tegra suspend state: %s\n", buf);
1071                 goto bad_name;
1072         }
1073
1074         for (new_mode = TEGRA_SUSPEND_NONE; \
1075                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1076                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1077                         current_suspend_mode = new_mode;
1078                         break;
1079                 }
1080         }
1081
1082 bad_name:
1083         return n;
1084 }
1085
1086 static struct kobj_attribute suspend_mode_attribute =
1087         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1088
1089 static struct kobject *suspend_kobj;
1090
1091 static int tegra_pm_enter_suspend(void)
1092 {
1093         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1094         suspend_cpu_dfll_mode();
1095         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1096                 tegra_lp0_cpu_mode(true);
1097         return 0;
1098 }
1099
1100 static void tegra_pm_enter_resume(void)
1101 {
1102         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1103                 tegra_lp0_cpu_mode(false);
1104         resume_cpu_dfll_mode();
1105         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1106 }
1107
1108 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1109         .suspend = tegra_pm_enter_suspend,
1110         .resume = tegra_pm_enter_resume,
1111 };
1112
1113 static __init int tegra_pm_enter_syscore_init(void)
1114 {
1115         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1116         return 0;
1117 }
1118 subsys_initcall(tegra_pm_enter_syscore_init);
1119 #endif
1120
1121 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1122 {
1123         u32 reg;
1124         u32 mode;
1125
1126 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1127         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1128         BUG_ON(IS_ERR(tegra_dfll));
1129 #endif
1130         tegra_pclk = clk_get_sys(NULL, "pclk");
1131         BUG_ON(IS_ERR(tegra_pclk));
1132         pdata = plat;
1133         (void)reg;
1134         (void)mode;
1135
1136         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1137                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1138
1139 #ifndef CONFIG_PM_SLEEP
1140         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1141                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1142                            "disabling suspend\n", __func__);
1143                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1144         }
1145 #else
1146         if (create_suspend_pgtable() < 0) {
1147                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1148                                 __func__);
1149                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1150                 goto fail;
1151         }
1152
1153         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1154             (tegra_revision == TEGRA_REVISION_A01) &&
1155             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1156                 /* Tegra 3 A01 supports only LP1 */
1157                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1158                            "-- disabling LP0\n", __func__);
1159                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1160         }
1161         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1162                 tegra_lp0_vec_relocate) {
1163                 unsigned char *reloc_lp0;
1164                 unsigned long tmp;
1165                 void __iomem *orig;
1166                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1167                                         GFP_KERNEL);
1168                 WARN_ON(!reloc_lp0);
1169                 if (!reloc_lp0) {
1170                         pr_err("%s: Failed to allocate reloc_lp0\n",
1171                                 __func__);
1172                         goto out;
1173                 }
1174
1175                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1176                 WARN_ON(!orig);
1177                 if (!orig) {
1178                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1179                                 __func__, tegra_lp0_vec_start);
1180                         kfree(reloc_lp0);
1181                         goto out;
1182                 }
1183
1184                 tmp = (unsigned long) reloc_lp0;
1185                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1186                 reloc_lp0 = (unsigned char *)tmp;
1187                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1188                 iounmap(orig);
1189                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1190         }
1191
1192 out:
1193         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1194                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1195                            "provided by bootlader -- disabling LP0\n",
1196                            __func__);
1197                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1198         }
1199
1200         iram_save_size = tegra_iram_end() - tegra_iram_start();
1201
1202         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1203         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1204                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1205                        "-- LP0/LP1 unavailable\n", __func__);
1206                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1207         }
1208
1209 #ifdef CONFIG_TEGRA_LP1_950
1210         if (pdata->lp1_lowvolt_support) {
1211                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1212                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1213                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1214
1215                 lp1_core_lowvolt = 0;
1216                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1217                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1218
1219                 lp1_core_highvolt = 0;
1220                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1221                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1222         }
1223 #endif
1224         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1225         /* Initialize scratch registers used for CPU LP2 synchronization */
1226         writel(0, pmc + PMC_SCRATCH37);
1227         writel(0, pmc + PMC_SCRATCH38);
1228         writel(0, pmc + PMC_SCRATCH39);
1229         writel(0, pmc + PMC_SCRATCH41);
1230
1231         /* Always enable CPU power request; just normal polarity is supported */
1232         reg = readl(pmc + PMC_CTRL);
1233         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1234         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1235         pmc_32kwritel(reg, PMC_CTRL);
1236
1237         /* Configure core power request and system clock control if LP0
1238            is supported */
1239         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1240         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1241
1242         reg = readl(pmc + PMC_CTRL);
1243
1244         if (!pdata->sysclkreq_high)
1245                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1246         else
1247                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1248
1249         if (!pdata->corereq_high)
1250                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1251         else
1252                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1253
1254         /* configure output inverters while the request is tristated */
1255         pmc_32kwritel(reg, PMC_CTRL);
1256
1257         /* now enable requests */
1258         reg |= TEGRA_POWER_SYSCLK_OE;
1259         if (!pdata->combined_req)
1260                 reg |= TEGRA_POWER_PWRREQ_OE;
1261         pmc_32kwritel(reg, PMC_CTRL);
1262
1263         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1264                 tegra_lp0_suspend_init();
1265
1266         suspend_set_ops(&tegra_suspend_ops);
1267
1268         /* Create /sys/power/suspend/type */
1269         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1270         if (suspend_kobj) {
1271                 if (sysfs_create_file(suspend_kobj, \
1272                                                 &suspend_mode_attribute.attr))
1273                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1274                                                                 __func__);
1275         }
1276
1277         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1278         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1279
1280         /* clear io dpd settings before kernel */
1281         tegra_bl_io_dpd_cleanup();
1282
1283 fail:
1284 #endif
1285         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1286                 tegra_lp2_in_idle(false);
1287
1288         current_suspend_mode = plat->suspend_mode;
1289 }
1290
1291 unsigned long debug_uart_port_base = 0;
1292 EXPORT_SYMBOL(debug_uart_port_base);
1293
1294 static int tegra_debug_uart_suspend(void)
1295 {
1296         void __iomem *uart;
1297         u32 lcr;
1298
1299         if (!debug_uart_port_base)
1300                 return 0;
1301
1302         uart = IO_ADDRESS(debug_uart_port_base);
1303
1304         lcr = readb(uart + UART_LCR * 4);
1305
1306         tegra_sctx.uart[0] = lcr;
1307         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1308
1309         /* DLAB = 0 */
1310         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1311
1312         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1313
1314         /* DLAB = 1 */
1315         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1316
1317         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1318         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1319
1320         writeb(lcr, uart + UART_LCR * 4);
1321
1322         return 0;
1323 }
1324
1325 static void tegra_debug_uart_resume(void)
1326 {
1327         void __iomem *uart;
1328         u32 lcr;
1329
1330         if (!debug_uart_port_base)
1331                 return;
1332
1333         uart = IO_ADDRESS(debug_uart_port_base);
1334
1335         lcr = tegra_sctx.uart[0];
1336
1337         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1338
1339         /* DLAB = 0 */
1340         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1341
1342         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1343                         uart + UART_FCR * 4);
1344
1345         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1346
1347         /* DLAB = 1 */
1348         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1349
1350         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1351         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1352
1353         writeb(lcr, uart + UART_LCR * 4);
1354 }
1355
1356 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1357         .suspend = tegra_debug_uart_suspend,
1358         .resume = tegra_debug_uart_resume,
1359 };
1360
1361 struct clk *debug_uart_clk = NULL;
1362 EXPORT_SYMBOL(debug_uart_clk);
1363
1364 void tegra_console_uart_suspend(void)
1365 {
1366         if (console_suspend_enabled && debug_uart_clk)
1367                 clk_disable(debug_uart_clk);
1368 }
1369
1370 void tegra_console_uart_resume(void)
1371 {
1372         if (console_suspend_enabled && debug_uart_clk)
1373                 clk_enable(debug_uart_clk);
1374 }
1375
1376 static int tegra_debug_uart_syscore_init(void)
1377 {
1378         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1379         return 0;
1380 }
1381 arch_initcall(tegra_debug_uart_syscore_init);