ARM: tegra11x: Define ncpu residency for 2 clusters
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2013, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50 #include <trace/events/nvsecurity.h>
51
52 #include <asm/cacheflush.h>
53 #include <asm/idmap.h>
54 #include <asm/localtimer.h>
55 #include <asm/pgalloc.h>
56 #include <asm/pgtable.h>
57 #include <asm/tlbflush.h>
58 #include <asm/suspend.h>
59 #include <asm/smp_plat.h>
60
61 #include <mach/irqs.h>
62 #include <mach/powergate.h>
63 #include <mach/hardware.h>
64
65 #include "board.h"
66 #include "clock.h"
67 #include "common.h"
68 #include "cpuidle.h"
69 #include "fuse.h"
70 #include "gic.h"
71 #include "iomap.h"
72 #include "pm.h"
73 #include "pm-irq.h"
74 #include "reset.h"
75 #include "sleep.h"
76 #include "timer.h"
77 #include "dvfs.h"
78 #include "cpu-tegra.h"
79
80 #define CREATE_TRACE_POINTS
81 #include <trace/events/nvpower.h>
82
83 struct suspend_context {
84         /*
85          * The next 7 values are referenced by offset in __restart_plls
86          * in headsmp-t2.S, and should not be moved
87          */
88         u32 pllx_misc;
89         u32 pllx_base;
90         u32 pllp_misc;
91         u32 pllp_base;
92         u32 pllp_outa;
93         u32 pllp_outb;
94         u32 pll_timeout;
95
96         u32 cpu_burst;
97         u32 clk_csite_src;
98         u32 cclk_divider;
99
100         u32 mc[3];
101         u8 uart[5];
102
103         struct tegra_twd_context twd;
104 };
105
106 #ifdef CONFIG_PM_SLEEP
107 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
108 static pgd_t *tegra_pgd;
109 static DEFINE_SPINLOCK(tegra_lp2_lock);
110 static cpumask_t tegra_in_lp2;
111 static cpumask_t *iram_cpu_lp2_mask;
112 static unsigned long *iram_cpu_lp1_mask;
113 static u8 *iram_save;
114 static unsigned long iram_save_size;
115 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
116 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
117 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
118 static int tegra_last_pclk;
119 #endif
120
121 struct suspend_context tegra_sctx;
122
123 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
124 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
125 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
126 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
127 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
128 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
129 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
130 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
131 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
132
133 #define PMC_CTRL                0x0
134 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
135 #define PMC_WAKE_MASK           0xc
136 #define PMC_WAKE_LEVEL          0x10
137 #define PMC_DPAD_ORIDE          0x1C
138 #define PMC_WAKE_DELAY          0xe0
139 #define PMC_DPD_SAMPLE          0x20
140 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
141 #define PMC_DPD_ENABLE          0x24
142 #endif
143 #define PMC_IO_DPD_REQ          0x1B8
144 #define PMC_IO_DPD2_REQ         0x1C0
145
146
147 #define PMC_WAKE_STATUS         0x14
148 #define PMC_SW_WAKE_STATUS      0x18
149 #define PMC_COREPWRGOOD_TIMER   0x3c
150 #define PMC_CPUPWRGOOD_TIMER    0xc8
151 #define PMC_CPUPWROFF_TIMER     0xcc
152 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
153
154 #define PMC_PWRGATE_TOGGLE      0x30
155 #define PWRGATE_TOGGLE_START    (1 << 8)
156 #define UN_PWRGATE_CPU          \
157         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
158
159 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
160 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
161 #endif
162
163 #define CLK_RESET_CCLK_BURST    0x20
164 #define CLK_RESET_CCLK_DIVIDER  0x24
165 #define CLK_RESET_PLLC_BASE     0x80
166 #define CLK_RESET_PLLM_BASE     0x90
167 #define CLK_RESET_PLLX_BASE     0xe0
168 #define CLK_RESET_PLLX_MISC     0xe4
169 #define CLK_RESET_PLLP_BASE     0xa0
170 #define CLK_RESET_PLLP_OUTA     0xa4
171 #define CLK_RESET_PLLP_OUTB     0xa8
172 #define CLK_RESET_PLLP_MISC     0xac
173
174 #define CLK_RESET_SOURCE_CSITE  0x1d4
175
176 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
177 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
178 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
179 #define CLK_RESET_CCLK_IDLE_POLICY         1
180 #define CLK_RESET_CCLK_RUN_POLICY          2
181 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
182 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
183
184 #define EMC_MRW_0               0x0e8
185 #define EMC_MRW_DEV_SELECTN     30
186 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
187
188 #define MC_SECURITY_START       0x6c
189 #define MC_SECURITY_SIZE        0x70
190 #define MC_SECURITY_CFG2        0x7c
191
192 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
193 static struct clk *tegra_dfll;
194 #endif
195 static struct clk *tegra_pclk;
196 static const struct tegra_suspend_platform_data *pdata;
197 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
198
199 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
200 enum tegra_cluster_switch_time_id {
201         tegra_cluster_switch_time_id_start = 0,
202         tegra_cluster_switch_time_id_prolog,
203         tegra_cluster_switch_time_id_switch,
204         tegra_cluster_switch_time_id_epilog,
205         tegra_cluster_switch_time_id_max
206 };
207
208 static unsigned long
209                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
210 #define tegra_cluster_switch_time(flags, id) \
211         do { \
212                 barrier(); \
213                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
214                         void __iomem *timer_us = \
215                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
216                         if (id < tegra_cluster_switch_time_id_max) \
217                                 tegra_cluster_switch_times[id] = \
218                                                         readl(timer_us); \
219                                 wmb(); \
220                 } \
221                 barrier(); \
222         } while(0)
223 #else
224 #define tegra_cluster_switch_time(flags, id) do {} while(0)
225 #endif
226
227 #ifdef CONFIG_PM_SLEEP
228 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
229         [TEGRA_SUSPEND_NONE]    = "none",
230         [TEGRA_SUSPEND_LP2]     = "lp2",
231         [TEGRA_SUSPEND_LP1]     = "lp1",
232         [TEGRA_SUSPEND_LP0]     = "lp0",
233 };
234
235 unsigned long tegra_cpu_power_good_time(void)
236 {
237         if (WARN_ON_ONCE(!pdata))
238                 return 5000;
239
240         return pdata->cpu_timer;
241 }
242
243 unsigned long tegra_cpu_power_off_time(void)
244 {
245         if (WARN_ON_ONCE(!pdata))
246                 return 5000;
247
248         return pdata->cpu_off_timer;
249 }
250
251 unsigned long tegra_cpu_lp2_min_residency(void)
252 {
253         if (WARN_ON_ONCE(!pdata))
254                 return 2000;
255
256         return pdata->cpu_lp2_min_residency;
257 }
258
259 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
260 unsigned long tegra_min_residency_crail(void)
261 {
262         return pdata->min_residency_crail;
263 }
264 #endif
265
266 static void suspend_cpu_dfll_mode(void)
267 {
268 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
269         /* If DFLL is used as CPU clock source go to open loop mode */
270         if (!is_lp_cluster() && tegra_dfll &&
271             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
272                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
273 #endif
274 }
275
276 static void resume_cpu_dfll_mode(void)
277 {
278 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
279         /* If DFLL is used as CPU clock source restore closed loop mode */
280         if (!is_lp_cluster() && tegra_dfll &&
281             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
282                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
283 #endif
284 }
285
286 /*
287  * create_suspend_pgtable
288  *
289  * Creates a page table with identity mappings of physical memory and IRAM
290  * for use when the MMU is off, in addition to all the regular kernel mappings.
291  */
292 static __init int create_suspend_pgtable(void)
293 {
294         tegra_pgd = pgd_alloc(&init_mm);
295         if (!tegra_pgd)
296                 return -ENOMEM;
297
298         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
299         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
300                 high_memory, 0);
301         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
302                 IO_IRAM_VIRT + SECTION_SIZE, 0);
303
304         /* inner/outer write-back/write-allocate, sharable */
305         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
306
307         return 0;
308 }
309
310 /* ensures that sufficient time is passed for a register write to
311  * serialize into the 32KHz domain */
312 static void pmc_32kwritel(u32 val, unsigned long offs)
313 {
314         writel(val, pmc + offs);
315         udelay(130);
316 }
317
318 static void set_power_timers(unsigned long us_on, unsigned long us_off,
319                              long rate)
320 {
321         static unsigned long last_us_off = 0;
322         unsigned long long ticks;
323         unsigned long long pclk;
324
325         if (WARN_ON_ONCE(rate <= 0))
326                 pclk = 100000000;
327         else
328                 pclk = rate;
329
330         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
331                 ticks = (us_on * pclk) + 999999ull;
332                 do_div(ticks, 1000000);
333                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
334
335                 ticks = (us_off * pclk) + 999999ull;
336                 do_div(ticks, 1000000);
337                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
338                 wmb();
339         }
340         tegra_last_pclk = pclk;
341         last_us_off = us_off;
342 }
343
344 /*
345  * restore_cpu_complex
346  *
347  * restores cpu clock setting, clears flow controller
348  *
349  * Always called on CPU 0.
350  */
351 static void restore_cpu_complex(u32 mode)
352 {
353         int cpu = cpu_logical_map(smp_processor_id());
354         unsigned int reg;
355 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
356         unsigned int policy;
357 #endif
358
359 /*
360  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
361  * or restored by common clock suspend/resume procedures. Hence, we don't
362  * need it here.
363  */
364 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
365         /* Is CPU complex already running on PLLX? */
366         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
367         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
368
369         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
370                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
371         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
372                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
373         else
374                 BUG();
375
376         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
377                 /* restore PLLX settings if CPU is on different PLL */
378                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
379                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
380
381                 /* wait for PLL stabilization if PLLX was enabled */
382                 if (tegra_sctx.pllx_base & (1<<30)) {
383 #if USE_PLL_LOCK_BITS
384                         /* Enable lock detector */
385                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
386                         reg |= 1<<18;
387                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
388                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
389                                  (1<<27)))
390                                 cpu_relax();
391
392                         udelay(PLL_POST_LOCK_DELAY);
393 #else
394                         udelay(300);
395 #endif
396                 }
397         }
398
399         /* Restore original burst policy setting for calls resulting from CPU
400            LP2 in idle or system suspend; keep cluster switch prolog setting
401            intact. */
402         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
403                 writel(tegra_sctx.cclk_divider, clk_rst +
404                        CLK_RESET_CCLK_DIVIDER);
405                 writel(tegra_sctx.cpu_burst, clk_rst +
406                        CLK_RESET_CCLK_BURST);
407         }
408 #endif
409         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
410
411         /* Do not power-gate CPU 0 when flow controlled */
412         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
413         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
414         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
415         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
416         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
417         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
418         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
419
420         /* If an immedidate cluster switch is being perfomed, restore the
421            local timer registers. For calls resulting from CPU LP2 in
422            idle or system suspend, the local timer was shut down and
423            timekeeping switched over to the global system timer. In this
424            case keep local timer disabled, and restore only periodic load. */
425 #ifdef CONFIG_HAVE_ARM_TWD
426         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
427                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
428                 tegra_sctx.twd.twd_ctrl = 0;
429         }
430         tegra_twd_resume(&tegra_sctx.twd);
431 #endif
432 }
433
434 /*
435  * suspend_cpu_complex
436  *
437  * saves pll state for use by restart_plls, prepares flow controller for
438  * transition to suspend state
439  *
440  * Must always be called on cpu 0.
441  */
442 static void suspend_cpu_complex(u32 mode)
443 {
444         int cpu = cpu_logical_map(smp_processor_id());
445         unsigned int reg;
446         int i;
447
448         BUG_ON(cpu != 0);
449
450         /* switch coresite to clk_m, save off original source */
451         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
452         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
453
454         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
455         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
456         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
457         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
458         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
459         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
460         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
461         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
462
463 #ifdef CONFIG_HAVE_ARM_TWD
464         tegra_twd_suspend(&tegra_sctx.twd);
465 #endif
466
467         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
468         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
469         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
470         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
471         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
472 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
473         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
474 #else
475         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
476 #endif
477         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
478         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
479
480         for (i = 0; i < num_possible_cpus(); i++) {
481                 if (i == cpu)
482                         continue;
483                 reg = readl(FLOW_CTRL_CPU_CSR(i));
484                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
485                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
486                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
487         }
488
489         tegra_gic_cpu_disable(true);
490 }
491
492 void tegra_clear_cpu_in_pd(int cpu)
493 {
494         spin_lock(&tegra_lp2_lock);
495         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
496         cpumask_clear_cpu(cpu, &tegra_in_lp2);
497
498         /* Update the IRAM copy used by the reset handler. The IRAM copy
499            can't use used directly by cpumask_clear_cpu() because it uses
500            LDREX/STREX which requires the addressed location to be inner
501            cacheable and sharable which IRAM isn't. */
502         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
503         dsb();
504
505         spin_unlock(&tegra_lp2_lock);
506 }
507
508 bool tegra_set_cpu_in_pd(int cpu)
509 {
510         bool last_cpu = false;
511
512         spin_lock(&tegra_lp2_lock);
513         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
514         cpumask_set_cpu(cpu, &tegra_in_lp2);
515
516         /* Update the IRAM copy used by the reset handler. The IRAM copy
517            can't use used directly by cpumask_set_cpu() because it uses
518            LDREX/STREX which requires the addressed location to be inner
519            cacheable and sharable which IRAM isn't. */
520         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
521         dsb();
522
523         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
524                 last_cpu = true;
525 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
526         else if (cpu == 1)
527                 tegra2_cpu_set_resettable_soon();
528 #endif
529
530         spin_unlock(&tegra_lp2_lock);
531         return last_cpu;
532 }
533
534 static void tegra_sleep_core(enum tegra_suspend_mode mode,
535                              unsigned long v2p)
536 {
537 #ifdef CONFIG_TRUSTED_FOUNDATIONS
538         outer_flush_range(__pa(&tegra_resume_timestamps_start),
539                           __pa(&tegra_resume_timestamps_end));
540
541         if (mode == TEGRA_SUSPEND_LP0) {
542                 trace_smc_sleep_core(NVSEC_SMC_START);
543
544                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
545                                   virt_to_phys(tegra_resume));
546         } else {
547                 trace_smc_sleep_core(NVSEC_SMC_START);
548
549                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
550                                   (TEGRA_RESET_HANDLER_BASE +
551                                    tegra_cpu_reset_handler_offset));
552         }
553
554         trace_smc_sleep_core(NVSEC_SMC_DONE);
555 #endif
556 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
557         cpu_suspend(v2p, tegra2_sleep_core_finish);
558 #else
559         cpu_suspend(v2p, tegra3_sleep_core_finish);
560 #endif
561 }
562
563 static inline void tegra_sleep_cpu(unsigned long v2p)
564 {
565         cpu_suspend(v2p, tegra_sleep_cpu_finish);
566 }
567
568 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
569                                         unsigned int flags)
570 {
571         u32 reg;
572         unsigned int remain;
573 #if defined(CONFIG_CACHE_L2X0) && !defined(CONFIG_ARCH_TEGRA_14x_SOC)
574         pgd_t *pgd;
575 #endif
576
577         /* Only the last cpu down does the final suspend steps */
578         reg = readl(pmc + PMC_CTRL);
579         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
580         if (pdata->combined_req)
581                 reg &= ~TEGRA_POWER_PWRREQ_OE;
582         else
583                 reg |= TEGRA_POWER_PWRREQ_OE;
584
585         reg &= ~TEGRA_POWER_EFFECT_LP0;
586         writel(reg, pmc + PMC_CTRL);
587
588         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
589
590         /*
591          * We can use clk_get_rate_all_locked() here, because all other cpus
592          * are in LP2 state and irqs are disabled
593          */
594         if (flags & TEGRA_POWER_CLUSTER_MASK) {
595                 trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_START);
596                 set_power_timers(pdata->cpu_timer, 2,
597                         clk_get_rate_all_locked(tegra_pclk));
598                 if (flags & TEGRA_POWER_CLUSTER_G) {
599                         /*
600                          * To reduce the vdd_cpu up latency when LP->G
601                          * transition. Before the transition, enable
602                          * the vdd_cpu rail.
603                          */
604                         if (is_lp_cluster()) {
605 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
606                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
607                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
608                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
609 #else
610                                 writel(UN_PWRGATE_CPU,
611                                        pmc + PMC_PWRGATE_TOGGLE);
612 #endif
613                         }
614                 }
615                 tegra_cluster_switch_prolog(flags);
616         } else {
617                 suspend_cpu_dfll_mode();
618                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
619                         clk_get_rate_all_locked(tegra_pclk));
620 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
621                 reg = readl(FLOW_CTRL_CPU_CSR(0));
622                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
623                 if (is_lp_cluster()) {
624                         /* for LP cluster, there is no option for rail gating */
625                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
626                                                 TEGRA_POWER_CLUSTER_PART_MASK)
627                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
628                         else if (flags)
629                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
630                 }
631                 else {
632                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
633                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
634                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
635                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
636                 }
637                 writel(reg, FLOW_CTRL_CPU_CSR(0));
638 #endif
639         }
640
641         if (sleep_time)
642                 tegra_pd_set_trigger(sleep_time);
643
644         cpu_cluster_pm_enter();
645         suspend_cpu_complex(flags);
646         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
647 #ifdef CONFIG_CACHE_L2X0
648         flush_cache_all();
649 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
650         outer_flush_all();
651 #else
652         /*
653          * No need to flush complete L2. Cleaning kernel and IO mappings
654          * is enough for the LP code sequence that has L2 disabled but
655          * MMU on.
656          */
657         pgd = cpu_get_pgd();
658         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
659                           __pa(pgd + PTRS_PER_PGD));
660 #endif
661         outer_disable();
662 #endif
663         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
664
665 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
666         tegra_init_cache(true);
667 #else
668         tegra_init_cache(false);
669 #endif
670
671 #ifdef CONFIG_TRUSTED_FOUNDATIONS
672 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
673         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
674         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
675 #endif
676 #endif
677
678         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
679         restore_cpu_complex(flags);
680         cpu_cluster_pm_exit();
681
682         remain = tegra_pd_timer_remain();
683         if (sleep_time)
684                 tegra_pd_set_trigger(0);
685
686         if (flags & TEGRA_POWER_CLUSTER_MASK) {
687                 tegra_cluster_switch_epilog(flags);
688                 if (is_idle_task(current))
689                         trace_nvcpu_cluster_rcuidle(NVPOWER_CPU_CLUSTER_DONE);
690                 else
691                         trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_DONE);
692         } else {
693                 resume_cpu_dfll_mode();
694         }
695         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
696
697 #if INSTRUMENT_CLUSTER_SWITCH
698         if (flags & TEGRA_POWER_CLUSTER_MASK) {
699                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
700                         is_lp_cluster() ? "G=>LP" : "LP=>G",
701                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
702                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
703                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
704                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
705                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
706                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
707                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
708                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
709         }
710 #endif
711         return remain;
712 }
713
714 static int tegra_common_suspend(void)
715 {
716         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
717
718         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
719         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
720         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
721
722         /* copy the reset vector and SDRAM shutdown code into IRAM */
723         memcpy(iram_save, iram_code, iram_save_size);
724         memcpy(iram_code, tegra_iram_start(), iram_save_size);
725
726         return 0;
727 }
728
729 static void tegra_common_resume(void)
730 {
731         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
732 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
733         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
734 #endif
735
736 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
737         /* Clear DPD Enable */
738         writel(0x0, pmc + PMC_DPD_ENABLE);
739 #endif
740
741         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
742         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
743         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
744 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
745         /* trigger emc mode write */
746         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
747         /* clear scratch registers shared by suspend and the reset pen */
748         writel(0x0, pmc + PMC_SCRATCH39);
749 #endif
750         writel(0x0, pmc + PMC_SCRATCH41);
751
752         /* restore IRAM */
753         memcpy(iram_code, iram_save, iram_save_size);
754 }
755
756 static int tegra_suspend_prepare_late(void)
757 {
758 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
759         disable_irq(INT_SYS_STATS_MON);
760 #endif
761         return 0;
762 }
763
764 static void tegra_suspend_wake(void)
765 {
766 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
767         enable_irq(INT_SYS_STATS_MON);
768 #endif
769 }
770
771 static void tegra_pm_set(enum tegra_suspend_mode mode)
772 {
773         u32 reg, boot_flag;
774         unsigned long rate = 32768;
775
776         reg = readl(pmc + PMC_CTRL);
777         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
778         if (pdata->combined_req)
779                 reg &= ~TEGRA_POWER_PWRREQ_OE;
780         else
781                 reg |= TEGRA_POWER_PWRREQ_OE;
782         reg &= ~TEGRA_POWER_EFFECT_LP0;
783
784         switch (mode) {
785         case TEGRA_SUSPEND_LP0:
786 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
787                 rate = clk_get_rate_all_locked(tegra_pclk);
788 #endif
789                 if (pdata->combined_req) {
790                         reg |= TEGRA_POWER_PWRREQ_OE;
791                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
792                 }
793
794 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
795                 /*
796                  * LP0 boots through the AVP, which then resumes the AVP to
797                  * the address in scratch 39, and the cpu to the address in
798                  * scratch 41 to tegra_resume
799                  */
800                 writel(0x0, pmc + PMC_SCRATCH39);
801 #endif
802
803                 /* Enable DPD sample to trigger sampling pads data and direction
804                  * in which pad will be driven during lp0 mode*/
805                 writel(0x1, pmc + PMC_DPD_SAMPLE);
806 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
807                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
808                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
809 #endif
810
811 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
812                 /* this is needed only for T11x, not for other chips */
813                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
814 #endif
815
816                 /* Set warmboot flag */
817                 boot_flag = readl(pmc + PMC_SCRATCH0);
818                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
819
820                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
821
822                 reg |= TEGRA_POWER_EFFECT_LP0;
823                 /* No break here. LP0 code falls through to write SCRATCH41 */
824         case TEGRA_SUSPEND_LP1:
825                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
826                 wmb();
827                 break;
828         case TEGRA_SUSPEND_LP2:
829                 rate = clk_get_rate(tegra_pclk);
830                 break;
831         case TEGRA_SUSPEND_NONE:
832                 return;
833         default:
834                 BUG();
835         }
836
837         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
838
839         pmc_32kwritel(reg, PMC_CTRL);
840 }
841
842 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
843         [TEGRA_SUSPEND_NONE] = "none",
844         [TEGRA_SUSPEND_LP2] = "LP2",
845         [TEGRA_SUSPEND_LP1] = "LP1",
846         [TEGRA_SUSPEND_LP0] = "LP0",
847 };
848
849 static int tegra_suspend_enter(suspend_state_t state)
850 {
851         int ret;
852         ktime_t delta;
853         struct timespec ts_entry, ts_exit;
854
855         if (pdata && pdata->board_suspend)
856                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
857
858         read_persistent_clock(&ts_entry);
859
860         ret = tegra_suspend_dram(current_suspend_mode, 0);
861         if (ret) {
862                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
863                 goto abort_suspend;
864         }
865
866         read_persistent_clock(&ts_exit);
867
868         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
869                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
870
871                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
872                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
873                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
874                 else
875                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
876         }
877
878 abort_suspend:
879         if (pdata && pdata->board_resume)
880                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
881
882         return ret;
883 }
884
885 static void tegra_suspend_check_pwr_stats(void)
886 {
887         /* cpus and l2 are powered off later */
888         unsigned long pwrgate_partid_mask =
889 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
890                 (1 << TEGRA_POWERGATE_HEG)      |
891                 (1 << TEGRA_POWERGATE_SATA)     |
892                 (1 << TEGRA_POWERGATE_3D1)      |
893 #endif
894                 (1 << TEGRA_POWERGATE_3D)       |
895                 (1 << TEGRA_POWERGATE_VENC)     |
896                 (1 << TEGRA_POWERGATE_PCIE)     |
897                 (1 << TEGRA_POWERGATE_VDEC)     |
898                 (1 << TEGRA_POWERGATE_MPE);
899
900         int partid;
901
902         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
903                 if ((1 << partid) & pwrgate_partid_mask)
904                         if (tegra_powergate_is_powered(partid))
905                                 pr_warning("partition %s is left on before suspend\n",
906                                         tegra_powergate_get_name(partid));
907
908         return;
909 }
910
911 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
912 {
913         int err = 0;
914         u32 scratch37 = 0xDEADBEEF;
915         u32 reg;
916
917         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
918                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
919                 err = -ENXIO;
920                 goto fail;
921         }
922
923         if (tegra_is_voice_call_active()) {
924                 /* backup the current value of scratch37 */
925                 scratch37 = readl(pmc + PMC_SCRATCH37);
926
927                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
928                 reg = TEGRA_POWER_LP1_AUDIO;
929                 pmc_32kwritel(reg, PMC_SCRATCH37);
930         }
931
932         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
933                 pr_info("LP0 not used due to unsupported wakeup events\n");
934                 mode = TEGRA_SUSPEND_LP1;
935         }
936
937         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
938                 tegra_suspend_check_pwr_stats();
939
940         tegra_common_suspend();
941
942         tegra_pm_set(mode);
943
944         if (pdata && pdata->board_suspend)
945                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
946
947         local_fiq_disable();
948
949         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
950
951         if (mode == TEGRA_SUSPEND_LP0) {
952 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
953                 reg = readl(pmc + PMC_SCRATCH4);
954                 if (is_lp_cluster())
955                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
956                 else
957                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
958                 pmc_32kwritel(reg, PMC_SCRATCH4);
959 #endif
960                 tegra_tsc_suspend();
961                 tegra_lp0_suspend_mc();
962                 tegra_cpu_reset_handler_save();
963                 tegra_tsc_wait_for_suspend();
964 #ifndef CONFIG_TEGRA_SIMULATION_PLATFORM
965                 tegra_smp_clear_power_mask();
966 #endif
967         }
968         else if (mode == TEGRA_SUSPEND_LP1)
969                 *iram_cpu_lp1_mask = 1;
970
971         suspend_cpu_complex(flags);
972
973 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
974         /* In case of LP0/1, program external power gating accordinly */
975         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
976                 reg = readl(FLOW_CTRL_CPU_CSR(0));
977                 if (is_lp_cluster())
978                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
979                 else
980                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
981                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
982         }
983 #endif
984
985         flush_cache_all();
986         outer_flush_all();
987         outer_disable();
988
989         if (mode == TEGRA_SUSPEND_LP2)
990                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
991         else
992                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
993
994         tegra_init_cache(true);
995
996 #ifdef CONFIG_TRUSTED_FOUNDATIONS
997 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
998         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
999         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
1000 #endif
1001
1002         if (mode == TEGRA_SUSPEND_LP0) {
1003                 trace_secureos_init(tegra_resume_entry_time,
1004                         NVSEC_SUSPEND_EXIT_DONE);
1005         }
1006 #endif
1007
1008         if (mode == TEGRA_SUSPEND_LP0) {
1009
1010                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
1011                 * Otherwise it is creating issue in cluster switch after LP0 *
1012 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1013                 reg = readl(pmc+PMC_CTRL);
1014                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
1015                 pmc_32kwritel(reg, PMC_CTRL);
1016 #endif
1017                 */
1018
1019                 tegra_tsc_resume();
1020                 tegra_cpu_reset_handler_restore();
1021                 tegra_lp0_resume_mc();
1022                 tegra_tsc_wait_for_resume();
1023         } else if (mode == TEGRA_SUSPEND_LP1)
1024                 *iram_cpu_lp1_mask = 0;
1025
1026         /* if scratch37 was clobbered during LP1, restore it */
1027         if (scratch37 != 0xDEADBEEF)
1028                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1029
1030         restore_cpu_complex(flags);
1031
1032         /* for platforms where the core & CPU power requests are
1033          * combined as a single request to the PMU, transition out
1034          * of LP0 state by temporarily enabling both requests
1035          */
1036         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1037                 reg = readl(pmc + PMC_CTRL);
1038                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1039                 pmc_32kwritel(reg, PMC_CTRL);
1040                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1041                 pmc_32kwritel(reg, PMC_CTRL);
1042         }
1043
1044         if (pdata && pdata->board_resume)
1045                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1046
1047         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1048
1049         local_fiq_enable();
1050
1051         tegra_common_resume();
1052
1053 fail:
1054         return err;
1055 }
1056
1057 /*
1058  * Function pointers to optional board specific function
1059  */
1060 void (*tegra_deep_sleep)(int);
1061 EXPORT_SYMBOL(tegra_deep_sleep);
1062
1063 static int tegra_suspend_prepare(void)
1064 {
1065         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1066                 tegra_deep_sleep(1);
1067         return 0;
1068 }
1069
1070 static void tegra_suspend_finish(void)
1071 {
1072         if (pdata && pdata->cpu_resume_boost) {
1073                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1074                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1075                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1076         }
1077
1078         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1079                 tegra_deep_sleep(0);
1080 }
1081
1082 static const struct platform_suspend_ops tegra_suspend_ops = {
1083         .valid          = suspend_valid_only_mem,
1084         .prepare        = tegra_suspend_prepare,
1085         .finish         = tegra_suspend_finish,
1086         .prepare_late   = tegra_suspend_prepare_late,
1087         .wake           = tegra_suspend_wake,
1088         .enter          = tegra_suspend_enter,
1089 };
1090
1091 static ssize_t suspend_mode_show(struct kobject *kobj,
1092                                         struct kobj_attribute *attr, char *buf)
1093 {
1094         char *start = buf;
1095         char *end = buf + PAGE_SIZE;
1096
1097         start += scnprintf(start, end - start, "%s ", \
1098                                 tegra_suspend_name[current_suspend_mode]);
1099         start += scnprintf(start, end - start, "\n");
1100
1101         return start - buf;
1102 }
1103
1104 static ssize_t suspend_mode_store(struct kobject *kobj,
1105                                         struct kobj_attribute *attr,
1106                                         const char *buf, size_t n)
1107 {
1108         int len;
1109         const char *name_ptr;
1110         enum tegra_suspend_mode new_mode;
1111
1112         name_ptr = buf;
1113         while (*name_ptr && !isspace(*name_ptr))
1114                 name_ptr++;
1115         len = name_ptr - buf;
1116         if (!len)
1117                 goto bad_name;
1118         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1119         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1120                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1121                 pr_info("Illegal tegra suspend state: %s\n", buf);
1122                 goto bad_name;
1123         }
1124
1125         for (new_mode = TEGRA_SUSPEND_NONE; \
1126                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1127                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1128                         current_suspend_mode = new_mode;
1129                         break;
1130                 }
1131         }
1132
1133 bad_name:
1134         return n;
1135 }
1136
1137 static struct kobj_attribute suspend_mode_attribute =
1138         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1139
1140 static struct kobject *suspend_kobj;
1141
1142 static int tegra_pm_enter_suspend(void)
1143 {
1144         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1145         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1146                 tegra_lp0_cpu_mode(true);
1147         return 0;
1148 }
1149
1150 static void tegra_pm_enter_resume(void)
1151 {
1152         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1153                 tegra_lp0_cpu_mode(false);
1154         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1155 }
1156
1157 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1158         .suspend = tegra_pm_enter_suspend,
1159         .resume = tegra_pm_enter_resume,
1160 };
1161
1162 static __init int tegra_pm_enter_syscore_init(void)
1163 {
1164         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1165         return 0;
1166 }
1167 subsys_initcall(tegra_pm_enter_syscore_init);
1168 #endif
1169
1170 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1171 {
1172         u32 reg;
1173         u32 mode;
1174
1175 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1176         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1177         BUG_ON(IS_ERR(tegra_dfll));
1178 #endif
1179         tegra_pclk = clk_get_sys(NULL, "pclk");
1180         BUG_ON(IS_ERR(tegra_pclk));
1181         pdata = plat;
1182         (void)reg;
1183         (void)mode;
1184
1185         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1186                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1187
1188 #ifndef CONFIG_PM_SLEEP
1189         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1190                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1191                            "disabling suspend\n", __func__);
1192                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1193         }
1194 #else
1195         if (create_suspend_pgtable() < 0) {
1196                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1197                                 __func__);
1198                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1199                 goto fail;
1200         }
1201
1202         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1203             (tegra_revision == TEGRA_REVISION_A01) &&
1204             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1205                 /* Tegra 3 A01 supports only LP1 */
1206                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1207                            "-- disabling LP0\n", __func__);
1208                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1209         }
1210         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1211                 tegra_lp0_vec_relocate) {
1212                 unsigned char *reloc_lp0;
1213                 unsigned long tmp;
1214                 void __iomem *orig;
1215                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1216                                         GFP_KERNEL);
1217                 WARN_ON(!reloc_lp0);
1218                 if (!reloc_lp0) {
1219                         pr_err("%s: Failed to allocate reloc_lp0\n",
1220                                 __func__);
1221                         goto out;
1222                 }
1223
1224                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1225                 WARN_ON(!orig);
1226                 if (!orig) {
1227                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1228                                 __func__, tegra_lp0_vec_start);
1229                         kfree(reloc_lp0);
1230                         goto out;
1231                 }
1232
1233                 tmp = (unsigned long) reloc_lp0;
1234                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1235                 reloc_lp0 = (unsigned char *)tmp;
1236                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1237                 iounmap(orig);
1238                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1239         }
1240
1241 out:
1242         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1243                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1244                            "provided by bootlader -- disabling LP0\n",
1245                            __func__);
1246                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1247         }
1248
1249         iram_save_size = tegra_iram_end() - tegra_iram_start();
1250
1251         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1252         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1253                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1254                        "-- LP0/LP1 unavailable\n", __func__);
1255                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1256         }
1257
1258 #ifdef CONFIG_TEGRA_LP1_950
1259         if (pdata->lp1_lowvolt_support) {
1260                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1261                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1262                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1263
1264                 lp1_core_lowvolt = 0;
1265                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1266                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1267
1268                 lp1_core_highvolt = 0;
1269                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1270                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1271         }
1272 #endif
1273         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1274         /* Initialize scratch registers used for CPU LP2 synchronization */
1275         writel(0, pmc + PMC_SCRATCH37);
1276         writel(0, pmc + PMC_SCRATCH38);
1277 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1278         writel(0, pmc + PMC_SCRATCH39);
1279 #endif
1280         writel(0, pmc + PMC_SCRATCH41);
1281
1282         /* Always enable CPU power request; just normal polarity is supported */
1283         reg = readl(pmc + PMC_CTRL);
1284         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1285         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1286         pmc_32kwritel(reg, PMC_CTRL);
1287
1288         /* Configure core power request and system clock control if LP0
1289            is supported */
1290         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1291         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1292
1293         reg = readl(pmc + PMC_CTRL);
1294
1295         if (!pdata->sysclkreq_high)
1296                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1297         else
1298                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1299
1300         if (!pdata->corereq_high)
1301                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1302         else
1303                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1304
1305         /* configure output inverters while the request is tristated */
1306         pmc_32kwritel(reg, PMC_CTRL);
1307
1308         /* now enable requests */
1309         reg |= TEGRA_POWER_SYSCLK_OE;
1310         if (!pdata->combined_req)
1311                 reg |= TEGRA_POWER_PWRREQ_OE;
1312         pmc_32kwritel(reg, PMC_CTRL);
1313
1314         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1315                 tegra_lp0_suspend_init();
1316
1317         suspend_set_ops(&tegra_suspend_ops);
1318
1319         /* Create /sys/power/suspend/type */
1320         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1321         if (suspend_kobj) {
1322                 if (sysfs_create_file(suspend_kobj, \
1323                                                 &suspend_mode_attribute.attr))
1324                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1325                                                                 __func__);
1326         }
1327
1328         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1329         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1330
1331         /* clear io dpd settings before kernel */
1332         tegra_bl_io_dpd_cleanup();
1333
1334 fail:
1335 #endif
1336         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1337                 tegra_pd_in_idle(false);
1338
1339         current_suspend_mode = plat->suspend_mode;
1340 }
1341
1342 unsigned long debug_uart_port_base = 0;
1343 EXPORT_SYMBOL(debug_uart_port_base);
1344
1345 static int tegra_debug_uart_suspend(void)
1346 {
1347         void __iomem *uart;
1348         u32 lcr;
1349
1350         if (!debug_uart_port_base)
1351                 return 0;
1352
1353         uart = IO_ADDRESS(debug_uart_port_base);
1354
1355         lcr = readb(uart + UART_LCR * 4);
1356
1357         tegra_sctx.uart[0] = lcr;
1358         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1359
1360         /* DLAB = 0 */
1361         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1362
1363         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1364
1365         /* DLAB = 1 */
1366         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1367
1368         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1369         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1370
1371         writeb(lcr, uart + UART_LCR * 4);
1372
1373         return 0;
1374 }
1375
1376 static void tegra_debug_uart_resume(void)
1377 {
1378         void __iomem *uart;
1379         u32 lcr;
1380
1381         if (!debug_uart_port_base)
1382                 return;
1383
1384         uart = IO_ADDRESS(debug_uart_port_base);
1385
1386         lcr = tegra_sctx.uart[0];
1387
1388         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1389
1390         /* DLAB = 0 */
1391         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1392
1393         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1394                         uart + UART_FCR * 4);
1395
1396         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1397
1398         /* DLAB = 1 */
1399         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1400
1401         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1402         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1403
1404         writeb(lcr, uart + UART_LCR * 4);
1405 }
1406
1407 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1408         .suspend = tegra_debug_uart_suspend,
1409         .resume = tegra_debug_uart_resume,
1410 };
1411
1412 struct clk *debug_uart_clk = NULL;
1413 EXPORT_SYMBOL(debug_uart_clk);
1414
1415 void tegra_console_uart_suspend(void)
1416 {
1417         if (console_suspend_enabled && debug_uart_clk)
1418                 tegra_clk_disable_unprepare(debug_uart_clk);
1419 }
1420
1421 void tegra_console_uart_resume(void)
1422 {
1423         if (console_suspend_enabled && debug_uart_clk)
1424                 tegra_clk_prepare_enable(debug_uart_clk);
1425 }
1426
1427 static int tegra_debug_uart_syscore_init(void)
1428 {
1429         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1430         return 0;
1431 }
1432 arch_initcall(tegra_debug_uart_syscore_init);