63973a5219da6416a5cc19a53a2e666b30f86482
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2013, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50 #include <trace/events/nvsecurity.h>
51
52 #include <asm/cacheflush.h>
53 #include <asm/idmap.h>
54 #include <asm/localtimer.h>
55 #include <asm/pgalloc.h>
56 #include <asm/pgtable.h>
57 #include <asm/tlbflush.h>
58 #include <asm/suspend.h>
59 #include <asm/smp_plat.h>
60
61 #include <mach/irqs.h>
62 #include <mach/powergate.h>
63 #include <mach/hardware.h>
64
65 #include "board.h"
66 #include "clock.h"
67 #include "common.h"
68 #include "cpuidle.h"
69 #include "fuse.h"
70 #include "gic.h"
71 #include "iomap.h"
72 #include "pm.h"
73 #include "pm-irq.h"
74 #include "reset.h"
75 #include "sleep.h"
76 #include "timer.h"
77 #include "dvfs.h"
78 #include "cpu-tegra.h"
79
80 #define CREATE_TRACE_POINTS
81 #include <trace/events/nvpower.h>
82
83 struct suspend_context {
84         /*
85          * The next 7 values are referenced by offset in __restart_plls
86          * in headsmp-t2.S, and should not be moved
87          */
88         u32 pllx_misc;
89         u32 pllx_base;
90         u32 pllp_misc;
91         u32 pllp_base;
92         u32 pllp_outa;
93         u32 pllp_outb;
94         u32 pll_timeout;
95
96         u32 cpu_burst;
97         u32 clk_csite_src;
98         u32 cclk_divider;
99
100         u32 mc[3];
101         u8 uart[5];
102
103         struct tegra_twd_context twd;
104 };
105
106 #ifdef CONFIG_PM_SLEEP
107 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
108 static pgd_t *tegra_pgd;
109 static DEFINE_SPINLOCK(tegra_lp2_lock);
110 static cpumask_t tegra_in_lp2;
111 static cpumask_t *iram_cpu_lp2_mask;
112 static unsigned long *iram_cpu_lp1_mask;
113 static u8 *iram_save;
114 static unsigned long iram_save_size;
115 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
116 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
117 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
118 static int tegra_last_pclk;
119 #endif
120
121 struct suspend_context tegra_sctx;
122
123 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
124 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
125 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
126 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
127 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
128 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
129 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
130 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
131 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
132
133 #define PMC_CTRL                0x0
134 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
135 #define PMC_WAKE_MASK           0xc
136 #define PMC_WAKE_LEVEL          0x10
137 #define PMC_DPAD_ORIDE          0x1C
138 #define PMC_WAKE_DELAY          0xe0
139 #define PMC_DPD_SAMPLE          0x20
140 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
141 #define PMC_DPD_ENABLE          0x24
142 #endif
143 #define PMC_IO_DPD_REQ          0x1B8
144 #define PMC_IO_DPD2_REQ         0x1C0
145
146
147 #define PMC_WAKE_STATUS         0x14
148 #define PMC_SW_WAKE_STATUS      0x18
149 #define PMC_COREPWRGOOD_TIMER   0x3c
150 #define PMC_CPUPWRGOOD_TIMER    0xc8
151 #define PMC_CPUPWROFF_TIMER     0xcc
152 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
153
154 #define PMC_PWRGATE_TOGGLE      0x30
155 #define PWRGATE_TOGGLE_START    (1 << 8)
156 #define UN_PWRGATE_CPU          \
157         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
158
159 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
160 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
161 #endif
162
163 #define CLK_RESET_CCLK_BURST    0x20
164 #define CLK_RESET_CCLK_DIVIDER  0x24
165 #define CLK_RESET_PLLC_BASE     0x80
166 #define CLK_RESET_PLLM_BASE     0x90
167 #define CLK_RESET_PLLX_BASE     0xe0
168 #define CLK_RESET_PLLX_MISC     0xe4
169 #define CLK_RESET_PLLP_BASE     0xa0
170 #define CLK_RESET_PLLP_OUTA     0xa4
171 #define CLK_RESET_PLLP_OUTB     0xa8
172 #define CLK_RESET_PLLP_MISC     0xac
173
174 #define CLK_RESET_SOURCE_CSITE  0x1d4
175
176 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
177 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
178 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
179 #define CLK_RESET_CCLK_IDLE_POLICY         1
180 #define CLK_RESET_CCLK_RUN_POLICY          2
181 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
182 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
183
184 #define EMC_MRW_0               0x0e8
185 #define EMC_MRW_DEV_SELECTN     30
186 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
187
188 #define MC_SECURITY_START       0x6c
189 #define MC_SECURITY_SIZE        0x70
190 #define MC_SECURITY_CFG2        0x7c
191
192 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
193 static struct clk *tegra_dfll;
194 #endif
195 static struct clk *tegra_pclk;
196 static const struct tegra_suspend_platform_data *pdata;
197 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
198
199 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
200 enum tegra_cluster_switch_time_id {
201         tegra_cluster_switch_time_id_start = 0,
202         tegra_cluster_switch_time_id_prolog,
203         tegra_cluster_switch_time_id_switch,
204         tegra_cluster_switch_time_id_epilog,
205         tegra_cluster_switch_time_id_max
206 };
207
208 static unsigned long
209                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
210 #define tegra_cluster_switch_time(flags, id) \
211         do { \
212                 barrier(); \
213                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
214                         void __iomem *timer_us = \
215                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
216                         if (id < tegra_cluster_switch_time_id_max) \
217                                 tegra_cluster_switch_times[id] = \
218                                                         readl(timer_us); \
219                                 wmb(); \
220                 } \
221                 barrier(); \
222         } while(0)
223 #else
224 #define tegra_cluster_switch_time(flags, id) do {} while(0)
225 #endif
226
227 #ifdef CONFIG_PM_SLEEP
228 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
229         [TEGRA_SUSPEND_NONE]    = "none",
230         [TEGRA_SUSPEND_LP2]     = "lp2",
231         [TEGRA_SUSPEND_LP1]     = "lp1",
232         [TEGRA_SUSPEND_LP0]     = "lp0",
233 };
234
235 unsigned long tegra_cpu_power_good_time(void)
236 {
237         if (WARN_ON_ONCE(!pdata))
238                 return 5000;
239
240         return pdata->cpu_timer;
241 }
242
243 unsigned long tegra_cpu_power_off_time(void)
244 {
245         if (WARN_ON_ONCE(!pdata))
246                 return 5000;
247
248         return pdata->cpu_off_timer;
249 }
250
251 unsigned long tegra_cpu_lp2_min_residency(void)
252 {
253         if (WARN_ON_ONCE(!pdata))
254                 return 2000;
255
256         return pdata->cpu_lp2_min_residency;
257 }
258
259 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
260 unsigned long tegra_min_residency_noncpu(void)
261 {
262         return pdata->min_residency_noncpu;
263 }
264
265 unsigned long tegra_min_residency_crail(void)
266 {
267         return pdata->min_residency_crail;
268 }
269 #endif
270
271 static void suspend_cpu_dfll_mode(void)
272 {
273 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
274         /* If DFLL is used as CPU clock source go to open loop mode */
275         if (!is_lp_cluster() && tegra_dfll &&
276             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
277                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
278 #endif
279 }
280
281 static void resume_cpu_dfll_mode(void)
282 {
283 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
284         /* If DFLL is used as CPU clock source restore closed loop mode */
285         if (!is_lp_cluster() && tegra_dfll &&
286             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
287                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
288 #endif
289 }
290
291 /*
292  * create_suspend_pgtable
293  *
294  * Creates a page table with identity mappings of physical memory and IRAM
295  * for use when the MMU is off, in addition to all the regular kernel mappings.
296  */
297 static __init int create_suspend_pgtable(void)
298 {
299         tegra_pgd = pgd_alloc(&init_mm);
300         if (!tegra_pgd)
301                 return -ENOMEM;
302
303         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
304         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
305                 high_memory, 0);
306         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
307                 IO_IRAM_VIRT + SECTION_SIZE, 0);
308
309         /* inner/outer write-back/write-allocate, sharable */
310         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
311
312         return 0;
313 }
314
315 /* ensures that sufficient time is passed for a register write to
316  * serialize into the 32KHz domain */
317 static void pmc_32kwritel(u32 val, unsigned long offs)
318 {
319         writel(val, pmc + offs);
320         udelay(130);
321 }
322
323 static void set_power_timers(unsigned long us_on, unsigned long us_off,
324                              long rate)
325 {
326         static unsigned long last_us_off = 0;
327         unsigned long long ticks;
328         unsigned long long pclk;
329
330         if (WARN_ON_ONCE(rate <= 0))
331                 pclk = 100000000;
332         else
333                 pclk = rate;
334
335         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
336                 ticks = (us_on * pclk) + 999999ull;
337                 do_div(ticks, 1000000);
338                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
339
340                 ticks = (us_off * pclk) + 999999ull;
341                 do_div(ticks, 1000000);
342                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
343                 wmb();
344         }
345         tegra_last_pclk = pclk;
346         last_us_off = us_off;
347 }
348
349 /*
350  * restore_cpu_complex
351  *
352  * restores cpu clock setting, clears flow controller
353  *
354  * Always called on CPU 0.
355  */
356 static void restore_cpu_complex(u32 mode)
357 {
358         int cpu = cpu_logical_map(smp_processor_id());
359         unsigned int reg;
360 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
361         unsigned int policy;
362 #endif
363
364 /*
365  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
366  * or restored by common clock suspend/resume procedures. Hence, we don't
367  * need it here.
368  */
369 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
370         /* Is CPU complex already running on PLLX? */
371         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
372         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
373
374         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
375                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
376         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
377                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
378         else
379                 BUG();
380
381         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
382                 /* restore PLLX settings if CPU is on different PLL */
383                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
384                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
385
386                 /* wait for PLL stabilization if PLLX was enabled */
387                 if (tegra_sctx.pllx_base & (1<<30)) {
388 #if USE_PLL_LOCK_BITS
389                         /* Enable lock detector */
390                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
391                         reg |= 1<<18;
392                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
393                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
394                                  (1<<27)))
395                                 cpu_relax();
396
397                         udelay(PLL_POST_LOCK_DELAY);
398 #else
399                         udelay(300);
400 #endif
401                 }
402         }
403
404         /* Restore original burst policy setting for calls resulting from CPU
405            LP2 in idle or system suspend; keep cluster switch prolog setting
406            intact. */
407         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
408                 writel(tegra_sctx.cclk_divider, clk_rst +
409                        CLK_RESET_CCLK_DIVIDER);
410                 writel(tegra_sctx.cpu_burst, clk_rst +
411                        CLK_RESET_CCLK_BURST);
412         }
413 #endif
414         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
415
416         /* Do not power-gate CPU 0 when flow controlled */
417         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
418         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
419         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
420         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
421         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
422         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
423         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
424
425         /* If an immedidate cluster switch is being perfomed, restore the
426            local timer registers. For calls resulting from CPU LP2 in
427            idle or system suspend, the local timer was shut down and
428            timekeeping switched over to the global system timer. In this
429            case keep local timer disabled, and restore only periodic load. */
430 #ifdef CONFIG_HAVE_ARM_TWD
431         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
432                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
433                 tegra_sctx.twd.twd_ctrl = 0;
434         }
435         tegra_twd_resume(&tegra_sctx.twd);
436 #endif
437 }
438
439 /*
440  * suspend_cpu_complex
441  *
442  * saves pll state for use by restart_plls, prepares flow controller for
443  * transition to suspend state
444  *
445  * Must always be called on cpu 0.
446  */
447 static void suspend_cpu_complex(u32 mode)
448 {
449         int cpu = cpu_logical_map(smp_processor_id());
450         unsigned int reg;
451         int i;
452
453         BUG_ON(cpu != 0);
454
455         /* switch coresite to clk_m, save off original source */
456         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
457         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
458
459         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
460         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
461         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
462         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
463         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
464         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
465         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
466         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
467
468 #ifdef CONFIG_HAVE_ARM_TWD
469         tegra_twd_suspend(&tegra_sctx.twd);
470 #endif
471
472         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
473         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
474         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
475         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
476         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
477 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
478         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
479 #else
480         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
481 #endif
482         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
483         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
484
485         for (i = 0; i < num_possible_cpus(); i++) {
486                 if (i == cpu)
487                         continue;
488                 reg = readl(FLOW_CTRL_CPU_CSR(i));
489                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
490                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
491                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
492         }
493
494         tegra_gic_cpu_disable(true);
495 }
496
497 void tegra_clear_cpu_in_pd(int cpu)
498 {
499         spin_lock(&tegra_lp2_lock);
500         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
501         cpumask_clear_cpu(cpu, &tegra_in_lp2);
502
503         /* Update the IRAM copy used by the reset handler. The IRAM copy
504            can't use used directly by cpumask_clear_cpu() because it uses
505            LDREX/STREX which requires the addressed location to be inner
506            cacheable and sharable which IRAM isn't. */
507         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
508         dsb();
509
510         spin_unlock(&tegra_lp2_lock);
511 }
512
513 bool tegra_set_cpu_in_pd(int cpu)
514 {
515         bool last_cpu = false;
516
517         spin_lock(&tegra_lp2_lock);
518         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
519         cpumask_set_cpu(cpu, &tegra_in_lp2);
520
521         /* Update the IRAM copy used by the reset handler. The IRAM copy
522            can't use used directly by cpumask_set_cpu() because it uses
523            LDREX/STREX which requires the addressed location to be inner
524            cacheable and sharable which IRAM isn't. */
525         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
526         dsb();
527
528         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
529                 last_cpu = true;
530 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
531         else if (cpu == 1)
532                 tegra2_cpu_set_resettable_soon();
533 #endif
534
535         spin_unlock(&tegra_lp2_lock);
536         return last_cpu;
537 }
538
539 static void tegra_sleep_core(enum tegra_suspend_mode mode,
540                              unsigned long v2p)
541 {
542 #ifdef CONFIG_TRUSTED_FOUNDATIONS
543         outer_flush_range(__pa(&tegra_resume_timestamps_start),
544                           __pa(&tegra_resume_timestamps_end));
545
546         if (mode == TEGRA_SUSPEND_LP0) {
547                 trace_smc_sleep_core(NVSEC_SMC_START);
548
549                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
550                                   virt_to_phys(tegra_resume));
551         } else {
552                 trace_smc_sleep_core(NVSEC_SMC_START);
553
554                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
555                                   (TEGRA_RESET_HANDLER_BASE +
556                                    tegra_cpu_reset_handler_offset));
557         }
558
559         trace_smc_sleep_core(NVSEC_SMC_DONE);
560 #endif
561 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
562         cpu_suspend(v2p, tegra2_sleep_core_finish);
563 #else
564         cpu_suspend(v2p, tegra3_sleep_core_finish);
565 #endif
566 }
567
568 static inline void tegra_sleep_cpu(unsigned long v2p)
569 {
570         cpu_suspend(v2p, tegra_sleep_cpu_finish);
571 }
572
573 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
574                                         unsigned int flags)
575 {
576         u32 reg;
577         unsigned int remain;
578 #if defined(CONFIG_CACHE_L2X0) && !defined(CONFIG_ARCH_TEGRA_14x_SOC)
579         pgd_t *pgd;
580 #endif
581
582         /* Only the last cpu down does the final suspend steps */
583         reg = readl(pmc + PMC_CTRL);
584         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
585         if (pdata->combined_req)
586                 reg &= ~TEGRA_POWER_PWRREQ_OE;
587         else
588                 reg |= TEGRA_POWER_PWRREQ_OE;
589
590         reg &= ~TEGRA_POWER_EFFECT_LP0;
591         writel(reg, pmc + PMC_CTRL);
592
593         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
594
595         /*
596          * We can use clk_get_rate_all_locked() here, because all other cpus
597          * are in LP2 state and irqs are disabled
598          */
599         if (flags & TEGRA_POWER_CLUSTER_MASK) {
600                 trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_START);
601                 set_power_timers(pdata->cpu_timer, 2,
602                         clk_get_rate_all_locked(tegra_pclk));
603                 if (flags & TEGRA_POWER_CLUSTER_G) {
604                         /*
605                          * To reduce the vdd_cpu up latency when LP->G
606                          * transition. Before the transition, enable
607                          * the vdd_cpu rail.
608                          */
609                         if (is_lp_cluster()) {
610 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
611                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
612                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
613                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
614 #else
615                                 writel(UN_PWRGATE_CPU,
616                                        pmc + PMC_PWRGATE_TOGGLE);
617 #endif
618                         }
619                 }
620                 tegra_cluster_switch_prolog(flags);
621         } else {
622                 suspend_cpu_dfll_mode();
623                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
624                         clk_get_rate_all_locked(tegra_pclk));
625 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
626                 reg = readl(FLOW_CTRL_CPU_CSR(0));
627                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
628                 if (is_lp_cluster()) {
629                         /* for LP cluster, there is no option for rail gating */
630                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
631                                                 TEGRA_POWER_CLUSTER_PART_MASK)
632                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
633                         else if (flags)
634                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
635                 }
636                 else {
637                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
638                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
639                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
640                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
641                 }
642                 writel(reg, FLOW_CTRL_CPU_CSR(0));
643 #endif
644         }
645
646         if (sleep_time)
647                 tegra_pd_set_trigger(sleep_time);
648
649         cpu_cluster_pm_enter();
650         suspend_cpu_complex(flags);
651         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
652 #ifdef CONFIG_CACHE_L2X0
653         flush_cache_all();
654 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
655         outer_flush_all();
656 #else
657         /*
658          * No need to flush complete L2. Cleaning kernel and IO mappings
659          * is enough for the LP code sequence that has L2 disabled but
660          * MMU on.
661          */
662         pgd = cpu_get_pgd();
663         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
664                           __pa(pgd + PTRS_PER_PGD));
665 #endif
666         outer_disable();
667 #endif
668         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
669
670 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
671         tegra_init_cache(true);
672 #else
673         tegra_init_cache(false);
674 #endif
675
676 #ifdef CONFIG_TRUSTED_FOUNDATIONS
677 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
678         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
679         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
680 #endif
681 #endif
682
683         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
684         restore_cpu_complex(flags);
685         cpu_cluster_pm_exit();
686
687         remain = tegra_pd_timer_remain();
688         if (sleep_time)
689                 tegra_pd_set_trigger(0);
690
691         if (flags & TEGRA_POWER_CLUSTER_MASK) {
692                 tegra_cluster_switch_epilog(flags);
693                 if (is_idle_task(current))
694                         trace_nvcpu_cluster_rcuidle(NVPOWER_CPU_CLUSTER_DONE);
695                 else
696                         trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_DONE);
697         } else {
698                 resume_cpu_dfll_mode();
699         }
700         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
701
702 #if INSTRUMENT_CLUSTER_SWITCH
703         if (flags & TEGRA_POWER_CLUSTER_MASK) {
704                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
705                         is_lp_cluster() ? "G=>LP" : "LP=>G",
706                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
707                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
708                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
709                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
710                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
711                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
712                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
713                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
714         }
715 #endif
716         return remain;
717 }
718
719 static int tegra_common_suspend(void)
720 {
721         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
722
723         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
724         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
725         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
726
727         /* copy the reset vector and SDRAM shutdown code into IRAM */
728         memcpy(iram_save, iram_code, iram_save_size);
729         memcpy(iram_code, tegra_iram_start(), iram_save_size);
730
731         return 0;
732 }
733
734 static void tegra_common_resume(void)
735 {
736         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
737 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
738         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
739 #endif
740
741 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
742         /* Clear DPD Enable */
743         writel(0x0, pmc + PMC_DPD_ENABLE);
744 #endif
745
746         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
747         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
748         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
749 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
750         /* trigger emc mode write */
751         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
752         /* clear scratch registers shared by suspend and the reset pen */
753         writel(0x0, pmc + PMC_SCRATCH39);
754 #endif
755         writel(0x0, pmc + PMC_SCRATCH41);
756
757         /* restore IRAM */
758         memcpy(iram_code, iram_save, iram_save_size);
759 }
760
761 static int tegra_suspend_prepare_late(void)
762 {
763 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
764         disable_irq(INT_SYS_STATS_MON);
765 #endif
766         return 0;
767 }
768
769 static void tegra_suspend_wake(void)
770 {
771 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
772         enable_irq(INT_SYS_STATS_MON);
773 #endif
774 }
775
776 static void tegra_pm_set(enum tegra_suspend_mode mode)
777 {
778         u32 reg, boot_flag;
779         unsigned long rate = 32768;
780
781         reg = readl(pmc + PMC_CTRL);
782         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
783         if (pdata->combined_req)
784                 reg &= ~TEGRA_POWER_PWRREQ_OE;
785         else
786                 reg |= TEGRA_POWER_PWRREQ_OE;
787         reg &= ~TEGRA_POWER_EFFECT_LP0;
788
789         switch (mode) {
790         case TEGRA_SUSPEND_LP0:
791 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
792                 rate = clk_get_rate_all_locked(tegra_pclk);
793 #endif
794                 if (pdata->combined_req) {
795                         reg |= TEGRA_POWER_PWRREQ_OE;
796                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
797                 }
798
799 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
800                 /*
801                  * LP0 boots through the AVP, which then resumes the AVP to
802                  * the address in scratch 39, and the cpu to the address in
803                  * scratch 41 to tegra_resume
804                  */
805                 writel(0x0, pmc + PMC_SCRATCH39);
806 #endif
807
808                 /* Enable DPD sample to trigger sampling pads data and direction
809                  * in which pad will be driven during lp0 mode*/
810                 writel(0x1, pmc + PMC_DPD_SAMPLE);
811 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
812                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
813                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
814 #endif
815
816 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
817                 /* this is needed only for T11x, not for other chips */
818                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
819 #endif
820
821                 /* Set warmboot flag */
822                 boot_flag = readl(pmc + PMC_SCRATCH0);
823                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
824
825                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
826
827                 reg |= TEGRA_POWER_EFFECT_LP0;
828                 /* No break here. LP0 code falls through to write SCRATCH41 */
829         case TEGRA_SUSPEND_LP1:
830                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
831                 wmb();
832                 break;
833         case TEGRA_SUSPEND_LP2:
834                 rate = clk_get_rate(tegra_pclk);
835                 break;
836         case TEGRA_SUSPEND_NONE:
837                 return;
838         default:
839                 BUG();
840         }
841
842         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
843
844         pmc_32kwritel(reg, PMC_CTRL);
845 }
846
847 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
848         [TEGRA_SUSPEND_NONE] = "none",
849         [TEGRA_SUSPEND_LP2] = "LP2",
850         [TEGRA_SUSPEND_LP1] = "LP1",
851         [TEGRA_SUSPEND_LP0] = "LP0",
852 };
853
854 static int tegra_suspend_enter(suspend_state_t state)
855 {
856         int ret;
857         ktime_t delta;
858         struct timespec ts_entry, ts_exit;
859
860         if (pdata && pdata->board_suspend)
861                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
862
863         read_persistent_clock(&ts_entry);
864
865         ret = tegra_suspend_dram(current_suspend_mode, 0);
866         if (ret) {
867                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
868                 goto abort_suspend;
869         }
870
871         read_persistent_clock(&ts_exit);
872
873         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
874                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
875
876                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
877                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
878                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
879                 else
880                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
881         }
882
883 abort_suspend:
884         if (pdata && pdata->board_resume)
885                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
886
887         return ret;
888 }
889
890 static void tegra_suspend_check_pwr_stats(void)
891 {
892         /* cpus and l2 are powered off later */
893         unsigned long pwrgate_partid_mask =
894 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
895                 (1 << TEGRA_POWERGATE_HEG)      |
896                 (1 << TEGRA_POWERGATE_SATA)     |
897                 (1 << TEGRA_POWERGATE_3D1)      |
898 #endif
899                 (1 << TEGRA_POWERGATE_3D)       |
900                 (1 << TEGRA_POWERGATE_VENC)     |
901                 (1 << TEGRA_POWERGATE_PCIE)     |
902                 (1 << TEGRA_POWERGATE_VDEC)     |
903                 (1 << TEGRA_POWERGATE_MPE);
904
905         int partid;
906
907         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
908                 if ((1 << partid) & pwrgate_partid_mask)
909                         if (tegra_powergate_is_powered(partid))
910                                 pr_warning("partition %s is left on before suspend\n",
911                                         tegra_powergate_get_name(partid));
912
913         return;
914 }
915
916 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
917 {
918         int err = 0;
919         u32 scratch37 = 0xDEADBEEF;
920         u32 reg;
921
922         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
923                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
924                 err = -ENXIO;
925                 goto fail;
926         }
927
928         if (tegra_is_voice_call_active()) {
929                 /* backup the current value of scratch37 */
930                 scratch37 = readl(pmc + PMC_SCRATCH37);
931
932                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
933                 reg = TEGRA_POWER_LP1_AUDIO;
934                 pmc_32kwritel(reg, PMC_SCRATCH37);
935         }
936
937         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
938                 pr_info("LP0 not used due to unsupported wakeup events\n");
939                 mode = TEGRA_SUSPEND_LP1;
940         }
941
942         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
943                 tegra_suspend_check_pwr_stats();
944
945         tegra_common_suspend();
946
947         tegra_pm_set(mode);
948
949         if (pdata && pdata->board_suspend)
950                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
951
952         local_fiq_disable();
953
954         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
955
956         if (mode == TEGRA_SUSPEND_LP0) {
957 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
958                 reg = readl(pmc + PMC_SCRATCH4);
959                 if (is_lp_cluster())
960                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
961                 else
962                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
963                 pmc_32kwritel(reg, PMC_SCRATCH4);
964 #endif
965                 tegra_tsc_suspend();
966                 tegra_lp0_suspend_mc();
967                 tegra_cpu_reset_handler_save();
968                 tegra_tsc_wait_for_suspend();
969 #ifndef CONFIG_TEGRA_SIMULATION_PLATFORM
970                 tegra_smp_clear_power_mask();
971 #endif
972         }
973         else if (mode == TEGRA_SUSPEND_LP1)
974                 *iram_cpu_lp1_mask = 1;
975
976         suspend_cpu_complex(flags);
977
978 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
979         /* In case of LP0/1, program external power gating accordinly */
980         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
981                 reg = readl(FLOW_CTRL_CPU_CSR(0));
982                 if (is_lp_cluster())
983                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
984                 else
985                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
986                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
987         }
988 #endif
989
990         flush_cache_all();
991         outer_flush_all();
992         outer_disable();
993
994         if (mode == TEGRA_SUSPEND_LP2)
995                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
996         else
997                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
998
999         tegra_init_cache(true);
1000
1001 #ifdef CONFIG_TRUSTED_FOUNDATIONS
1002 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
1003         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
1004         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
1005 #endif
1006
1007         if (mode == TEGRA_SUSPEND_LP0) {
1008                 trace_secureos_init(tegra_resume_entry_time,
1009                         NVSEC_SUSPEND_EXIT_DONE);
1010         }
1011 #endif
1012
1013         if (mode == TEGRA_SUSPEND_LP0) {
1014
1015                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
1016                 * Otherwise it is creating issue in cluster switch after LP0 *
1017 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1018                 reg = readl(pmc+PMC_CTRL);
1019                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
1020                 pmc_32kwritel(reg, PMC_CTRL);
1021 #endif
1022                 */
1023
1024                 tegra_tsc_resume();
1025                 tegra_cpu_reset_handler_restore();
1026                 tegra_lp0_resume_mc();
1027                 tegra_tsc_wait_for_resume();
1028         } else if (mode == TEGRA_SUSPEND_LP1)
1029                 *iram_cpu_lp1_mask = 0;
1030
1031         /* if scratch37 was clobbered during LP1, restore it */
1032         if (scratch37 != 0xDEADBEEF)
1033                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1034
1035         restore_cpu_complex(flags);
1036
1037         /* for platforms where the core & CPU power requests are
1038          * combined as a single request to the PMU, transition out
1039          * of LP0 state by temporarily enabling both requests
1040          */
1041         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1042                 reg = readl(pmc + PMC_CTRL);
1043                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1044                 pmc_32kwritel(reg, PMC_CTRL);
1045                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1046                 pmc_32kwritel(reg, PMC_CTRL);
1047         }
1048
1049         if (pdata && pdata->board_resume)
1050                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1051
1052         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1053
1054         local_fiq_enable();
1055
1056         tegra_common_resume();
1057
1058 fail:
1059         return err;
1060 }
1061
1062 /*
1063  * Function pointers to optional board specific function
1064  */
1065 void (*tegra_deep_sleep)(int);
1066 EXPORT_SYMBOL(tegra_deep_sleep);
1067
1068 static int tegra_suspend_prepare(void)
1069 {
1070         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1071                 tegra_deep_sleep(1);
1072         return 0;
1073 }
1074
1075 static void tegra_suspend_finish(void)
1076 {
1077         if (pdata && pdata->cpu_resume_boost) {
1078                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1079                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1080                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1081         }
1082
1083         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1084                 tegra_deep_sleep(0);
1085 }
1086
1087 static const struct platform_suspend_ops tegra_suspend_ops = {
1088         .valid          = suspend_valid_only_mem,
1089         .prepare        = tegra_suspend_prepare,
1090         .finish         = tegra_suspend_finish,
1091         .prepare_late   = tegra_suspend_prepare_late,
1092         .wake           = tegra_suspend_wake,
1093         .enter          = tegra_suspend_enter,
1094 };
1095
1096 static ssize_t suspend_mode_show(struct kobject *kobj,
1097                                         struct kobj_attribute *attr, char *buf)
1098 {
1099         char *start = buf;
1100         char *end = buf + PAGE_SIZE;
1101
1102         start += scnprintf(start, end - start, "%s ", \
1103                                 tegra_suspend_name[current_suspend_mode]);
1104         start += scnprintf(start, end - start, "\n");
1105
1106         return start - buf;
1107 }
1108
1109 static ssize_t suspend_mode_store(struct kobject *kobj,
1110                                         struct kobj_attribute *attr,
1111                                         const char *buf, size_t n)
1112 {
1113         int len;
1114         const char *name_ptr;
1115         enum tegra_suspend_mode new_mode;
1116
1117         name_ptr = buf;
1118         while (*name_ptr && !isspace(*name_ptr))
1119                 name_ptr++;
1120         len = name_ptr - buf;
1121         if (!len)
1122                 goto bad_name;
1123         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1124         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1125                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1126                 pr_info("Illegal tegra suspend state: %s\n", buf);
1127                 goto bad_name;
1128         }
1129
1130         for (new_mode = TEGRA_SUSPEND_NONE; \
1131                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1132                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1133                         current_suspend_mode = new_mode;
1134                         break;
1135                 }
1136         }
1137
1138 bad_name:
1139         return n;
1140 }
1141
1142 static struct kobj_attribute suspend_mode_attribute =
1143         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1144
1145 static struct kobject *suspend_kobj;
1146
1147 static int tegra_pm_enter_suspend(void)
1148 {
1149         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1150         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1151                 tegra_lp0_cpu_mode(true);
1152         return 0;
1153 }
1154
1155 static void tegra_pm_enter_resume(void)
1156 {
1157         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1158                 tegra_lp0_cpu_mode(false);
1159         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1160 }
1161
1162 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1163         .suspend = tegra_pm_enter_suspend,
1164         .resume = tegra_pm_enter_resume,
1165 };
1166
1167 static __init int tegra_pm_enter_syscore_init(void)
1168 {
1169         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1170         return 0;
1171 }
1172 subsys_initcall(tegra_pm_enter_syscore_init);
1173 #endif
1174
1175 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1176 {
1177         u32 reg;
1178         u32 mode;
1179
1180 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1181         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1182         BUG_ON(IS_ERR(tegra_dfll));
1183 #endif
1184         tegra_pclk = clk_get_sys(NULL, "pclk");
1185         BUG_ON(IS_ERR(tegra_pclk));
1186         pdata = plat;
1187         (void)reg;
1188         (void)mode;
1189
1190         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1191                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1192
1193 #ifndef CONFIG_PM_SLEEP
1194         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1195                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1196                            "disabling suspend\n", __func__);
1197                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1198         }
1199 #else
1200         if (create_suspend_pgtable() < 0) {
1201                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1202                                 __func__);
1203                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1204                 goto fail;
1205         }
1206
1207         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1208             (tegra_revision == TEGRA_REVISION_A01) &&
1209             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1210                 /* Tegra 3 A01 supports only LP1 */
1211                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1212                            "-- disabling LP0\n", __func__);
1213                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1214         }
1215         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1216                 tegra_lp0_vec_relocate) {
1217                 unsigned char *reloc_lp0;
1218                 unsigned long tmp;
1219                 void __iomem *orig;
1220                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1221                                         GFP_KERNEL);
1222                 WARN_ON(!reloc_lp0);
1223                 if (!reloc_lp0) {
1224                         pr_err("%s: Failed to allocate reloc_lp0\n",
1225                                 __func__);
1226                         goto out;
1227                 }
1228
1229                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1230                 WARN_ON(!orig);
1231                 if (!orig) {
1232                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1233                                 __func__, tegra_lp0_vec_start);
1234                         kfree(reloc_lp0);
1235                         goto out;
1236                 }
1237
1238                 tmp = (unsigned long) reloc_lp0;
1239                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1240                 reloc_lp0 = (unsigned char *)tmp;
1241                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1242                 iounmap(orig);
1243                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1244         }
1245
1246 out:
1247         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1248                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1249                            "provided by bootlader -- disabling LP0\n",
1250                            __func__);
1251                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1252         }
1253
1254         iram_save_size = tegra_iram_end() - tegra_iram_start();
1255
1256         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1257         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1258                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1259                        "-- LP0/LP1 unavailable\n", __func__);
1260                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1261         }
1262
1263 #ifdef CONFIG_TEGRA_LP1_950
1264         if (pdata->lp1_lowvolt_support) {
1265                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1266                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1267                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1268
1269                 lp1_core_lowvolt = 0;
1270                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1271                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1272
1273                 lp1_core_highvolt = 0;
1274                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1275                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1276         }
1277 #endif
1278         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1279         /* Initialize scratch registers used for CPU LP2 synchronization */
1280         writel(0, pmc + PMC_SCRATCH37);
1281         writel(0, pmc + PMC_SCRATCH38);
1282 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1283         writel(0, pmc + PMC_SCRATCH39);
1284 #endif
1285         writel(0, pmc + PMC_SCRATCH41);
1286
1287         /* Always enable CPU power request; just normal polarity is supported */
1288         reg = readl(pmc + PMC_CTRL);
1289         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1290         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1291         pmc_32kwritel(reg, PMC_CTRL);
1292
1293         /* Configure core power request and system clock control if LP0
1294            is supported */
1295         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1296         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1297
1298         reg = readl(pmc + PMC_CTRL);
1299
1300         if (!pdata->sysclkreq_high)
1301                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1302         else
1303                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1304
1305         if (!pdata->corereq_high)
1306                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1307         else
1308                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1309
1310         /* configure output inverters while the request is tristated */
1311         pmc_32kwritel(reg, PMC_CTRL);
1312
1313         /* now enable requests */
1314         reg |= TEGRA_POWER_SYSCLK_OE;
1315         if (!pdata->combined_req)
1316                 reg |= TEGRA_POWER_PWRREQ_OE;
1317         pmc_32kwritel(reg, PMC_CTRL);
1318
1319         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1320                 tegra_lp0_suspend_init();
1321
1322         suspend_set_ops(&tegra_suspend_ops);
1323
1324         /* Create /sys/power/suspend/type */
1325         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1326         if (suspend_kobj) {
1327                 if (sysfs_create_file(suspend_kobj, \
1328                                                 &suspend_mode_attribute.attr))
1329                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1330                                                                 __func__);
1331         }
1332
1333         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1334         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1335
1336         /* clear io dpd settings before kernel */
1337         tegra_bl_io_dpd_cleanup();
1338
1339 fail:
1340 #endif
1341         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1342                 tegra_pd_in_idle(false);
1343
1344         current_suspend_mode = plat->suspend_mode;
1345 }
1346
1347 unsigned long debug_uart_port_base = 0;
1348 EXPORT_SYMBOL(debug_uart_port_base);
1349
1350 static int tegra_debug_uart_suspend(void)
1351 {
1352         void __iomem *uart;
1353         u32 lcr;
1354
1355         if (!debug_uart_port_base)
1356                 return 0;
1357
1358         uart = IO_ADDRESS(debug_uart_port_base);
1359
1360         lcr = readb(uart + UART_LCR * 4);
1361
1362         tegra_sctx.uart[0] = lcr;
1363         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1364
1365         /* DLAB = 0 */
1366         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1367
1368         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1369
1370         /* DLAB = 1 */
1371         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1372
1373         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1374         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1375
1376         writeb(lcr, uart + UART_LCR * 4);
1377
1378         return 0;
1379 }
1380
1381 static void tegra_debug_uart_resume(void)
1382 {
1383         void __iomem *uart;
1384         u32 lcr;
1385
1386         if (!debug_uart_port_base)
1387                 return;
1388
1389         uart = IO_ADDRESS(debug_uart_port_base);
1390
1391         lcr = tegra_sctx.uart[0];
1392
1393         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1394
1395         /* DLAB = 0 */
1396         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1397
1398         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1399                         uart + UART_FCR * 4);
1400
1401         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1402
1403         /* DLAB = 1 */
1404         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1405
1406         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1407         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1408
1409         writeb(lcr, uart + UART_LCR * 4);
1410 }
1411
1412 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1413         .suspend = tegra_debug_uart_suspend,
1414         .resume = tegra_debug_uart_resume,
1415 };
1416
1417 struct clk *debug_uart_clk = NULL;
1418 EXPORT_SYMBOL(debug_uart_clk);
1419
1420 void tegra_console_uart_suspend(void)
1421 {
1422         if (console_suspend_enabled && debug_uart_clk)
1423                 tegra_clk_disable_unprepare(debug_uart_clk);
1424 }
1425
1426 void tegra_console_uart_resume(void)
1427 {
1428         if (console_suspend_enabled && debug_uart_clk)
1429                 tegra_clk_prepare_enable(debug_uart_clk);
1430 }
1431
1432 static int tegra_debug_uart_syscore_init(void)
1433 {
1434         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1435         return 0;
1436 }
1437 arch_initcall(tegra_debug_uart_syscore_init);