ARM: tegra11x: residency time for different LP2 mode
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58 #include <asm/smp_plat.h>
59
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62 #include <mach/hardware.h>
63
64 #include "board.h"
65 #include "clock.h"
66 #include "common.h"
67 #include "cpuidle.h"
68 #include "fuse.h"
69 #include "gic.h"
70 #include "iomap.h"
71 #include "pm.h"
72 #include "pm-irq.h"
73 #include "reset.h"
74 #include "sleep.h"
75 #include "timer.h"
76 #include "dvfs.h"
77 #include "cpu-tegra.h"
78
79 struct suspend_context {
80         /*
81          * The next 7 values are referenced by offset in __restart_plls
82          * in headsmp-t2.S, and should not be moved
83          */
84         u32 pllx_misc;
85         u32 pllx_base;
86         u32 pllp_misc;
87         u32 pllp_base;
88         u32 pllp_outa;
89         u32 pllp_outb;
90         u32 pll_timeout;
91
92         u32 cpu_burst;
93         u32 clk_csite_src;
94         u32 cclk_divider;
95
96         u32 mc[3];
97         u8 uart[5];
98
99         struct tegra_twd_context twd;
100 #ifdef CONFIG_ARM_ARCH_TIMER
101         struct arch_timer_context arch_timer;
102 #endif
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
107 static pgd_t *tegra_pgd;
108 static DEFINE_SPINLOCK(tegra_lp2_lock);
109 static cpumask_t tegra_in_lp2;
110 static cpumask_t *iram_cpu_lp2_mask;
111 static unsigned long *iram_cpu_lp1_mask;
112 static u8 *iram_save;
113 static unsigned long iram_save_size;
114 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
115 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
116 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
117 static int tegra_last_pclk;
118 #endif
119
120 struct suspend_context tegra_sctx;
121
122 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
123 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
124 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
125 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
126 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
127 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
128 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
129 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
130 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
131
132 #define PMC_CTRL                0x0
133 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
134 #define PMC_WAKE_MASK           0xc
135 #define PMC_WAKE_LEVEL          0x10
136 #define PMC_DPAD_ORIDE          0x1C
137 #define PMC_WAKE_DELAY          0xe0
138 #define PMC_DPD_SAMPLE          0x20
139 #define PMC_IO_DPD_REQ_0        0x1b8
140 #define PMC_IO_DPD2_REQ_0       0X1C0
141
142 #define PMC_WAKE_STATUS         0x14
143 #define PMC_SW_WAKE_STATUS      0x18
144 #define PMC_COREPWRGOOD_TIMER   0x3c
145 #define PMC_CPUPWRGOOD_TIMER    0xc8
146 #define PMC_CPUPWROFF_TIMER     0xcc
147 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
148
149 #define PMC_PWRGATE_TOGGLE      0x30
150 #define PWRGATE_TOGGLE_START    (1 << 8)
151 #define UN_PWRGATE_CPU          \
152         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
153
154 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
155 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
156 #endif
157
158 #define CLK_RESET_CCLK_BURST    0x20
159 #define CLK_RESET_CCLK_DIVIDER  0x24
160 #define CLK_RESET_PLLC_BASE     0x80
161 #define CLK_RESET_PLLM_BASE     0x90
162 #define CLK_RESET_PLLX_BASE     0xe0
163 #define CLK_RESET_PLLX_MISC     0xe4
164 #define CLK_RESET_PLLP_BASE     0xa0
165 #define CLK_RESET_PLLP_OUTA     0xa4
166 #define CLK_RESET_PLLP_OUTB     0xa8
167 #define CLK_RESET_PLLP_MISC     0xac
168
169 #define CLK_RESET_SOURCE_CSITE  0x1d4
170
171 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
172 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
173 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
174 #define CLK_RESET_CCLK_IDLE_POLICY         1
175 #define CLK_RESET_CCLK_RUN_POLICY          2
176 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
177 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
178
179 #define EMC_MRW_0               0x0e8
180 #define EMC_MRW_DEV_SELECTN     30
181 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
182
183 #define MC_SECURITY_START       0x6c
184 #define MC_SECURITY_SIZE        0x70
185 #define MC_SECURITY_CFG2        0x7c
186
187 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
188 static struct clk *tegra_dfll;
189 #endif
190 static struct clk *tegra_pclk;
191 static const struct tegra_suspend_platform_data *pdata;
192 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
193
194 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
195 enum tegra_cluster_switch_time_id {
196         tegra_cluster_switch_time_id_start = 0,
197         tegra_cluster_switch_time_id_prolog,
198         tegra_cluster_switch_time_id_switch,
199         tegra_cluster_switch_time_id_epilog,
200         tegra_cluster_switch_time_id_max
201 };
202
203 static unsigned long
204                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
205 #define tegra_cluster_switch_time(flags, id) \
206         do { \
207                 barrier(); \
208                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
209                         void __iomem *timer_us = \
210                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
211                         if (id < tegra_cluster_switch_time_id_max) \
212                                 tegra_cluster_switch_times[id] = \
213                                                         readl(timer_us); \
214                                 wmb(); \
215                 } \
216                 barrier(); \
217         } while(0)
218 #else
219 #define tegra_cluster_switch_time(flags, id) do {} while(0)
220 #endif
221
222 #ifdef CONFIG_PM_SLEEP
223 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
224         [TEGRA_SUSPEND_NONE]    = "none",
225         [TEGRA_SUSPEND_LP2]     = "lp2",
226         [TEGRA_SUSPEND_LP1]     = "lp1",
227         [TEGRA_SUSPEND_LP0]     = "lp0",
228 };
229
230 unsigned long tegra_cpu_power_good_time(void)
231 {
232         if (WARN_ON_ONCE(!pdata))
233                 return 5000;
234
235         return pdata->cpu_timer;
236 }
237
238 unsigned long tegra_cpu_power_off_time(void)
239 {
240         if (WARN_ON_ONCE(!pdata))
241                 return 5000;
242
243         return pdata->cpu_off_timer;
244 }
245
246 unsigned long tegra_cpu_lp2_min_residency(void)
247 {
248         if (WARN_ON_ONCE(!pdata))
249                 return 2000;
250
251         return pdata->cpu_lp2_min_residency;
252 }
253
254 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
255 unsigned long tegra_min_residency_noncpu(void)
256 {
257         return pdata->min_residency_noncpu;
258 }
259
260 unsigned long tegra_min_residency_crail(void)
261 {
262         return pdata->min_residency_crail;
263 }
264 #endif
265
266 static void suspend_cpu_dfll_mode(void)
267 {
268 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
269         /* If DFLL is used as CPU clock source go to open loop mode */
270         if (!is_lp_cluster() && tegra_dfll &&
271             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
272                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
273 #endif
274 }
275
276 static void resume_cpu_dfll_mode(void)
277 {
278 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
279         /* If DFLL is used as CPU clock source restore closed loop mode */
280         if (!is_lp_cluster() && tegra_dfll &&
281             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
282                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
283 #endif
284 }
285
286 /*
287  * create_suspend_pgtable
288  *
289  * Creates a page table with identity mappings of physical memory and IRAM
290  * for use when the MMU is off, in addition to all the regular kernel mappings.
291  */
292 static __init int create_suspend_pgtable(void)
293 {
294         tegra_pgd = pgd_alloc(&init_mm);
295         if (!tegra_pgd)
296                 return -ENOMEM;
297
298         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
299         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
300                 high_memory, 0);
301         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
302                 IO_IRAM_VIRT + SECTION_SIZE, 0);
303
304         /* inner/outer write-back/write-allocate, sharable */
305         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
306
307         return 0;
308 }
309
310 /* ensures that sufficient time is passed for a register write to
311  * serialize into the 32KHz domain */
312 static void pmc_32kwritel(u32 val, unsigned long offs)
313 {
314         writel(val, pmc + offs);
315         udelay(130);
316 }
317
318 static void set_power_timers(unsigned long us_on, unsigned long us_off,
319                              long rate)
320 {
321         static unsigned long last_us_off = 0;
322         unsigned long long ticks;
323         unsigned long long pclk;
324
325         if (WARN_ON_ONCE(rate <= 0))
326                 pclk = 100000000;
327         else
328                 pclk = rate;
329
330         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
331                 ticks = (us_on * pclk) + 999999ull;
332                 do_div(ticks, 1000000);
333                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
334
335                 ticks = (us_off * pclk) + 999999ull;
336                 do_div(ticks, 1000000);
337                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
338                 wmb();
339         }
340         tegra_last_pclk = pclk;
341         last_us_off = us_off;
342 }
343
344 /*
345  * restore_cpu_complex
346  *
347  * restores cpu clock setting, clears flow controller
348  *
349  * Always called on CPU 0.
350  */
351 static void restore_cpu_complex(u32 mode)
352 {
353         int cpu = smp_processor_id();
354         unsigned int reg;
355 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
356         unsigned int policy;
357 #endif
358
359         BUG_ON(cpu != 0);
360
361 #ifdef CONFIG_SMP
362         cpu = cpu_logical_map(cpu);
363 #endif
364
365 /*
366  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
367  * or restored by common clock suspend/resume procedures. Hence, we don't
368  * need it here.
369  */
370 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
371         /* Is CPU complex already running on PLLX? */
372         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
373         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
374
375         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
376                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
377         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
378                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
379         else
380                 BUG();
381
382         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
383                 /* restore PLLX settings if CPU is on different PLL */
384                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
385                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
386
387                 /* wait for PLL stabilization if PLLX was enabled */
388                 if (tegra_sctx.pllx_base & (1<<30)) {
389 #if USE_PLL_LOCK_BITS
390                         /* Enable lock detector */
391                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
392                         reg |= 1<<18;
393                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
394                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
395                                  (1<<27)))
396                                 cpu_relax();
397
398                         udelay(PLL_POST_LOCK_DELAY);
399 #else
400                         udelay(300);
401 #endif
402                 }
403         }
404
405         /* Restore original burst policy setting for calls resulting from CPU
406            LP2 in idle or system suspend; keep cluster switch prolog setting
407            intact. */
408         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
409                 writel(tegra_sctx.cclk_divider, clk_rst +
410                        CLK_RESET_CCLK_DIVIDER);
411                 writel(tegra_sctx.cpu_burst, clk_rst +
412                        CLK_RESET_CCLK_BURST);
413         }
414 #endif
415         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
416
417         /* Do not power-gate CPU 0 when flow controlled */
418         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
419         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
420         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
421         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
422         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
423         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
424         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
425
426         /* If an immedidate cluster switch is being perfomed, restore the
427            local timer registers. For calls resulting from CPU LP2 in
428            idle or system suspend, the local timer was shut down and
429            timekeeping switched over to the global system timer. In this
430            case keep local timer disabled, and restore only periodic load. */
431         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
432                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
433 #ifdef CONFIG_ARM_ARCH_TIMER
434                 tegra_sctx.arch_timer.cntp_ctl = 0;
435 #endif
436 #ifdef CONFIG_HAVE_ARM_TWD
437                 tegra_sctx.twd.twd_ctrl = 0;
438 #endif
439         }
440 #ifdef CONFIG_ARM_ARCH_TIMER
441         arch_timer_resume(&tegra_sctx.arch_timer);
442 #endif
443 #ifdef CONFIG_HAVE_ARM_TWD
444         tegra_twd_resume(&tegra_sctx.twd);
445 #endif
446 }
447
448 /*
449  * suspend_cpu_complex
450  *
451  * saves pll state for use by restart_plls, prepares flow controller for
452  * transition to suspend state
453  *
454  * Must always be called on cpu 0.
455  */
456 static void suspend_cpu_complex(u32 mode)
457 {
458         int cpu = smp_processor_id();
459         unsigned int reg;
460         int i;
461
462         BUG_ON(cpu != 0);
463
464 #ifdef CONFIG_SMP
465         cpu = cpu_logical_map(cpu);
466 #endif
467         /* switch coresite to clk_m, save off original source */
468         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
469         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
470
471         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
472         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
473         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
474         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
475         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
476         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
477         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
478         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
479
480 #ifdef CONFIG_HAVE_ARM_TWD
481         tegra_twd_suspend(&tegra_sctx.twd);
482 #endif
483 #ifdef CONFIG_ARM_ARCH_TIMER
484         arch_timer_suspend(&tegra_sctx.arch_timer);
485 #endif
486
487         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
488         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
489         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
490         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
491         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
492 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
493         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
494 #else
495         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
496 #endif
497         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
498         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
499
500         for (i = 0; i < num_possible_cpus(); i++) {
501                 if (i == cpu)
502                         continue;
503                 reg = readl(FLOW_CTRL_CPU_CSR(i));
504                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
505                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
506                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
507         }
508
509         tegra_gic_cpu_disable(true);
510 }
511
512 void tegra_clear_cpu_in_lp2(int cpu)
513 {
514         spin_lock(&tegra_lp2_lock);
515         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
516         cpumask_clear_cpu(cpu, &tegra_in_lp2);
517
518         /* Update the IRAM copy used by the reset handler. The IRAM copy
519            can't use used directly by cpumask_clear_cpu() because it uses
520            LDREX/STREX which requires the addressed location to be inner
521            cacheable and sharable which IRAM isn't. */
522         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
523         dsb();
524
525         spin_unlock(&tegra_lp2_lock);
526 }
527
528 bool tegra_set_cpu_in_lp2(int cpu)
529 {
530         bool last_cpu = false;
531
532         spin_lock(&tegra_lp2_lock);
533         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
534         cpumask_set_cpu(cpu, &tegra_in_lp2);
535
536         /* Update the IRAM copy used by the reset handler. The IRAM copy
537            can't use used directly by cpumask_set_cpu() because it uses
538            LDREX/STREX which requires the addressed location to be inner
539            cacheable and sharable which IRAM isn't. */
540         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
541         dsb();
542
543         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
544                 last_cpu = true;
545 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
546         else if (cpu == 1)
547                 tegra2_cpu_set_resettable_soon();
548 #endif
549
550         spin_unlock(&tegra_lp2_lock);
551         return last_cpu;
552 }
553
554 static void tegra_sleep_core(enum tegra_suspend_mode mode,
555                              unsigned long v2p)
556 {
557 #ifdef CONFIG_TRUSTED_FOUNDATIONS
558         if (mode == TEGRA_SUSPEND_LP0) {
559                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
560                                   virt_to_phys(tegra_resume));
561         } else {
562                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
563                                   (TEGRA_RESET_HANDLER_BASE +
564                                    tegra_cpu_reset_handler_offset));
565         }
566 #endif
567 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
568         cpu_suspend(v2p, tegra2_sleep_core_finish);
569 #else
570         cpu_suspend(v2p, tegra3_sleep_core_finish);
571 #endif
572 }
573
574 static inline void tegra_sleep_cpu(unsigned long v2p)
575 {
576 #ifdef CONFIG_TRUSTED_FOUNDATIONS
577         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
578                           (TEGRA_RESET_HANDLER_BASE +
579                            tegra_cpu_reset_handler_offset));
580 #endif
581         cpu_suspend(v2p, tegra_sleep_cpu_finish);
582 }
583
584 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
585 {
586         u32 reg;
587         unsigned int remain;
588 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
589         pgd_t *pgd;
590 #endif
591
592         /* Only the last cpu down does the final suspend steps */
593         reg = readl(pmc + PMC_CTRL);
594         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
595         if (pdata->combined_req)
596                 reg &= ~TEGRA_POWER_PWRREQ_OE;
597         else
598                 reg |= TEGRA_POWER_PWRREQ_OE;
599
600         reg &= ~TEGRA_POWER_EFFECT_LP0;
601         writel(reg, pmc + PMC_CTRL);
602
603         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
604
605         /*
606          * We can use clk_get_rate_all_locked() here, because all other cpus
607          * are in LP2 state and irqs are disabled
608          */
609         if (flags & TEGRA_POWER_CLUSTER_MASK) {
610                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
611                 set_power_timers(pdata->cpu_timer, 2,
612                         clk_get_rate_all_locked(tegra_pclk));
613                 if (flags & TEGRA_POWER_CLUSTER_G) {
614                         /*
615                          * To reduce the vdd_cpu up latency when LP->G
616                          * transition. Before the transition, enable
617                          * the vdd_cpu rail.
618                          */
619                         if (is_lp_cluster()) {
620 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
621                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
622                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
623                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
624 #else
625                                 writel(UN_PWRGATE_CPU,
626                                        pmc + PMC_PWRGATE_TOGGLE);
627 #endif
628                         }
629                 }
630                 tegra_cluster_switch_prolog(flags);
631         } else {
632                 suspend_cpu_dfll_mode();
633                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
634                         clk_get_rate_all_locked(tegra_pclk));
635 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
636                 reg = readl(FLOW_CTRL_CPU_CSR(0));
637                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
638                 if (is_lp_cluster()) {
639                         /* for LP cluster, there is no option for rail gating */
640                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
641                                                 TEGRA_POWER_CLUSTER_PART_MASK)
642                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
643                         else if (flags)
644                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
645                 }
646                 else {
647                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
648                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
649                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
650                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
651                 }
652                 writel(reg, FLOW_CTRL_CPU_CSR(0));
653 #endif
654         }
655
656         if (sleep_time)
657                 tegra_lp2_set_trigger(sleep_time);
658
659         cpu_cluster_pm_enter();
660         suspend_cpu_complex(flags);
661         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
662 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
663         flush_cache_all();
664         /*
665          * No need to flush complete L2. Cleaning kernel and IO mappings
666          * is enough for the LP code sequence that has L2 disabled but
667          * MMU on.
668          */
669         pgd = cpu_get_pgd();
670         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
671                           __pa(pgd + PTRS_PER_PGD));
672         outer_disable();
673 #endif
674         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
675
676         tegra_init_cache(false);
677         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
678         restore_cpu_complex(flags);
679         cpu_cluster_pm_exit();
680
681         remain = tegra_lp2_timer_remain();
682         if (sleep_time)
683                 tegra_lp2_set_trigger(0);
684
685         if (flags & TEGRA_POWER_CLUSTER_MASK) {
686                 tegra_cluster_switch_epilog(flags);
687                 if (is_idle_task(current))
688                         trace_cpu_cluster_rcuidle(POWER_CPU_CLUSTER_DONE);
689                 else
690                         trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
691         } else {
692                 resume_cpu_dfll_mode();
693         }
694         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
695
696 #if INSTRUMENT_CLUSTER_SWITCH
697         if (flags & TEGRA_POWER_CLUSTER_MASK) {
698                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
699                         is_lp_cluster() ? "G=>LP" : "LP=>G",
700                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
701                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
702                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
703                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
704                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
705                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
706                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
707                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
708         }
709 #endif
710         return remain;
711 }
712
713 static int tegra_common_suspend(void)
714 {
715         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
716
717         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
718         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
719         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
720
721         /* copy the reset vector and SDRAM shutdown code into IRAM */
722         memcpy(iram_save, iram_code, iram_save_size);
723         memcpy(iram_code, tegra_iram_start(), iram_save_size);
724
725         return 0;
726 }
727
728 static void tegra_common_resume(void)
729 {
730         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
731 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
732         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
733 #endif
734
735         /* Clear DPD sample */
736         writel(0x0, pmc + PMC_DPD_SAMPLE);
737
738         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
739         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
740         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
741 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
742         /* trigger emc mode write */
743         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
744 #endif
745         /* clear scratch registers shared by suspend and the reset pen */
746         writel(0x0, pmc + PMC_SCRATCH39);
747         writel(0x0, pmc + PMC_SCRATCH41);
748
749         /* restore IRAM */
750         memcpy(iram_code, iram_save, iram_save_size);
751 }
752
753 static int tegra_suspend_prepare_late(void)
754 {
755 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
756         disable_irq(INT_SYS_STATS_MON);
757 #endif
758         return 0;
759 }
760
761 static void tegra_suspend_wake(void)
762 {
763 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
764         enable_irq(INT_SYS_STATS_MON);
765 #endif
766 }
767
768 static void tegra_pm_set(enum tegra_suspend_mode mode)
769 {
770         u32 reg, boot_flag;
771         unsigned long rate = 32768;
772
773         reg = readl(pmc + PMC_CTRL);
774         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
775         if (pdata->combined_req)
776                 reg &= ~TEGRA_POWER_PWRREQ_OE;
777         else
778                 reg |= TEGRA_POWER_PWRREQ_OE;
779         reg &= ~TEGRA_POWER_EFFECT_LP0;
780
781         switch (mode) {
782         case TEGRA_SUSPEND_LP0:
783 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
784                 rate = clk_get_rate_all_locked(tegra_pclk);
785 #endif
786                 if (pdata->combined_req) {
787                         reg |= TEGRA_POWER_PWRREQ_OE;
788                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
789                 }
790
791                 /*
792                  * LP0 boots through the AVP, which then resumes the AVP to
793                  * the address in scratch 39, and the cpu to the address in
794                  * scratch 41 to tegra_resume
795                  */
796                 writel(0x0, pmc + PMC_SCRATCH39);
797
798                 /* Enable DPD sample to trigger sampling pads data and direction
799                  * in which pad will be driven during lp0 mode*/
800                 writel(0x1, pmc + PMC_DPD_SAMPLE);
801 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
802                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ_0);
803                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ_0);
804 #endif
805 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
806                 /* this is needed only for T11x, not for other chips */
807                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
808 #endif
809
810                 /* Set warmboot flag */
811                 boot_flag = readl(pmc + PMC_SCRATCH0);
812                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
813
814                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
815
816                 reg |= TEGRA_POWER_EFFECT_LP0;
817                 /* No break here. LP0 code falls through to write SCRATCH41 */
818         case TEGRA_SUSPEND_LP1:
819                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
820                 wmb();
821                 break;
822         case TEGRA_SUSPEND_LP2:
823                 rate = clk_get_rate(tegra_pclk);
824                 break;
825         case TEGRA_SUSPEND_NONE:
826                 return;
827         default:
828                 BUG();
829         }
830
831         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
832
833         pmc_32kwritel(reg, PMC_CTRL);
834 }
835
836 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
837         [TEGRA_SUSPEND_NONE] = "none",
838         [TEGRA_SUSPEND_LP2] = "LP2",
839         [TEGRA_SUSPEND_LP1] = "LP1",
840         [TEGRA_SUSPEND_LP0] = "LP0",
841 };
842
843 static int tegra_suspend_enter(suspend_state_t state)
844 {
845         int ret;
846         ktime_t delta;
847         struct timespec ts_entry, ts_exit;
848
849         if (pdata && pdata->board_suspend)
850                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
851
852         read_persistent_clock(&ts_entry);
853
854         ret = tegra_suspend_dram(current_suspend_mode, 0);
855         if (ret) {
856                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
857                 goto abort_suspend;
858         }
859
860         read_persistent_clock(&ts_exit);
861
862         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
863                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
864
865                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
866                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
867                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
868                 else
869                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
870         }
871
872 abort_suspend:
873         if (pdata && pdata->board_resume)
874                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
875
876         return ret;
877 }
878
879 static void tegra_suspend_check_pwr_stats(void)
880 {
881         /* cpus and l2 are powered off later */
882         unsigned long pwrgate_partid_mask =
883 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
884                 (1 << TEGRA_POWERGATE_HEG)      |
885                 (1 << TEGRA_POWERGATE_SATA)     |
886                 (1 << TEGRA_POWERGATE_3D1)      |
887 #endif
888                 (1 << TEGRA_POWERGATE_3D)       |
889                 (1 << TEGRA_POWERGATE_VENC)     |
890                 (1 << TEGRA_POWERGATE_PCIE)     |
891                 (1 << TEGRA_POWERGATE_VDEC)     |
892                 (1 << TEGRA_POWERGATE_MPE);
893
894         int partid;
895
896         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
897                 if ((1 << partid) & pwrgate_partid_mask)
898                         if (tegra_powergate_is_powered(partid))
899                                 pr_warning("partition %s is left on before suspend\n",
900                                         tegra_powergate_get_name(partid));
901
902         return;
903 }
904
905 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
906 {
907         int err = 0;
908         u32 scratch37 = 0xDEADBEEF;
909         u32 reg;
910
911         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
912                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
913                 err = -ENXIO;
914                 goto fail;
915         }
916
917         if (tegra_is_voice_call_active()) {
918                 /* backup the current value of scratch37 */
919                 scratch37 = readl(pmc + PMC_SCRATCH37);
920
921                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
922                 reg = TEGRA_POWER_LP1_AUDIO;
923                 pmc_32kwritel(reg, PMC_SCRATCH37);
924         }
925
926         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
927                 pr_info("LP0 not used due to unsupported wakeup events\n");
928                 mode = TEGRA_SUSPEND_LP1;
929         }
930
931         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
932                 tegra_suspend_check_pwr_stats();
933
934         tegra_common_suspend();
935
936         tegra_pm_set(mode);
937
938         if (pdata && pdata->board_suspend)
939                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
940
941         local_fiq_disable();
942
943         trace_cpu_suspend(CPU_SUSPEND_START);
944
945         if (mode == TEGRA_SUSPEND_LP0) {
946 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
947                 reg = readl(pmc + PMC_SCRATCH4);
948                 if (is_lp_cluster())
949                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
950                 else
951                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
952                 pmc_32kwritel(reg, PMC_SCRATCH4);
953 #endif
954                 tegra_tsc_suspend();
955                 tegra_lp0_suspend_mc();
956                 tegra_cpu_reset_handler_save();
957                 tegra_tsc_wait_for_suspend();
958                 tegra_smp_clear_power_mask();
959         }
960         else if (mode == TEGRA_SUSPEND_LP1)
961                 *iram_cpu_lp1_mask = 1;
962
963         suspend_cpu_complex(flags);
964
965 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
966         /* In case of LP0, program external power gating accordinly */
967         if (mode == TEGRA_SUSPEND_LP0) {
968                 reg = readl(FLOW_CTRL_CPU_CSR(0));
969                 if (is_lp_cluster())
970                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
971                 else
972                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
973                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
974         }
975 #endif
976
977         flush_cache_all();
978         outer_flush_all();
979         outer_disable();
980
981         if (mode == TEGRA_SUSPEND_LP2)
982                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
983         else
984                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
985
986         tegra_init_cache(true);
987
988         if (mode == TEGRA_SUSPEND_LP0) {
989 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
990                 reg = readl(pmc+PMC_CTRL);
991                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
992                 pmc_32kwritel(reg, PMC_CTRL);
993 #endif
994                 tegra_tsc_resume();
995                 tegra_cpu_reset_handler_restore();
996                 tegra_lp0_resume_mc();
997                 tegra_tsc_wait_for_resume();
998         } else if (mode == TEGRA_SUSPEND_LP1)
999                 *iram_cpu_lp1_mask = 0;
1000
1001         /* if scratch37 was clobbered during LP1, restore it */
1002         if (scratch37 != 0xDEADBEEF)
1003                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1004
1005         restore_cpu_complex(flags);
1006
1007         /* for platforms where the core & CPU power requests are
1008          * combined as a single request to the PMU, transition out
1009          * of LP0 state by temporarily enabling both requests
1010          */
1011         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1012                 reg = readl(pmc + PMC_CTRL);
1013                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1014                 pmc_32kwritel(reg, PMC_CTRL);
1015                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1016                 pmc_32kwritel(reg, PMC_CTRL);
1017         }
1018
1019         if (pdata && pdata->board_resume)
1020                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1021
1022         trace_cpu_suspend(CPU_SUSPEND_DONE);
1023
1024         local_fiq_enable();
1025
1026         tegra_common_resume();
1027
1028 fail:
1029         return err;
1030 }
1031
1032 /*
1033  * Function pointers to optional board specific function
1034  */
1035 void (*tegra_deep_sleep)(int);
1036 EXPORT_SYMBOL(tegra_deep_sleep);
1037
1038 static int tegra_suspend_prepare(void)
1039 {
1040         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1041                 tegra_deep_sleep(1);
1042         return 0;
1043 }
1044
1045 static void tegra_suspend_finish(void)
1046 {
1047         if (pdata && pdata->cpu_resume_boost) {
1048                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1049                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1050                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1051         }
1052
1053         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1054                 tegra_deep_sleep(0);
1055 }
1056
1057 static const struct platform_suspend_ops tegra_suspend_ops = {
1058         .valid          = suspend_valid_only_mem,
1059         .prepare        = tegra_suspend_prepare,
1060         .finish         = tegra_suspend_finish,
1061         .prepare_late   = tegra_suspend_prepare_late,
1062         .wake           = tegra_suspend_wake,
1063         .enter          = tegra_suspend_enter,
1064 };
1065
1066 static ssize_t suspend_mode_show(struct kobject *kobj,
1067                                         struct kobj_attribute *attr, char *buf)
1068 {
1069         char *start = buf;
1070         char *end = buf + PAGE_SIZE;
1071
1072         start += scnprintf(start, end - start, "%s ", \
1073                                 tegra_suspend_name[current_suspend_mode]);
1074         start += scnprintf(start, end - start, "\n");
1075
1076         return start - buf;
1077 }
1078
1079 static ssize_t suspend_mode_store(struct kobject *kobj,
1080                                         struct kobj_attribute *attr,
1081                                         const char *buf, size_t n)
1082 {
1083         int len;
1084         const char *name_ptr;
1085         enum tegra_suspend_mode new_mode;
1086
1087         name_ptr = buf;
1088         while (*name_ptr && !isspace(*name_ptr))
1089                 name_ptr++;
1090         len = name_ptr - buf;
1091         if (!len)
1092                 goto bad_name;
1093         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1094         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1095                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1096                 pr_info("Illegal tegra suspend state: %s\n", buf);
1097                 goto bad_name;
1098         }
1099
1100         for (new_mode = TEGRA_SUSPEND_NONE; \
1101                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1102                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1103                         current_suspend_mode = new_mode;
1104                         break;
1105                 }
1106         }
1107
1108 bad_name:
1109         return n;
1110 }
1111
1112 static struct kobj_attribute suspend_mode_attribute =
1113         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1114
1115 static struct kobject *suspend_kobj;
1116
1117 static int tegra_pm_enter_suspend(void)
1118 {
1119         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1120         suspend_cpu_dfll_mode();
1121         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1122                 tegra_lp0_cpu_mode(true);
1123         return 0;
1124 }
1125
1126 static void tegra_pm_enter_resume(void)
1127 {
1128         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1129                 tegra_lp0_cpu_mode(false);
1130         resume_cpu_dfll_mode();
1131         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1132 }
1133
1134 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1135         .suspend = tegra_pm_enter_suspend,
1136         .resume = tegra_pm_enter_resume,
1137 };
1138
1139 static __init int tegra_pm_enter_syscore_init(void)
1140 {
1141         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1142         return 0;
1143 }
1144 subsys_initcall(tegra_pm_enter_syscore_init);
1145 #endif
1146
1147 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1148 {
1149         u32 reg;
1150         u32 mode;
1151
1152 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1153         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1154         BUG_ON(IS_ERR(tegra_dfll));
1155 #endif
1156         tegra_pclk = clk_get_sys(NULL, "pclk");
1157         BUG_ON(IS_ERR(tegra_pclk));
1158         pdata = plat;
1159         (void)reg;
1160         (void)mode;
1161
1162         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1163                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1164
1165 #ifndef CONFIG_PM_SLEEP
1166         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1167                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1168                            "disabling suspend\n", __func__);
1169                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1170         }
1171 #else
1172         if (create_suspend_pgtable() < 0) {
1173                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1174                                 __func__);
1175                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1176                 goto fail;
1177         }
1178
1179         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1180             (tegra_revision == TEGRA_REVISION_A01) &&
1181             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1182                 /* Tegra 3 A01 supports only LP1 */
1183                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1184                            "-- disabling LP0\n", __func__);
1185                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1186         }
1187         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1188                 tegra_lp0_vec_relocate) {
1189                 unsigned char *reloc_lp0;
1190                 unsigned long tmp;
1191                 void __iomem *orig;
1192                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1193                                         GFP_KERNEL);
1194                 WARN_ON(!reloc_lp0);
1195                 if (!reloc_lp0) {
1196                         pr_err("%s: Failed to allocate reloc_lp0\n",
1197                                 __func__);
1198                         goto out;
1199                 }
1200
1201                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1202                 WARN_ON(!orig);
1203                 if (!orig) {
1204                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1205                                 __func__, tegra_lp0_vec_start);
1206                         kfree(reloc_lp0);
1207                         goto out;
1208                 }
1209
1210                 tmp = (unsigned long) reloc_lp0;
1211                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1212                 reloc_lp0 = (unsigned char *)tmp;
1213                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1214                 iounmap(orig);
1215                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1216         }
1217
1218 out:
1219         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1220                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1221                            "provided by bootlader -- disabling LP0\n",
1222                            __func__);
1223                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1224         }
1225
1226         iram_save_size = tegra_iram_end() - tegra_iram_start();
1227
1228         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1229         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1230                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1231                        "-- LP0/LP1 unavailable\n", __func__);
1232                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1233         }
1234
1235 #ifdef CONFIG_TEGRA_LP1_950
1236         if (pdata->lp1_lowvolt_support) {
1237                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1238                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1239                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1240
1241                 lp1_core_lowvolt = 0;
1242                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1243                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1244
1245                 lp1_core_highvolt = 0;
1246                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1247                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1248         }
1249 #endif
1250         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1251         /* Initialize scratch registers used for CPU LP2 synchronization */
1252         writel(0, pmc + PMC_SCRATCH37);
1253         writel(0, pmc + PMC_SCRATCH38);
1254         writel(0, pmc + PMC_SCRATCH39);
1255         writel(0, pmc + PMC_SCRATCH41);
1256
1257         /* Always enable CPU power request; just normal polarity is supported */
1258         reg = readl(pmc + PMC_CTRL);
1259         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1260         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1261         pmc_32kwritel(reg, PMC_CTRL);
1262
1263         /* Configure core power request and system clock control if LP0
1264            is supported */
1265         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1266         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1267
1268         reg = readl(pmc + PMC_CTRL);
1269
1270         if (!pdata->sysclkreq_high)
1271                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1272         else
1273                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1274
1275         if (!pdata->corereq_high)
1276                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1277         else
1278                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1279
1280         /* configure output inverters while the request is tristated */
1281         pmc_32kwritel(reg, PMC_CTRL);
1282
1283         /* now enable requests */
1284         reg |= TEGRA_POWER_SYSCLK_OE;
1285         if (!pdata->combined_req)
1286                 reg |= TEGRA_POWER_PWRREQ_OE;
1287         pmc_32kwritel(reg, PMC_CTRL);
1288
1289         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1290                 tegra_lp0_suspend_init();
1291
1292         suspend_set_ops(&tegra_suspend_ops);
1293
1294         /* Create /sys/power/suspend/type */
1295         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1296         if (suspend_kobj) {
1297                 if (sysfs_create_file(suspend_kobj, \
1298                                                 &suspend_mode_attribute.attr))
1299                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1300                                                                 __func__);
1301         }
1302
1303         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1304         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1305
1306         /* clear io dpd settings before kernel */
1307         tegra_bl_io_dpd_cleanup();
1308
1309 fail:
1310 #endif
1311         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1312                 tegra_lp2_in_idle(false);
1313
1314         current_suspend_mode = plat->suspend_mode;
1315 }
1316
1317 unsigned long debug_uart_port_base = 0;
1318 EXPORT_SYMBOL(debug_uart_port_base);
1319
1320 static int tegra_debug_uart_suspend(void)
1321 {
1322         void __iomem *uart;
1323         u32 lcr;
1324
1325         if (!debug_uart_port_base)
1326                 return 0;
1327
1328         uart = IO_ADDRESS(debug_uart_port_base);
1329
1330         lcr = readb(uart + UART_LCR * 4);
1331
1332         tegra_sctx.uart[0] = lcr;
1333         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1334
1335         /* DLAB = 0 */
1336         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1337
1338         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1339
1340         /* DLAB = 1 */
1341         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1342
1343         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1344         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1345
1346         writeb(lcr, uart + UART_LCR * 4);
1347
1348         return 0;
1349 }
1350
1351 static void tegra_debug_uart_resume(void)
1352 {
1353         void __iomem *uart;
1354         u32 lcr;
1355
1356         if (!debug_uart_port_base)
1357                 return;
1358
1359         uart = IO_ADDRESS(debug_uart_port_base);
1360
1361         lcr = tegra_sctx.uart[0];
1362
1363         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1364
1365         /* DLAB = 0 */
1366         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1367
1368         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1369                         uart + UART_FCR * 4);
1370
1371         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1372
1373         /* DLAB = 1 */
1374         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1375
1376         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1377         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1378
1379         writeb(lcr, uart + UART_LCR * 4);
1380 }
1381
1382 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1383         .suspend = tegra_debug_uart_suspend,
1384         .resume = tegra_debug_uart_resume,
1385 };
1386
1387 struct clk *debug_uart_clk = NULL;
1388 EXPORT_SYMBOL(debug_uart_clk);
1389
1390 void tegra_console_uart_suspend(void)
1391 {
1392         if (console_suspend_enabled && debug_uart_clk)
1393                 clk_disable(debug_uart_clk);
1394 }
1395
1396 void tegra_console_uart_resume(void)
1397 {
1398         if (console_suspend_enabled && debug_uart_clk)
1399                 clk_enable(debug_uart_clk);
1400 }
1401
1402 static int tegra_debug_uart_syscore_init(void)
1403 {
1404         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1405         return 0;
1406 }
1407 arch_initcall(tegra_debug_uart_syscore_init);