arm: tegra: cpu: changing cpu min. freq to 51MHz
[linux-2.6.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/earlysuspend.h>
36 #include <linux/suspend.h>
37 #include <linux/slab.h>
38 #include <linux/serial_reg.h>
39 #include <linux/seq_file.h>
40 #include <linux/uaccess.h>
41 #include <linux/syscore_ops.h>
42 #include <linux/cpu_pm.h>
43 #include <linux/vmalloc.h>
44 #include <linux/memblock.h>
45 #include <linux/console.h>
46 #include <linux/pm_qos.h>
47 #include <linux/export.h>
48 #include <linux/tegra_audio.h>
49
50 #include <trace/events/power.h>
51
52 #include <asm/cacheflush.h>
53 #include <asm/hardware/gic.h>
54 #include <asm/idmap.h>
55 #include <asm/localtimer.h>
56 #include <asm/pgalloc.h>
57 #include <asm/pgtable.h>
58 #include <asm/tlbflush.h>
59 #include <asm/suspend.h>
60 #include <asm/smp_plat.h>
61
62 #include <mach/clk.h>
63 #include <mach/iomap.h>
64 #include <mach/irqs.h>
65 #include <mach/powergate.h>
66 #include <mach/hardware.h>
67
68 #include "board.h"
69 #include "clock.h"
70 #include "common.h"
71 #include "cpuidle.h"
72 #include "fuse.h"
73 #include "gic.h"
74 #include "pm.h"
75 #include "pm-irq.h"
76 #include "reset.h"
77 #include "sleep.h"
78 #include "timer.h"
79 #include "dvfs.h"
80 #include "cpu-tegra.h"
81
82 struct suspend_context {
83         /*
84          * The next 7 values are referenced by offset in __restart_plls
85          * in headsmp-t2.S, and should not be moved
86          */
87         u32 pllx_misc;
88         u32 pllx_base;
89         u32 pllp_misc;
90         u32 pllp_base;
91         u32 pllp_outa;
92         u32 pllp_outb;
93         u32 pll_timeout;
94
95         u32 cpu_burst;
96         u32 clk_csite_src;
97         u32 cclk_divider;
98
99         u32 mc[3];
100         u8 uart[5];
101
102         struct tegra_twd_context twd;
103 };
104
105 #ifdef CONFIG_PM_SLEEP
106 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
107 static pgd_t *tegra_pgd;
108 static DEFINE_SPINLOCK(tegra_lp2_lock);
109 static cpumask_t tegra_in_lp2;
110 static cpumask_t *iram_cpu_lp2_mask;
111 static unsigned long *iram_cpu_lp1_mask;
112 static u8 *iram_save;
113 static unsigned long iram_save_size;
114 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
115 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
116 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
117 static int tegra_last_pclk;
118 #endif
119
120 struct suspend_context tegra_sctx;
121
122 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
123 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
124 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
125 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
126 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
127 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
128 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
129 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
130
131 #define PMC_CTRL                0x0
132 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
133 #define PMC_WAKE_MASK           0xc
134 #define PMC_WAKE_LEVEL          0x10
135 #define PMC_DPAD_ORIDE          0x1C
136 #define PMC_WAKE_DELAY          0xe0
137 #define PMC_DPD_SAMPLE          0x20
138 #define PMC_IO_DPD_REQ_0        0x1b8
139 #define PMC_IO_DPD2_REQ_0       0X1C0
140
141 #define PMC_WAKE_STATUS         0x14
142 #define PMC_SW_WAKE_STATUS      0x18
143 #define PMC_COREPWRGOOD_TIMER   0x3c
144 #define PMC_CPUPWRGOOD_TIMER    0xc8
145 #define PMC_CPUPWROFF_TIMER     0xcc
146 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
147
148 #define PMC_PWRGATE_TOGGLE      0x30
149 #define PWRGATE_TOGGLE_START    (1 << 8)
150 #define UN_PWRGATE_CPU          \
151         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
152
153 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
154 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
155 #endif
156
157 #define CLK_RESET_CCLK_BURST    0x20
158 #define CLK_RESET_CCLK_DIVIDER  0x24
159 #define CLK_RESET_PLLC_BASE     0x80
160 #define CLK_RESET_PLLM_BASE     0x90
161 #define CLK_RESET_PLLX_BASE     0xe0
162 #define CLK_RESET_PLLX_MISC     0xe4
163 #define CLK_RESET_PLLP_BASE     0xa0
164 #define CLK_RESET_PLLP_OUTA     0xa4
165 #define CLK_RESET_PLLP_OUTB     0xa8
166 #define CLK_RESET_PLLP_MISC     0xac
167
168 #define CLK_RESET_SOURCE_CSITE  0x1d4
169
170 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
171 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
172 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
173 #define CLK_RESET_CCLK_IDLE_POLICY         1
174 #define CLK_RESET_CCLK_RUN_POLICY          2
175 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
176 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
177
178 #define EMC_MRW_0               0x0e8
179 #define EMC_MRW_DEV_SELECTN     30
180 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
181
182 #define MC_SECURITY_START       0x6c
183 #define MC_SECURITY_SIZE        0x70
184 #define MC_SECURITY_CFG2        0x7c
185
186 #define AWAKE_CPU_FREQ_MIN      51000
187 static struct pm_qos_request awake_cpu_freq_req;
188
189 struct dvfs_rail *tegra_cpu_rail;
190 static struct dvfs_rail *tegra_core_rail;
191 static struct clk *tegra_pclk;
192 static const struct tegra_suspend_platform_data *pdata;
193 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
194
195 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
196 enum tegra_cluster_switch_time_id {
197         tegra_cluster_switch_time_id_start = 0,
198         tegra_cluster_switch_time_id_prolog,
199         tegra_cluster_switch_time_id_switch,
200         tegra_cluster_switch_time_id_epilog,
201         tegra_cluster_switch_time_id_max
202 };
203
204 static unsigned long
205                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
206 #define tegra_cluster_switch_time(flags, id) \
207         do { \
208                 barrier(); \
209                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
210                         void __iomem *timer_us = \
211                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
212                         if (id < tegra_cluster_switch_time_id_max) \
213                                 tegra_cluster_switch_times[id] = \
214                                                         readl(timer_us); \
215                                 wmb(); \
216                 } \
217                 barrier(); \
218         } while(0)
219 #else
220 #define tegra_cluster_switch_time(flags, id) do {} while(0)
221 #endif
222
223 #ifdef CONFIG_PM_SLEEP
224 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
225         [TEGRA_SUSPEND_NONE]    = "none",
226         [TEGRA_SUSPEND_LP2]     = "lp2",
227         [TEGRA_SUSPEND_LP1]     = "lp1",
228         [TEGRA_SUSPEND_LP0]     = "lp0",
229 };
230
231 unsigned long tegra_cpu_power_good_time(void)
232 {
233         if (WARN_ON_ONCE(!pdata))
234                 return 5000;
235
236         return pdata->cpu_timer;
237 }
238
239 unsigned long tegra_cpu_power_off_time(void)
240 {
241         if (WARN_ON_ONCE(!pdata))
242                 return 5000;
243
244         return pdata->cpu_off_timer;
245 }
246
247 unsigned long tegra_cpu_lp2_min_residency(void)
248 {
249         if (WARN_ON_ONCE(!pdata))
250                 return 2000;
251
252         return pdata->cpu_lp2_min_residency;
253 }
254
255 /*
256  * create_suspend_pgtable
257  *
258  * Creates a page table with identity mappings of physical memory and IRAM
259  * for use when the MMU is off, in addition to all the regular kernel mappings.
260  */
261 static __init int create_suspend_pgtable(void)
262 {
263         tegra_pgd = pgd_alloc(&init_mm);
264         if (!tegra_pgd)
265                 return -ENOMEM;
266
267         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
268         identity_mapping_add(tegra_pgd, PHYS_OFFSET,
269                 PHYS_OFFSET + (unsigned long)high_memory - PAGE_OFFSET);
270         identity_mapping_add(tegra_pgd, IO_IRAM_PHYS,
271                 IO_IRAM_PHYS + SECTION_SIZE);
272
273         /* inner/outer write-back/write-allocate, sharable */
274         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
275
276         return 0;
277 }
278
279 /* ensures that sufficient time is passed for a register write to
280  * serialize into the 32KHz domain */
281 static void pmc_32kwritel(u32 val, unsigned long offs)
282 {
283         writel(val, pmc + offs);
284         udelay(130);
285 }
286
287 static void set_power_timers(unsigned long us_on, unsigned long us_off,
288                              long rate)
289 {
290         static unsigned long last_us_off = 0;
291         unsigned long long ticks;
292         unsigned long long pclk;
293
294         if (WARN_ON_ONCE(rate <= 0))
295                 pclk = 100000000;
296         else
297                 pclk = rate;
298
299         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
300                 ticks = (us_on * pclk) + 999999ull;
301                 do_div(ticks, 1000000);
302                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
303
304                 ticks = (us_off * pclk) + 999999ull;
305                 do_div(ticks, 1000000);
306                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
307                 wmb();
308         }
309         tegra_last_pclk = pclk;
310         last_us_off = us_off;
311 }
312
313 /*
314  * restore_cpu_complex
315  *
316  * restores cpu clock setting, clears flow controller
317  *
318  * Always called on CPU 0.
319  */
320 static void restore_cpu_complex(u32 mode)
321 {
322         int cpu = smp_processor_id();
323         unsigned int reg, policy;
324
325         BUG_ON(cpu != 0);
326
327 #ifdef CONFIG_SMP
328         cpu = cpu_logical_map(cpu);
329 #endif
330         /* Is CPU complex already running on PLLX? */
331         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
332         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
333
334         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
335                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
336         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
337                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
338         else
339                 BUG();
340
341         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
342                 /* restore PLLX settings if CPU is on different PLL */
343                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
344                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
345
346                 /* wait for PLL stabilization if PLLX was enabled */
347                 if (tegra_sctx.pllx_base & (1<<30)) {
348 #if USE_PLL_LOCK_BITS
349                         /* Enable lock detector */
350                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
351                         reg |= 1<<18;
352                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
353                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
354                                  (1<<27)))
355                                 cpu_relax();
356
357                         udelay(PLL_POST_LOCK_DELAY);
358 #else
359                         udelay(300);
360 #endif
361                 }
362         }
363
364         /* Restore original burst policy setting for calls resulting from CPU
365            LP2 in idle or system suspend; keep cluster switch prolog setting
366            intact. */
367         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
368                 writel(tegra_sctx.cclk_divider, clk_rst +
369                        CLK_RESET_CCLK_DIVIDER);
370                 writel(tegra_sctx.cpu_burst, clk_rst +
371                        CLK_RESET_CCLK_BURST);
372         }
373
374         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
375
376         /* Do not power-gate CPU 0 when flow controlled */
377         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
378         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
379         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
380         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
381         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
382         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
383         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
384
385         /* If an immedidate cluster switch is being perfomed, restore the
386            local timer registers. For calls resulting from CPU LP2 in
387            idle or system suspend, the local timer was shut down and
388            timekeeping switched over to the global system timer. In this
389            case keep local timer disabled, and restore only periodic load. */
390         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
391                       TEGRA_POWER_CLUSTER_IMMEDIATE)))
392                 tegra_sctx.twd.twd_ctrl = 0;
393         tegra_twd_resume(&tegra_sctx.twd);
394 }
395
396 /*
397  * suspend_cpu_complex
398  *
399  * saves pll state for use by restart_plls, prepares flow controller for
400  * transition to suspend state
401  *
402  * Must always be called on cpu 0.
403  */
404 static void suspend_cpu_complex(u32 mode)
405 {
406         int cpu = smp_processor_id();
407         unsigned int reg;
408         int i;
409
410         BUG_ON(cpu != 0);
411
412 #ifdef CONFIG_SMP
413         cpu = cpu_logical_map(cpu);
414 #endif
415         /* switch coresite to clk_m, save off original source */
416         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
417         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
418
419         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
420         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
421         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
422         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
423         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
424         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
425         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
426         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
427
428         tegra_twd_suspend(&tegra_sctx.twd);
429
430         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
431         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
432         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
433         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
434         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
435 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
436         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
437 #else
438         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
439 #endif
440         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
441         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
442
443         for (i = 0; i < num_possible_cpus(); i++) {
444                 if (i == cpu)
445                         continue;
446                 reg = readl(FLOW_CTRL_CPU_CSR(i));
447                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
448                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
449                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
450         }
451
452         tegra_gic_cpu_disable(true);
453 }
454
455 void tegra_clear_cpu_in_lp2(int cpu)
456 {
457         spin_lock(&tegra_lp2_lock);
458         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
459         cpumask_clear_cpu(cpu, &tegra_in_lp2);
460
461         /* Update the IRAM copy used by the reset handler. The IRAM copy
462            can't use used directly by cpumask_clear_cpu() because it uses
463            LDREX/STREX which requires the addressed location to be inner
464            cacheable and sharable which IRAM isn't. */
465         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
466         dsb();
467
468         spin_unlock(&tegra_lp2_lock);
469 }
470
471 bool tegra_set_cpu_in_lp2(int cpu)
472 {
473         bool last_cpu = false;
474
475         spin_lock(&tegra_lp2_lock);
476         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
477         cpumask_set_cpu(cpu, &tegra_in_lp2);
478
479         /* Update the IRAM copy used by the reset handler. The IRAM copy
480            can't use used directly by cpumask_set_cpu() because it uses
481            LDREX/STREX which requires the addressed location to be inner
482            cacheable and sharable which IRAM isn't. */
483         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
484         dsb();
485
486         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
487                 last_cpu = true;
488 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
489         else if (cpu == 1)
490                 tegra2_cpu_set_resettable_soon();
491 #endif
492
493         spin_unlock(&tegra_lp2_lock);
494         return last_cpu;
495 }
496
497 static void tegra_sleep_core(enum tegra_suspend_mode mode,
498                              unsigned long v2p)
499 {
500 #ifdef CONFIG_TRUSTED_FOUNDATIONS
501         if (mode == TEGRA_SUSPEND_LP0) {
502                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
503                                   virt_to_phys(tegra_resume));
504         } else {
505                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
506                                   (TEGRA_RESET_HANDLER_BASE +
507                                    tegra_cpu_reset_handler_offset));
508         }
509 #endif
510 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
511         cpu_suspend(v2p, tegra2_sleep_core_finish);
512 #else
513         cpu_suspend(v2p, tegra3_sleep_core_finish);
514 #endif
515 }
516
517 static inline void tegra_sleep_cpu(unsigned long v2p)
518 {
519 #ifdef CONFIG_TRUSTED_FOUNDATIONS
520         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
521                           (TEGRA_RESET_HANDLER_BASE +
522                            tegra_cpu_reset_handler_offset));
523 #endif
524         cpu_suspend(v2p, tegra_sleep_cpu_finish);
525 }
526
527 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
528 {
529         u32 reg;
530         unsigned int remain;
531         pgd_t *pgd;
532
533         /* Only the last cpu down does the final suspend steps */
534         reg = readl(pmc + PMC_CTRL);
535         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
536         if (pdata->combined_req)
537                 reg &= ~TEGRA_POWER_PWRREQ_OE;
538         else
539                 reg |= TEGRA_POWER_PWRREQ_OE;
540
541         reg &= ~TEGRA_POWER_EFFECT_LP0;
542         writel(reg, pmc + PMC_CTRL);
543
544         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
545
546         /*
547          * We can use clk_get_rate_all_locked() here, because all other cpus
548          * are in LP2 state and irqs are disabled
549          */
550         if (flags & TEGRA_POWER_CLUSTER_MASK) {
551                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
552                 set_power_timers(pdata->cpu_timer, 0,
553                         clk_get_rate_all_locked(tegra_pclk));
554                 if (flags & TEGRA_POWER_CLUSTER_G) {
555                         /*
556                          * To reduce the vdd_cpu up latency when LP->G
557                          * transition. Before the transition, enable
558                          * the vdd_cpu rail.
559                          */
560                         if (is_lp_cluster())
561                                 writel(UN_PWRGATE_CPU,
562                                        pmc + PMC_PWRGATE_TOGGLE);
563                 }
564                 tegra_cluster_switch_prolog(flags);
565         } else {
566                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
567                         clk_get_rate_all_locked(tegra_pclk));
568 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
569                 reg = readl(FLOW_CTRL_CPU_CSR(0));
570                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
571                 if (is_lp_cluster()) {
572                         /* for LP cluster, there is no option for rail gating */
573                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
574                                                 TEGRA_POWER_CLUSTER_PART_MASK)
575                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
576                         else if (flags)
577                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
578                 }
579                 else {
580                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
581                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
582                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
583                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
584                 }
585                 writel(reg, FLOW_CTRL_CPU_CSR(0));
586 #endif
587         }
588
589         if (sleep_time)
590                 tegra_lp2_set_trigger(sleep_time);
591
592         cpu_cluster_pm_enter();
593         suspend_cpu_complex(flags);
594         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
595         flush_cache_all();
596         /*
597          * No need to flush complete L2. Cleaning kernel and IO mappings
598          * is enough for the LP code sequence that has L2 disabled but
599          * MMU on.
600          */
601         pgd = cpu_get_pgd();
602         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
603                           __pa(pgd + PTRS_PER_PGD));
604         outer_disable();
605
606         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
607
608         tegra_init_cache(false);
609         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
610         restore_cpu_complex(flags);
611         cpu_cluster_pm_exit();
612
613         remain = tegra_lp2_timer_remain();
614         if (sleep_time)
615                 tegra_lp2_set_trigger(0);
616
617         if (flags & TEGRA_POWER_CLUSTER_MASK) {
618                 tegra_cluster_switch_epilog(flags);
619                 trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
620         }
621         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
622
623 #if INSTRUMENT_CLUSTER_SWITCH
624         if (flags & TEGRA_POWER_CLUSTER_MASK) {
625                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
626                         is_lp_cluster() ? "G=>LP" : "LP=>G",
627                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
628                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
629                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
630                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
631                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
632                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
633                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
634                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
635         }
636 #endif
637         return remain;
638 }
639
640 static int tegra_common_suspend(void)
641 {
642         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
643
644         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
645         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
646         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
647
648         /* copy the reset vector and SDRAM shutdown code into IRAM */
649         memcpy(iram_save, iram_code, iram_save_size);
650         memcpy(iram_code, tegra_iram_start(), iram_save_size);
651
652         return 0;
653 }
654
655 static void tegra_common_resume(void)
656 {
657         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
658 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
659         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
660 #endif
661
662         /* Clear DPD sample */
663         writel(0x0, pmc + PMC_DPD_SAMPLE);
664
665         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
666         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
667         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
668 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
669         /* trigger emc mode write */
670         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
671 #endif
672         /* clear scratch registers shared by suspend and the reset pen */
673         writel(0x0, pmc + PMC_SCRATCH39);
674         writel(0x0, pmc + PMC_SCRATCH41);
675
676         /* restore IRAM */
677         memcpy(iram_code, iram_save, iram_save_size);
678 }
679
680 static int tegra_suspend_prepare_late(void)
681 {
682 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
683         disable_irq(INT_SYS_STATS_MON);
684 #endif
685         return 0;
686 }
687
688 static void tegra_suspend_wake(void)
689 {
690 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
691         enable_irq(INT_SYS_STATS_MON);
692 #endif
693 }
694
695 static void tegra_pm_set(enum tegra_suspend_mode mode)
696 {
697         u32 reg, boot_flag;
698         unsigned long rate = 32768;
699
700         reg = readl(pmc + PMC_CTRL);
701         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
702         if (pdata->combined_req)
703                 reg &= ~TEGRA_POWER_PWRREQ_OE;
704         else
705                 reg |= TEGRA_POWER_PWRREQ_OE;
706         reg &= ~TEGRA_POWER_EFFECT_LP0;
707
708         switch (mode) {
709         case TEGRA_SUSPEND_LP0:
710 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
711                 rate = clk_get_rate_all_locked(tegra_pclk);
712 #endif
713                 if (pdata->combined_req) {
714                         reg |= TEGRA_POWER_PWRREQ_OE;
715                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
716                 }
717
718                 /*
719                  * LP0 boots through the AVP, which then resumes the AVP to
720                  * the address in scratch 39, and the cpu to the address in
721                  * scratch 41 to tegra_resume
722                  */
723                 writel(0x0, pmc + PMC_SCRATCH39);
724
725                 /* Enable DPD sample to trigger sampling pads data and direction
726                  * in which pad will be driven during lp0 mode*/
727                 writel(0x1, pmc + PMC_DPD_SAMPLE);
728 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
729                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ_0);
730                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ_0);
731 #endif
732                 /* Set warmboot flag */
733                 boot_flag = readl(pmc + PMC_SCRATCH0);
734                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
735
736                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
737
738                 reg |= TEGRA_POWER_EFFECT_LP0;
739                 /* No break here. LP0 code falls through to write SCRATCH41 */
740         case TEGRA_SUSPEND_LP1:
741                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
742                 wmb();
743                 break;
744         case TEGRA_SUSPEND_LP2:
745                 rate = clk_get_rate(tegra_pclk);
746                 break;
747         case TEGRA_SUSPEND_NONE:
748                 return;
749         default:
750                 BUG();
751         }
752
753         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
754
755         pmc_32kwritel(reg, PMC_CTRL);
756 }
757
758 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
759         [TEGRA_SUSPEND_NONE] = "none",
760         [TEGRA_SUSPEND_LP2] = "LP2",
761         [TEGRA_SUSPEND_LP1] = "LP1",
762         [TEGRA_SUSPEND_LP0] = "LP0",
763 };
764
765 static int tegra_suspend_enter(suspend_state_t state)
766 {
767         int ret;
768         ktime_t delta;
769         struct timespec ts_entry, ts_exit;
770
771         if (pdata && pdata->board_suspend)
772                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
773
774         read_persistent_clock(&ts_entry);
775
776         ret = tegra_suspend_dram(current_suspend_mode, 0);
777         if (ret) {
778                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
779                 goto abort_suspend;
780         }
781
782         read_persistent_clock(&ts_exit);
783
784         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
785                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
786
787                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
788                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
789                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
790                 else
791                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
792         }
793
794 abort_suspend:
795         if (pdata && pdata->board_resume)
796                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
797
798         return ret;
799 }
800
801 static void tegra_suspend_check_pwr_stats(void)
802 {
803         /* cpus and l2 are powered off later */
804         unsigned long pwrgate_partid_mask =
805 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
806                 (1 << TEGRA_POWERGATE_HEG)      |
807                 (1 << TEGRA_POWERGATE_SATA)     |
808                 (1 << TEGRA_POWERGATE_3D1)      |
809 #endif
810                 (1 << TEGRA_POWERGATE_3D)       |
811                 (1 << TEGRA_POWERGATE_VENC)     |
812                 (1 << TEGRA_POWERGATE_PCIE)     |
813                 (1 << TEGRA_POWERGATE_VDEC)     |
814                 (1 << TEGRA_POWERGATE_MPE);
815
816         int partid;
817
818         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
819                 if ((1 << partid) & pwrgate_partid_mask)
820                         if (tegra_powergate_is_powered(partid))
821                                 pr_warning("partition %s is left on before suspend\n",
822                                         tegra_powergate_get_name(partid));
823
824         return;
825 }
826
827 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
828 {
829         int err = 0;
830         u32 scratch37 = 0xDEADBEEF;
831
832         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
833                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
834                 err = -ENXIO;
835                 goto fail;
836         }
837
838         if (tegra_is_voice_call_active()) {
839                 u32 reg;
840
841                 /* backup the current value of scratch37 */
842                 scratch37 = readl(pmc + PMC_SCRATCH37);
843
844                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
845                 reg = TEGRA_POWER_LP1_AUDIO;
846                 pmc_32kwritel(reg, PMC_SCRATCH37);
847         }
848
849         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
850                 pr_info("LP0 not used due to unsupported wakeup events\n");
851                 mode = TEGRA_SUSPEND_LP1;
852         }
853
854         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
855                 tegra_suspend_check_pwr_stats();
856
857         tegra_common_suspend();
858
859         tegra_pm_set(mode);
860
861         if (pdata && pdata->board_suspend)
862                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
863
864         local_fiq_disable();
865
866         trace_cpu_suspend(CPU_SUSPEND_START);
867
868         if (mode == TEGRA_SUSPEND_LP0) {
869 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
870                 u32 reg = readl(pmc + PMC_SCRATCH4);
871                 if (is_lp_cluster())
872                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
873                 else
874                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
875                 pmc_32kwritel(reg, PMC_SCRATCH4);
876 #endif
877                 tegra_tsc_suspend();
878                 tegra_lp0_suspend_mc();
879                 tegra_cpu_reset_handler_save();
880                 tegra_tsc_wait_for_suspend();
881         }
882         else if (mode == TEGRA_SUSPEND_LP1)
883                 *iram_cpu_lp1_mask = 1;
884
885         suspend_cpu_complex(flags);
886
887         flush_cache_all();
888         outer_flush_all();
889         outer_disable();
890
891         if (mode == TEGRA_SUSPEND_LP2)
892                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
893         else
894                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
895
896         tegra_init_cache(true);
897
898         if (mode == TEGRA_SUSPEND_LP0) {
899                 tegra_tsc_resume();
900                 tegra_cpu_reset_handler_restore();
901                 tegra_lp0_resume_mc();
902                 tegra_tsc_wait_for_resume();
903         } else if (mode == TEGRA_SUSPEND_LP1)
904                 *iram_cpu_lp1_mask = 0;
905
906         /* if scratch37 was clobbered during LP1, restore it */
907         if (scratch37 != 0xDEADBEEF)
908                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
909
910         restore_cpu_complex(flags);
911
912         /* for platforms where the core & CPU power requests are
913          * combined as a single request to the PMU, transition out
914          * of LP0 state by temporarily enabling both requests
915          */
916         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
917                 u32 reg;
918                 reg = readl(pmc + PMC_CTRL);
919                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
920                 pmc_32kwritel(reg, PMC_CTRL);
921                 reg &= ~TEGRA_POWER_PWRREQ_OE;
922                 pmc_32kwritel(reg, PMC_CTRL);
923         }
924
925         if (pdata && pdata->board_resume)
926                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
927
928         trace_cpu_suspend(CPU_SUSPEND_DONE);
929
930         local_fiq_enable();
931
932         tegra_common_resume();
933
934 fail:
935         return err;
936 }
937
938 /*
939  * Function pointers to optional board specific function
940  */
941 void (*tegra_deep_sleep)(int);
942 EXPORT_SYMBOL(tegra_deep_sleep);
943
944 static int tegra_suspend_prepare(void)
945 {
946         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
947                 tegra_deep_sleep(1);
948         return 0;
949 }
950
951 static void tegra_suspend_finish(void)
952 {
953         if (pdata && pdata->cpu_resume_boost) {
954                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
955                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
956                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
957         }
958
959         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
960                 tegra_deep_sleep(0);
961 }
962
963 static const struct platform_suspend_ops tegra_suspend_ops = {
964         .valid          = suspend_valid_only_mem,
965         .prepare        = tegra_suspend_prepare,
966         .finish         = tegra_suspend_finish,
967         .prepare_late   = tegra_suspend_prepare_late,
968         .wake           = tegra_suspend_wake,
969         .enter          = tegra_suspend_enter,
970 };
971
972 static ssize_t suspend_mode_show(struct kobject *kobj,
973                                         struct kobj_attribute *attr, char *buf)
974 {
975         char *start = buf;
976         char *end = buf + PAGE_SIZE;
977
978         start += scnprintf(start, end - start, "%s ", \
979                                 tegra_suspend_name[current_suspend_mode]);
980         start += scnprintf(start, end - start, "\n");
981
982         return start - buf;
983 }
984
985 static ssize_t suspend_mode_store(struct kobject *kobj,
986                                         struct kobj_attribute *attr,
987                                         const char *buf, size_t n)
988 {
989         int len;
990         const char *name_ptr;
991         enum tegra_suspend_mode new_mode;
992
993         name_ptr = buf;
994         while (*name_ptr && !isspace(*name_ptr))
995                 name_ptr++;
996         len = name_ptr - buf;
997         if (!len)
998                 goto bad_name;
999         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1000         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1001                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1002                 pr_info("Illegal tegra suspend state: %s\n", buf);
1003                 goto bad_name;
1004         }
1005
1006         for (new_mode = TEGRA_SUSPEND_NONE; \
1007                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1008                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1009                         current_suspend_mode = new_mode;
1010                         break;
1011                 }
1012         }
1013
1014 bad_name:
1015         return n;
1016 }
1017
1018 static struct kobj_attribute suspend_mode_attribute =
1019         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1020
1021 static struct kobject *suspend_kobj;
1022
1023 static int tegra_pm_enter_suspend(void)
1024 {
1025         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1026         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1027                 tegra_lp0_cpu_mode(true);
1028         return 0;
1029 }
1030
1031 static void tegra_pm_enter_resume(void)
1032 {
1033         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1034                 tegra_lp0_cpu_mode(false);
1035         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1036 }
1037
1038 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1039         .suspend = tegra_pm_enter_suspend,
1040         .resume = tegra_pm_enter_resume,
1041 };
1042
1043 static __init int tegra_pm_enter_syscore_init(void)
1044 {
1045         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1046         return 0;
1047 }
1048 subsys_initcall(tegra_pm_enter_syscore_init);
1049 #endif
1050
1051 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1052 {
1053         u32 reg;
1054         u32 mode;
1055
1056         tegra_cpu_rail = tegra_dvfs_get_rail_by_name("vdd_cpu");
1057         tegra_core_rail = tegra_dvfs_get_rail_by_name("vdd_core");
1058         pm_qos_add_request(&awake_cpu_freq_req, PM_QOS_CPU_FREQ_MIN,
1059                            AWAKE_CPU_FREQ_MIN);
1060
1061         tegra_pclk = clk_get_sys(NULL, "pclk");
1062         BUG_ON(IS_ERR(tegra_pclk));
1063         pdata = plat;
1064         (void)reg;
1065         (void)mode;
1066
1067         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1068                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1069
1070 #ifndef CONFIG_PM_SLEEP
1071         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1072                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1073                            "disabling suspend\n", __func__);
1074                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1075         }
1076 #else
1077         if (create_suspend_pgtable() < 0) {
1078                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1079                                 __func__);
1080                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1081                 goto fail;
1082         }
1083
1084         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1085             (tegra_revision == TEGRA_REVISION_A01) &&
1086             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1087                 /* Tegra 3 A01 supports only LP1 */
1088                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1089                            "-- disabling LP0\n", __func__);
1090                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1091         }
1092         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1093                 tegra_lp0_vec_relocate) {
1094                 unsigned char *reloc_lp0;
1095                 unsigned long tmp;
1096                 void __iomem *orig;
1097                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1098                                         GFP_KERNEL);
1099                 WARN_ON(!reloc_lp0);
1100                 if (!reloc_lp0) {
1101                         pr_err("%s: Failed to allocate reloc_lp0\n",
1102                                 __func__);
1103                         goto out;
1104                 }
1105
1106                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1107                 WARN_ON(!orig);
1108                 if (!orig) {
1109                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1110                                 __func__, tegra_lp0_vec_start);
1111                         kfree(reloc_lp0);
1112                         goto out;
1113                 }
1114
1115                 tmp = (unsigned long) reloc_lp0;
1116                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1117                 reloc_lp0 = (unsigned char *)tmp;
1118                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1119                 iounmap(orig);
1120                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1121         }
1122
1123 out:
1124         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1125                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1126                            "provided by bootlader -- disabling LP0\n",
1127                            __func__);
1128                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1129         }
1130
1131         iram_save_size = tegra_iram_end() - tegra_iram_start();
1132
1133         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1134         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1135                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1136                        "-- LP0/LP1 unavailable\n", __func__);
1137                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1138         }
1139
1140         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1141         /* Initialize scratch registers used for CPU LP2 synchronization */
1142         writel(0, pmc + PMC_SCRATCH37);
1143         writel(0, pmc + PMC_SCRATCH38);
1144         writel(0, pmc + PMC_SCRATCH39);
1145         writel(0, pmc + PMC_SCRATCH41);
1146
1147         /* Always enable CPU power request; just normal polarity is supported */
1148         reg = readl(pmc + PMC_CTRL);
1149         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1150         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1151         pmc_32kwritel(reg, PMC_CTRL);
1152
1153         /* Configure core power request and system clock control if LP0
1154            is supported */
1155         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1156         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1157
1158         reg = readl(pmc + PMC_CTRL);
1159
1160         if (!pdata->sysclkreq_high)
1161                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1162         else
1163                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1164
1165         if (!pdata->corereq_high)
1166                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1167         else
1168                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1169
1170         /* configure output inverters while the request is tristated */
1171         pmc_32kwritel(reg, PMC_CTRL);
1172
1173         /* now enable requests */
1174         reg |= TEGRA_POWER_SYSCLK_OE;
1175         if (!pdata->combined_req)
1176                 reg |= TEGRA_POWER_PWRREQ_OE;
1177         pmc_32kwritel(reg, PMC_CTRL);
1178
1179         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1180                 tegra_lp0_suspend_init();
1181
1182         suspend_set_ops(&tegra_suspend_ops);
1183
1184         /* Create /sys/power/suspend/type */
1185         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1186         if (suspend_kobj) {
1187                 if (sysfs_create_file(suspend_kobj, \
1188                                                 &suspend_mode_attribute.attr))
1189                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1190                                                                 __func__);
1191         }
1192
1193         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1194         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1195 fail:
1196 #endif
1197         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1198                 tegra_lp2_in_idle(false);
1199
1200         current_suspend_mode = plat->suspend_mode;
1201 }
1202
1203 unsigned long debug_uart_port_base = 0;
1204 EXPORT_SYMBOL(debug_uart_port_base);
1205
1206 static int tegra_debug_uart_suspend(void)
1207 {
1208         void __iomem *uart;
1209         u32 lcr;
1210
1211         if (!debug_uart_port_base)
1212                 return 0;
1213
1214         uart = IO_ADDRESS(debug_uart_port_base);
1215
1216         lcr = readb(uart + UART_LCR * 4);
1217
1218         tegra_sctx.uart[0] = lcr;
1219         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1220
1221         /* DLAB = 0 */
1222         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1223
1224         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1225
1226         /* DLAB = 1 */
1227         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1228
1229         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1230         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1231
1232         writeb(lcr, uart + UART_LCR * 4);
1233
1234         return 0;
1235 }
1236
1237 static void tegra_debug_uart_resume(void)
1238 {
1239         void __iomem *uart;
1240         u32 lcr;
1241
1242         if (!debug_uart_port_base)
1243                 return;
1244
1245         uart = IO_ADDRESS(debug_uart_port_base);
1246
1247         lcr = tegra_sctx.uart[0];
1248
1249         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1250
1251         /* DLAB = 0 */
1252         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1253
1254         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1255                         uart + UART_FCR * 4);
1256
1257         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1258
1259         /* DLAB = 1 */
1260         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1261
1262         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1263         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1264
1265         writeb(lcr, uart + UART_LCR * 4);
1266 }
1267
1268 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1269         .suspend = tegra_debug_uart_suspend,
1270         .resume = tegra_debug_uart_resume,
1271 };
1272
1273 struct clk *debug_uart_clk = NULL;
1274 EXPORT_SYMBOL(debug_uart_clk);
1275
1276 void tegra_console_uart_suspend(void)
1277 {
1278         if (console_suspend_enabled && debug_uart_clk)
1279                 clk_disable(debug_uart_clk);
1280 }
1281
1282 void tegra_console_uart_resume(void)
1283 {
1284         if (console_suspend_enabled && debug_uart_clk)
1285                 clk_enable(debug_uart_clk);
1286 }
1287
1288 static int tegra_debug_uart_syscore_init(void)
1289 {
1290         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1291         return 0;
1292 }
1293 arch_initcall(tegra_debug_uart_syscore_init);
1294
1295 #ifdef CONFIG_HAS_EARLYSUSPEND
1296 static struct clk *clk_wake;
1297 static void pm_early_suspend(struct early_suspend *h)
1298 {
1299         if (clk_wake)
1300                 clk_disable(clk_wake);
1301         pm_qos_update_request(&awake_cpu_freq_req, PM_QOS_DEFAULT_VALUE);
1302 }
1303
1304 static void pm_late_resume(struct early_suspend *h)
1305 {
1306         if (clk_wake)
1307                 clk_enable(clk_wake);
1308         pm_qos_update_request(&awake_cpu_freq_req, (s32)AWAKE_CPU_FREQ_MIN);
1309 }
1310
1311 static struct early_suspend pm_early_suspender = {
1312                 .suspend = pm_early_suspend,
1313                 .resume = pm_late_resume,
1314 };
1315
1316 static int pm_init_wake_behavior(void)
1317 {
1318         clk_wake = tegra_get_clock_by_name("wake.sclk");
1319         register_early_suspend(&pm_early_suspender);
1320         return 0;
1321 }
1322
1323 late_initcall(pm_init_wake_behavior);
1324 #endif