unknown changes from android-tegra-nv-3.4
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50
51 #include <asm/cacheflush.h>
52 #include <asm/idmap.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57 #include <asm/suspend.h>
58
59 #include <mach/irqs.h>
60 #include <mach/powergate.h>
61 #include <mach/hardware.h>
62
63 #include "board.h"
64 #include "clock.h"
65 #include "cpuidle.h"
66 #include "fuse.h"
67 #include "gic.h"
68 #include "iomap.h"
69 #include "pm.h"
70 #include "pm-irq.h"
71 #include "reset.h"
72 #include "sleep.h"
73 #include "timer.h"
74 #include "dvfs.h"
75 #include "cpu-tegra.h"
76
77 struct suspend_context {
78         /*
79          * The next 7 values are referenced by offset in __restart_plls
80          * in headsmp-t2.S, and should not be moved
81          */
82         u32 pllx_misc;
83         u32 pllx_base;
84         u32 pllp_misc;
85         u32 pllp_base;
86         u32 pllp_outa;
87         u32 pllp_outb;
88         u32 pll_timeout;
89
90         u32 cpu_burst;
91         u32 clk_csite_src;
92         u32 cclk_divider;
93
94         u32 mc[3];
95         u8 uart[5];
96
97         struct tegra_twd_context twd;
98 };
99
100 #ifdef CONFIG_PM_SLEEP
101 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
102 static pgd_t *tegra_pgd;
103 static DEFINE_SPINLOCK(tegra_lp2_lock);
104 static cpumask_t tegra_in_lp2;
105 static cpumask_t *iram_cpu_lp2_mask;
106 static unsigned long *iram_cpu_lp1_mask;
107 static u8 *iram_save;
108 static unsigned long iram_save_size;
109 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
110 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
111 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
112 static int tegra_last_pclk;
113 #endif
114
115 struct suspend_context tegra_sctx;
116
117 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
118 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
119 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
120 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
121 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
122 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
123 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
124 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
125
126 #define PMC_CTRL                0x0
127 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
128 #define PMC_WAKE_MASK           0xc
129 #define PMC_WAKE_LEVEL          0x10
130 #define PMC_DPAD_ORIDE          0x1C
131 #define PMC_WAKE_DELAY          0xe0
132 #define PMC_DPD_SAMPLE          0x20
133
134 #define PMC_WAKE_STATUS         0x14
135 #define PMC_SW_WAKE_STATUS      0x18
136 #define PMC_COREPWRGOOD_TIMER   0x3c
137 #define PMC_CPUPWRGOOD_TIMER    0xc8
138 #define PMC_CPUPWROFF_TIMER     0xcc
139 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
140
141 #define PMC_PWRGATE_TOGGLE      0x30
142 #define PWRGATE_TOGGLE_START    (1 << 8)
143 #define UN_PWRGATE_CPU          \
144         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
145
146 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
147 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
148 #endif
149
150 #define CLK_RESET_CCLK_BURST    0x20
151 #define CLK_RESET_CCLK_DIVIDER  0x24
152 #define CLK_RESET_PLLC_BASE     0x80
153 #define CLK_RESET_PLLM_BASE     0x90
154 #define CLK_RESET_PLLX_BASE     0xe0
155 #define CLK_RESET_PLLX_MISC     0xe4
156 #define CLK_RESET_PLLP_BASE     0xa0
157 #define CLK_RESET_PLLP_OUTA     0xa4
158 #define CLK_RESET_PLLP_OUTB     0xa8
159 #define CLK_RESET_PLLP_MISC     0xac
160
161 #define CLK_RESET_SOURCE_CSITE  0x1d4
162
163 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
164 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
165 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
166 #define CLK_RESET_CCLK_IDLE_POLICY         1
167 #define CLK_RESET_CCLK_RUN_POLICY          2
168 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
169 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
170
171 #define EMC_MRW_0               0x0e8
172 #define EMC_MRW_DEV_SELECTN     30
173 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
174
175 #define MC_SECURITY_START       0x6c
176 #define MC_SECURITY_SIZE        0x70
177 #define MC_SECURITY_CFG2        0x7c
178
179 struct dvfs_rail *tegra_cpu_rail;
180 static struct dvfs_rail *tegra_core_rail;
181 static struct clk *tegra_pclk;
182 static const struct tegra_suspend_platform_data *pdata;
183 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
184
185 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
186 enum tegra_cluster_switch_time_id {
187         tegra_cluster_switch_time_id_start = 0,
188         tegra_cluster_switch_time_id_prolog,
189         tegra_cluster_switch_time_id_switch,
190         tegra_cluster_switch_time_id_epilog,
191         tegra_cluster_switch_time_id_max
192 };
193
194 static unsigned long
195                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
196 #define tegra_cluster_switch_time(flags, id) \
197         do { \
198                 barrier(); \
199                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
200                         void __iomem *timer_us = \
201                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
202                         if (id < tegra_cluster_switch_time_id_max) \
203                                 tegra_cluster_switch_times[id] = \
204                                                         readl(timer_us); \
205                                 wmb(); \
206                 } \
207                 barrier(); \
208         } while(0)
209 #else
210 #define tegra_cluster_switch_time(flags, id) do {} while(0)
211 #endif
212
213 #ifdef CONFIG_PM_SLEEP
214 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
215         [TEGRA_SUSPEND_NONE]    = "none",
216         [TEGRA_SUSPEND_LP2]     = "lp2",
217         [TEGRA_SUSPEND_LP1]     = "lp1",
218         [TEGRA_SUSPEND_LP0]     = "lp0",
219 };
220
221 unsigned long tegra_cpu_power_good_time(void)
222 {
223         if (WARN_ON_ONCE(!pdata))
224                 return 5000;
225
226         return pdata->cpu_timer;
227 }
228
229 unsigned long tegra_cpu_power_off_time(void)
230 {
231         if (WARN_ON_ONCE(!pdata))
232                 return 5000;
233
234         return pdata->cpu_off_timer;
235 }
236
237 unsigned long tegra_cpu_lp2_min_residency(void)
238 {
239         if (WARN_ON_ONCE(!pdata))
240                 return 2000;
241
242         return pdata->cpu_lp2_min_residency;
243 }
244
245 /*
246  * create_suspend_pgtable
247  *
248  * Creates a page table with identity mappings of physical memory and IRAM
249  * for use when the MMU is off, in addition to all the regular kernel mappings.
250  */
251 static __init int create_suspend_pgtable(void)
252 {
253         tegra_pgd = pgd_alloc(&init_mm);
254         if (!tegra_pgd)
255                 return -ENOMEM;
256
257         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
258         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
259                 high_memory, 0);
260         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
261                 IO_IRAM_VIRT + SECTION_SIZE, 0);
262
263         /* inner/outer write-back/write-allocate, sharable */
264         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
265
266         return 0;
267 }
268
269 /* ensures that sufficient time is passed for a register write to
270  * serialize into the 32KHz domain */
271 static void pmc_32kwritel(u32 val, unsigned long offs)
272 {
273         writel(val, pmc + offs);
274         udelay(130);
275 }
276
277 static void set_power_timers(unsigned long us_on, unsigned long us_off,
278                              long rate)
279 {
280         static unsigned long last_us_off = 0;
281         unsigned long long ticks;
282         unsigned long long pclk;
283
284         if (WARN_ON_ONCE(rate <= 0))
285                 pclk = 100000000;
286         else
287                 pclk = rate;
288
289         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
290                 ticks = (us_on * pclk) + 999999ull;
291                 do_div(ticks, 1000000);
292                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
293
294                 ticks = (us_off * pclk) + 999999ull;
295                 do_div(ticks, 1000000);
296                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
297                 wmb();
298         }
299         tegra_last_pclk = pclk;
300         last_us_off = us_off;
301 }
302
303 /*
304  * restore_cpu_complex
305  *
306  * restores cpu clock setting, clears flow controller
307  *
308  * Always called on CPU 0.
309  */
310 static void restore_cpu_complex(u32 mode)
311 {
312         int cpu = smp_processor_id();
313         unsigned int reg, policy;
314
315         BUG_ON(cpu != 0);
316
317         /* Is CPU complex already running on PLLX? */
318         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
319         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
320
321         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
322                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
323         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
324                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
325         else
326                 BUG();
327
328         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
329                 /* restore PLLX settings if CPU is on different PLL */
330                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
331                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
332
333                 /* wait for PLL stabilization if PLLX was enabled */
334                 if (tegra_sctx.pllx_base & (1<<30)) {
335 #if USE_PLL_LOCK_BITS
336                         /* Enable lock detector */
337                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
338                         reg |= 1<<18;
339                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
340                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
341                                  (1<<27)))
342                                 cpu_relax();
343
344                         udelay(PLL_POST_LOCK_DELAY);
345 #else
346                         udelay(300);
347 #endif
348                 }
349         }
350
351         /* Restore original burst policy setting for calls resulting from CPU
352            LP2 in idle or system suspend; keep cluster switch prolog setting
353            intact. */
354         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
355                 writel(tegra_sctx.cclk_divider, clk_rst +
356                        CLK_RESET_CCLK_DIVIDER);
357                 writel(tegra_sctx.cpu_burst, clk_rst +
358                        CLK_RESET_CCLK_BURST);
359         }
360
361         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
362
363         /* Do not power-gate CPU 0 when flow controlled */
364         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
365         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
366         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
367         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
368         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
369         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
370         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
371
372         /* If an immedidate cluster switch is being perfomed, restore the
373            local timer registers. For calls resulting from CPU LP2 in
374            idle or system suspend, the local timer was shut down and
375            timekeeping switched over to the global system timer. In this
376            case keep local timer disabled, and restore only periodic load. */
377         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
378                       TEGRA_POWER_CLUSTER_IMMEDIATE)))
379                 tegra_sctx.twd.twd_ctrl = 0;
380         tegra_twd_resume(&tegra_sctx.twd);
381 }
382
383 /*
384  * suspend_cpu_complex
385  *
386  * saves pll state for use by restart_plls, prepares flow controller for
387  * transition to suspend state
388  *
389  * Must always be called on cpu 0.
390  */
391 static void suspend_cpu_complex(u32 mode)
392 {
393         int cpu = smp_processor_id();
394         unsigned int reg;
395         int i;
396
397         BUG_ON(cpu != 0);
398
399         /* switch coresite to clk_m, save off original source */
400         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
401         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
402
403         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
404         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
405         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
406         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
407         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
408         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
409         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
410         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
411
412         tegra_twd_suspend(&tegra_sctx.twd);
413
414         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
415         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
416         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
417         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
418         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
419 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
420         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
421 #else
422         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
423 #endif
424         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
425         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
426
427         for (i = 0; i < num_possible_cpus(); i++) {
428                 if (i == cpu)
429                         continue;
430                 reg = readl(FLOW_CTRL_CPU_CSR(i));
431                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
432                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
433                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
434         }
435
436         tegra_gic_cpu_disable();
437 #ifndef CONFIG_ARCH_TEGRA_2x_SOC
438         /* Tegra3 enters LPx states via WFI - do not propagate legacy IRQs
439            to CPU core to avoid fall through WFI (IRQ-to-flow controller wake
440            path is not affected). */
441         tegra_gic_pass_through_disable();
442 #endif
443 }
444
445 void tegra_clear_cpu_in_lp2(int cpu)
446 {
447         spin_lock(&tegra_lp2_lock);
448         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
449         cpumask_clear_cpu(cpu, &tegra_in_lp2);
450
451         /* Update the IRAM copy used by the reset handler. The IRAM copy
452            can't use used directly by cpumask_clear_cpu() because it uses
453            LDREX/STREX which requires the addressed location to be inner
454            cacheable and sharable which IRAM isn't. */
455         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
456         dsb();
457
458         spin_unlock(&tegra_lp2_lock);
459 }
460
461 bool tegra_set_cpu_in_lp2(int cpu)
462 {
463         bool last_cpu = false;
464
465         spin_lock(&tegra_lp2_lock);
466         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
467         cpumask_set_cpu(cpu, &tegra_in_lp2);
468
469         /* Update the IRAM copy used by the reset handler. The IRAM copy
470            can't use used directly by cpumask_set_cpu() because it uses
471            LDREX/STREX which requires the addressed location to be inner
472            cacheable and sharable which IRAM isn't. */
473         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
474         dsb();
475
476         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
477                 last_cpu = true;
478 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
479         else if (cpu == 1)
480                 tegra2_cpu_set_resettable_soon();
481 #endif
482
483         spin_unlock(&tegra_lp2_lock);
484         return last_cpu;
485 }
486
487 static void tegra_sleep_core(enum tegra_suspend_mode mode,
488                              unsigned long v2p)
489 {
490 #ifdef CONFIG_TRUSTED_FOUNDATIONS
491         if (mode == TEGRA_SUSPEND_LP0) {
492                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
493                                   virt_to_phys(tegra_resume));
494         } else {
495                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
496                                   (TEGRA_RESET_HANDLER_BASE +
497                                    tegra_cpu_reset_handler_offset));
498         }
499 #endif
500 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
501         cpu_suspend(v2p, tegra2_sleep_core_finish);
502 #else
503         cpu_suspend(v2p, tegra3_sleep_core_finish);
504 #endif
505 }
506
507 static inline void tegra_sleep_cpu(unsigned long v2p)
508 {
509 #ifdef CONFIG_TRUSTED_FOUNDATIONS
510         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
511                           (TEGRA_RESET_HANDLER_BASE +
512                            tegra_cpu_reset_handler_offset));
513 #endif
514         cpu_suspend(v2p, tegra_sleep_cpu_finish);
515 }
516
517 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
518 {
519         u32 mode;       /* hardware + software power mode flags */
520         unsigned int remain;
521         pgd_t *pgd;
522
523         /* Only the last cpu down does the final suspend steps */
524         mode = readl(pmc + PMC_CTRL);
525         mode |= TEGRA_POWER_CPU_PWRREQ_OE;
526         if (pdata->combined_req)
527                 mode &= ~TEGRA_POWER_PWRREQ_OE;
528         else
529                 mode |= TEGRA_POWER_PWRREQ_OE;
530         mode &= ~TEGRA_POWER_EFFECT_LP0;
531         writel(mode, pmc + PMC_CTRL);
532         mode |= flags;
533
534         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
535
536         /*
537          * We can use clk_get_rate_all_locked() here, because all other cpus
538          * are in LP2 state and irqs are disabled
539          */
540         if (flags & TEGRA_POWER_CLUSTER_MASK) {
541                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
542                 set_power_timers(pdata->cpu_timer, 0,
543                         clk_get_rate_all_locked(tegra_pclk));
544                 if (flags & TEGRA_POWER_CLUSTER_G) {
545                         /*
546                          * To reduce the vdd_cpu up latency when LP->G
547                          * transition. Before the transition, enable
548                          * the vdd_cpu rail.
549                          */
550                         if (is_lp_cluster())
551                                 writel(UN_PWRGATE_CPU,
552                                        pmc + PMC_PWRGATE_TOGGLE);
553                 }
554                 tegra_cluster_switch_prolog(mode);
555         } else {
556                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
557                         clk_get_rate_all_locked(tegra_pclk));
558         }
559
560         if (sleep_time)
561                 tegra_lp2_set_trigger(sleep_time);
562
563         cpu_cluster_pm_enter();
564         suspend_cpu_complex(mode);
565         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
566         flush_cache_all();
567         /*
568          * No need to flush complete L2. Cleaning kernel and IO mappings
569          * is enough for the LP code sequence that has L2 disabled but
570          * MMU on.
571          */
572         pgd = cpu_get_pgd();
573         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
574                           __pa(pgd + PTRS_PER_PGD));
575         outer_disable();
576
577         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
578
579         tegra_init_cache(false);
580         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
581         restore_cpu_complex(mode);
582         cpu_cluster_pm_exit();
583
584         remain = tegra_lp2_timer_remain();
585         if (sleep_time)
586                 tegra_lp2_set_trigger(0);
587
588         if (flags & TEGRA_POWER_CLUSTER_MASK) {
589                 tegra_cluster_switch_epilog(mode);
590                 trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
591         }
592         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
593
594 #if INSTRUMENT_CLUSTER_SWITCH
595         if (flags & TEGRA_POWER_CLUSTER_MASK) {
596                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
597                         is_lp_cluster() ? "G=>LP" : "LP=>G",
598                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
599                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
600                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
601                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
602                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
603                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
604                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
605                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
606         }
607 #endif
608         return remain;
609 }
610
611 static int tegra_common_suspend(void)
612 {
613         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
614
615         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
616         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
617         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
618
619         /* copy the reset vector and SDRAM shutdown code into IRAM */
620         memcpy(iram_save, iram_code, iram_save_size);
621         memcpy(iram_code, tegra_iram_start(), iram_save_size);
622
623         return 0;
624 }
625
626 static void tegra_common_resume(void)
627 {
628         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
629 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
630         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
631 #endif
632
633         /* Clear DPD sample */
634         writel(0x0, pmc + PMC_DPD_SAMPLE);
635
636         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
637         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
638         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
639 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
640         /* trigger emc mode write */
641         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
642 #endif
643         /* clear scratch registers shared by suspend and the reset pen */
644         writel(0x0, pmc + PMC_SCRATCH39);
645         writel(0x0, pmc + PMC_SCRATCH41);
646
647         /* restore IRAM */
648         memcpy(iram_code, iram_save, iram_save_size);
649 }
650
651 static int tegra_suspend_prepare_late(void)
652 {
653 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
654         disable_irq(INT_SYS_STATS_MON);
655 #endif
656         return 0;
657 }
658
659 static void tegra_suspend_wake(void)
660 {
661 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
662         enable_irq(INT_SYS_STATS_MON);
663 #endif
664 }
665
666 static void tegra_pm_set(enum tegra_suspend_mode mode)
667 {
668         u32 reg, boot_flag;
669         unsigned long rate = 32768;
670
671         reg = readl(pmc + PMC_CTRL);
672         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
673         if (pdata->combined_req)
674                 reg &= ~TEGRA_POWER_PWRREQ_OE;
675         else
676                 reg |= TEGRA_POWER_PWRREQ_OE;
677         reg &= ~TEGRA_POWER_EFFECT_LP0;
678
679         switch (mode) {
680         case TEGRA_SUSPEND_LP0:
681 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
682                 rate = clk_get_rate_all_locked(tegra_pclk);
683 #endif
684                 if (pdata->combined_req) {
685                         reg |= TEGRA_POWER_PWRREQ_OE;
686                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
687                 }
688
689                 /*
690                  * LP0 boots through the AVP, which then resumes the AVP to
691                  * the address in scratch 39, and the cpu to the address in
692                  * scratch 41 to tegra_resume
693                  */
694                 writel(0x0, pmc + PMC_SCRATCH39);
695
696                 /* Enable DPD sample to trigger sampling pads data and direction
697                  * in which pad will be driven during lp0 mode*/
698                 writel(0x1, pmc + PMC_DPD_SAMPLE);
699
700                 /* Set warmboot flag */
701                 boot_flag = readl(pmc + PMC_SCRATCH0);
702                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
703
704                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
705
706                 reg |= TEGRA_POWER_EFFECT_LP0;
707                 /* No break here. LP0 code falls through to write SCRATCH41 */
708         case TEGRA_SUSPEND_LP1:
709                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
710                 wmb();
711                 break;
712         case TEGRA_SUSPEND_LP2:
713                 rate = clk_get_rate(tegra_pclk);
714                 break;
715         case TEGRA_SUSPEND_NONE:
716                 return;
717         default:
718                 BUG();
719         }
720
721         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
722
723         pmc_32kwritel(reg, PMC_CTRL);
724 }
725
726 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
727         [TEGRA_SUSPEND_NONE] = "none",
728         [TEGRA_SUSPEND_LP2] = "LP2",
729         [TEGRA_SUSPEND_LP1] = "LP1",
730         [TEGRA_SUSPEND_LP0] = "LP0",
731 };
732
733 static int tegra_suspend_enter(suspend_state_t state)
734 {
735         int ret;
736         ktime_t delta;
737         struct timespec ts_entry, ts_exit;
738
739         if (pdata && pdata->board_suspend)
740                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
741
742         read_persistent_clock(&ts_entry);
743
744         ret = tegra_suspend_dram(current_suspend_mode, 0);
745         if (ret) {
746                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
747                 goto abort_suspend;
748         }
749
750         read_persistent_clock(&ts_exit);
751
752         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
753                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
754
755                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
756                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
757                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
758                 else
759                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
760         }
761
762 abort_suspend:
763         if (pdata && pdata->board_resume)
764                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
765
766         return ret;
767 }
768
769 static void tegra_suspend_check_pwr_stats(void)
770 {
771         /* cpus and l2 are powered off later */
772         unsigned long pwrgate_partid_mask =
773 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
774                 (1 << TEGRA_POWERGATE_HEG)      |
775                 (1 << TEGRA_POWERGATE_SATA)     |
776                 (1 << TEGRA_POWERGATE_3D1)      |
777 #endif
778                 (1 << TEGRA_POWERGATE_3D)       |
779                 (1 << TEGRA_POWERGATE_VENC)     |
780                 (1 << TEGRA_POWERGATE_PCIE)     |
781                 (1 << TEGRA_POWERGATE_VDEC)     |
782                 (1 << TEGRA_POWERGATE_MPE);
783
784         int partid;
785
786         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
787                 if ((1 << partid) & pwrgate_partid_mask)
788                         if (tegra_powergate_is_powered(partid))
789                                 pr_warning("partition %s is left on before suspend\n",
790                                         tegra_powergate_get_name(partid));
791
792         return;
793 }
794
795 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
796 {
797         int err = 0;
798         u32 scratch37 = 0xDEADBEEF;
799
800         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
801                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
802                 err = -ENXIO;
803                 goto fail;
804         }
805
806         if (tegra_is_voice_call_active()) {
807                 u32 reg;
808
809                 /* backup the current value of scratch37 */
810                 scratch37 = readl(pmc + PMC_SCRATCH37);
811
812                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
813                 reg = TEGRA_POWER_LP1_AUDIO;
814                 pmc_32kwritel(reg, PMC_SCRATCH37);
815         }
816
817         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
818                 pr_info("LP0 not used due to unsupported wakeup events\n");
819                 mode = TEGRA_SUSPEND_LP1;
820         }
821
822         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
823                 tegra_suspend_check_pwr_stats();
824
825         tegra_common_suspend();
826
827         tegra_pm_set(mode);
828
829         if (pdata && pdata->board_suspend)
830                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
831
832         local_fiq_disable();
833
834         trace_cpu_suspend(CPU_SUSPEND_START);
835
836         if (mode == TEGRA_SUSPEND_LP0) {
837 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
838                 u32 reg = readl(pmc + PMC_SCRATCH4);
839                 if (is_lp_cluster())
840                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
841                 else
842                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
843                 pmc_32kwritel(reg, PMC_SCRATCH4);
844 #endif
845                 tegra_lp0_suspend_mc();
846                 tegra_cpu_reset_handler_save();
847
848         }
849         else if (mode == TEGRA_SUSPEND_LP1)
850                 *iram_cpu_lp1_mask = 1;
851
852         suspend_cpu_complex(flags);
853
854         flush_cache_all();
855         outer_flush_all();
856         outer_disable();
857
858         if (mode == TEGRA_SUSPEND_LP2)
859                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
860         else
861                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
862
863         tegra_init_cache(true);
864
865         if (mode == TEGRA_SUSPEND_LP0) {
866                 tegra_cpu_reset_handler_restore();
867                 tegra_lp0_resume_mc();
868         } else if (mode == TEGRA_SUSPEND_LP1)
869                 *iram_cpu_lp1_mask = 0;
870
871         /* if scratch37 was clobbered during LP1, restore it */
872         if (scratch37 != 0xDEADBEEF)
873                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
874
875         restore_cpu_complex(flags);
876
877         /* for platforms where the core & CPU power requests are
878          * combined as a single request to the PMU, transition out
879          * of LP0 state by temporarily enabling both requests
880          */
881         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
882                 u32 reg;
883                 reg = readl(pmc + PMC_CTRL);
884                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
885                 pmc_32kwritel(reg, PMC_CTRL);
886                 reg &= ~TEGRA_POWER_PWRREQ_OE;
887                 pmc_32kwritel(reg, PMC_CTRL);
888         }
889
890         if (pdata && pdata->board_resume)
891                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
892
893         trace_cpu_suspend(CPU_SUSPEND_DONE);
894
895         local_fiq_enable();
896
897         tegra_common_resume();
898
899 fail:
900         return err;
901 }
902
903 /*
904  * Function pointers to optional board specific function
905  */
906 void (*tegra_deep_sleep)(int);
907 EXPORT_SYMBOL(tegra_deep_sleep);
908
909 static int tegra_suspend_prepare(void)
910 {
911         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
912                 tegra_deep_sleep(1);
913         return 0;
914 }
915
916 static void tegra_suspend_finish(void)
917 {
918         if (pdata && pdata->cpu_resume_boost) {
919                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
920                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
921                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
922         }
923
924         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
925                 tegra_deep_sleep(0);
926 }
927
928 static const struct platform_suspend_ops tegra_suspend_ops = {
929         .valid          = suspend_valid_only_mem,
930         .prepare        = tegra_suspend_prepare,
931         .finish         = tegra_suspend_finish,
932         .prepare_late   = tegra_suspend_prepare_late,
933         .wake           = tegra_suspend_wake,
934         .enter          = tegra_suspend_enter,
935 };
936
937 static ssize_t suspend_mode_show(struct kobject *kobj,
938                                         struct kobj_attribute *attr, char *buf)
939 {
940         char *start = buf;
941         char *end = buf + PAGE_SIZE;
942
943         start += scnprintf(start, end - start, "%s ", \
944                                 tegra_suspend_name[current_suspend_mode]);
945         start += scnprintf(start, end - start, "\n");
946
947         return start - buf;
948 }
949
950 static ssize_t suspend_mode_store(struct kobject *kobj,
951                                         struct kobj_attribute *attr,
952                                         const char *buf, size_t n)
953 {
954         int len;
955         const char *name_ptr;
956         enum tegra_suspend_mode new_mode;
957
958         name_ptr = buf;
959         while (*name_ptr && !isspace(*name_ptr))
960                 name_ptr++;
961         len = name_ptr - buf;
962         if (!len)
963                 goto bad_name;
964         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
965         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
966                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
967                 pr_info("Illegal tegra suspend state: %s\n", buf);
968                 goto bad_name;
969         }
970
971         for (new_mode = TEGRA_SUSPEND_NONE; \
972                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
973                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
974                         current_suspend_mode = new_mode;
975                         break;
976                 }
977         }
978
979 bad_name:
980         return n;
981 }
982
983 static struct kobj_attribute suspend_mode_attribute =
984         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
985
986 static struct kobject *suspend_kobj;
987
988 static int tegra_pm_enter_suspend(void)
989 {
990         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
991         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
992                 tegra_lp0_cpu_mode(true);
993         return 0;
994 }
995
996 static void tegra_pm_enter_resume(void)
997 {
998         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
999                 tegra_lp0_cpu_mode(false);
1000         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1001 }
1002
1003 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1004         .suspend = tegra_pm_enter_suspend,
1005         .resume = tegra_pm_enter_resume,
1006 };
1007
1008 static __init int tegra_pm_enter_syscore_init(void)
1009 {
1010         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1011         return 0;
1012 }
1013 subsys_initcall(tegra_pm_enter_syscore_init);
1014 #endif
1015
1016 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1017 {
1018         u32 reg;
1019         u32 mode;
1020
1021         tegra_cpu_rail = tegra_dvfs_get_rail_by_name("vdd_cpu");
1022         tegra_core_rail = tegra_dvfs_get_rail_by_name("vdd_core");
1023         tegra_pclk = clk_get_sys(NULL, "pclk");
1024         BUG_ON(IS_ERR(tegra_pclk));
1025         pdata = plat;
1026         (void)reg;
1027         (void)mode;
1028
1029         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1030                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1031
1032 #ifndef CONFIG_PM_SLEEP
1033         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1034                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1035                            "disabling suspend\n", __func__);
1036                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1037         }
1038 #else
1039         if (create_suspend_pgtable() < 0) {
1040                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1041                                 __func__);
1042                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1043                 goto fail;
1044         }
1045
1046         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1047             (tegra_revision == TEGRA_REVISION_A01) &&
1048             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1049                 /* Tegra 3 A01 supports only LP1 */
1050                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1051                            "-- disabling LP0\n", __func__);
1052                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1053         }
1054         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1055                 tegra_lp0_vec_relocate) {
1056                 unsigned char *reloc_lp0;
1057                 unsigned long tmp;
1058                 void __iomem *orig;
1059                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1060                                         GFP_KERNEL);
1061                 WARN_ON(!reloc_lp0);
1062                 if (!reloc_lp0) {
1063                         pr_err("%s: Failed to allocate reloc_lp0\n",
1064                                 __func__);
1065                         goto out;
1066                 }
1067
1068                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1069                 WARN_ON(!orig);
1070                 if (!orig) {
1071                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1072                                 __func__, tegra_lp0_vec_start);
1073                         kfree(reloc_lp0);
1074                         goto out;
1075                 }
1076
1077                 tmp = (unsigned long) reloc_lp0;
1078                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1079                 reloc_lp0 = (unsigned char *)tmp;
1080                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1081                 iounmap(orig);
1082                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1083         }
1084
1085 out:
1086         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1087                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1088                            "provided by bootlader -- disabling LP0\n",
1089                            __func__);
1090                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1091         }
1092
1093         iram_save_size = tegra_iram_end() - tegra_iram_start();
1094
1095         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1096         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1097                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1098                        "-- LP0/LP1 unavailable\n", __func__);
1099                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1100         }
1101
1102         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1103         /* Initialize scratch registers used for CPU LP2 synchronization */
1104         writel(0, pmc + PMC_SCRATCH37);
1105         writel(0, pmc + PMC_SCRATCH38);
1106         writel(0, pmc + PMC_SCRATCH39);
1107         writel(0, pmc + PMC_SCRATCH41);
1108
1109         /* Always enable CPU power request; just normal polarity is supported */
1110         reg = readl(pmc + PMC_CTRL);
1111         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1112         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1113         pmc_32kwritel(reg, PMC_CTRL);
1114
1115         /* Configure core power request and system clock control if LP0
1116            is supported */
1117         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1118         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1119
1120         reg = readl(pmc + PMC_CTRL);
1121
1122         if (!pdata->sysclkreq_high)
1123                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1124         else
1125                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1126
1127         if (!pdata->corereq_high)
1128                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1129         else
1130                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1131
1132         /* configure output inverters while the request is tristated */
1133         pmc_32kwritel(reg, PMC_CTRL);
1134
1135         /* now enable requests */
1136         reg |= TEGRA_POWER_SYSCLK_OE;
1137         if (!pdata->combined_req)
1138                 reg |= TEGRA_POWER_PWRREQ_OE;
1139         pmc_32kwritel(reg, PMC_CTRL);
1140
1141         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1142                 tegra_lp0_suspend_init();
1143
1144         suspend_set_ops(&tegra_suspend_ops);
1145
1146         /* Create /sys/power/suspend/type */
1147         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1148         if (suspend_kobj) {
1149                 if (sysfs_create_file(suspend_kobj, \
1150                                                 &suspend_mode_attribute.attr))
1151                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1152                                                                 __func__);
1153         }
1154
1155         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1156         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1157 fail:
1158 #endif
1159         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1160                 tegra_lp2_in_idle(false);
1161
1162         current_suspend_mode = plat->suspend_mode;
1163 }
1164
1165 unsigned long debug_uart_port_base = 0;
1166 EXPORT_SYMBOL(debug_uart_port_base);
1167
1168 static int tegra_debug_uart_suspend(void)
1169 {
1170         void __iomem *uart;
1171         u32 lcr;
1172
1173         if (!debug_uart_port_base)
1174                 return 0;
1175
1176         uart = IO_ADDRESS(debug_uart_port_base);
1177
1178         lcr = readb(uart + UART_LCR * 4);
1179
1180         tegra_sctx.uart[0] = lcr;
1181         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1182
1183         /* DLAB = 0 */
1184         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1185
1186         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1187
1188         /* DLAB = 1 */
1189         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1190
1191         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1192         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1193
1194         writeb(lcr, uart + UART_LCR * 4);
1195
1196         return 0;
1197 }
1198
1199 static void tegra_debug_uart_resume(void)
1200 {
1201         void __iomem *uart;
1202         u32 lcr;
1203
1204         if (!debug_uart_port_base)
1205                 return;
1206
1207         uart = IO_ADDRESS(debug_uart_port_base);
1208
1209         lcr = tegra_sctx.uart[0];
1210
1211         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1212
1213         /* DLAB = 0 */
1214         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1215
1216         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1217                         uart + UART_FCR * 4);
1218
1219         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1220
1221         /* DLAB = 1 */
1222         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1223
1224         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1225         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1226
1227         writeb(lcr, uart + UART_LCR * 4);
1228 }
1229
1230 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1231         .suspend = tegra_debug_uart_suspend,
1232         .resume = tegra_debug_uart_resume,
1233 };
1234
1235 struct clk *debug_uart_clk = NULL;
1236 EXPORT_SYMBOL(debug_uart_clk);
1237
1238 void tegra_console_uart_suspend(void)
1239 {
1240         if (console_suspend_enabled && debug_uart_clk)
1241                 clk_disable(debug_uart_clk);
1242 }
1243
1244 void tegra_console_uart_resume(void)
1245 {
1246         if (console_suspend_enabled && debug_uart_clk)
1247                 clk_enable(debug_uart_clk);
1248 }
1249
1250 static int tegra_debug_uart_syscore_init(void)
1251 {
1252         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1253         return 0;
1254 }
1255 arch_initcall(tegra_debug_uart_syscore_init);