rtc: tps80031: register as mfd sub device
[linux-2.6.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2012, NVIDIA Corporation. All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/earlysuspend.h>
37 #include <linux/slab.h>
38 #include <linux/serial_reg.h>
39 #include <linux/seq_file.h>
40 #include <linux/uaccess.h>
41 #include <linux/syscore_ops.h>
42 #include <linux/vmalloc.h>
43 #include <linux/memblock.h>
44 #include <linux/console.h>
45 #include <linux/pm_qos_params.h>
46 #include <linux/tegra_audio.h>
47
48 #include <trace/events/power.h>
49
50 #include <asm/cacheflush.h>
51 #include <asm/cpu_pm.h>
52 #include <asm/hardware/gic.h>
53 #include <asm/localtimer.h>
54 #include <asm/pgalloc.h>
55 #include <asm/pgtable.h>
56 #include <asm/tlbflush.h>
57
58 #include <mach/clk.h>
59 #include <mach/iomap.h>
60 #include <mach/irqs.h>
61 #include <mach/powergate.h>
62
63 #include "board.h"
64 #include "clock.h"
65 #include "cpuidle.h"
66 #include "fuse.h"
67 #include "gic.h"
68 #include "pm.h"
69 #include "pm-irq.h"
70 #include "reset.h"
71 #include "sleep.h"
72 #include "timer.h"
73 #include "dvfs.h"
74 #include "cpu-tegra.h"
75
76 struct suspend_context {
77         /*
78          * The next 7 values are referenced by offset in __restart_plls
79          * in headsmp-t2.S, and should not be moved
80          */
81         u32 pllx_misc;
82         u32 pllx_base;
83         u32 pllp_misc;
84         u32 pllp_base;
85         u32 pllp_outa;
86         u32 pllp_outb;
87         u32 pll_timeout;
88
89         u32 cpu_burst;
90         u32 clk_csite_src;
91         u32 cclk_divider;
92
93         u32 mc[3];
94         u8 uart[5];
95
96         struct tegra_twd_context twd;
97 };
98
99 #ifdef CONFIG_PM_SLEEP
100 #if USE_TEGRA_CPU_SUSPEND
101 void *tegra_cpu_context;        /* non-cacheable page for CPU context */
102 #endif
103 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
104 static pgd_t *tegra_pgd;
105 static DEFINE_SPINLOCK(tegra_lp2_lock);
106 static cpumask_t tegra_in_lp2;
107 static cpumask_t *iram_cpu_lp2_mask;
108 static unsigned long *iram_cpu_lp1_mask;
109 static u8 *iram_save;
110 static unsigned long iram_save_size;
111 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
112 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
113 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
114 static int tegra_last_pclk;
115 #endif
116
117 struct suspend_context tegra_sctx;
118
119 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
120 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
121 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
122 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
123 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
124 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
125 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
126 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
127
128 #define PMC_CTRL                0x0
129 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
130 #define PMC_WAKE_MASK           0xc
131 #define PMC_WAKE_LEVEL          0x10
132 #define PMC_DPAD_ORIDE          0x1C
133 #define PMC_WAKE_DELAY          0xe0
134 #define PMC_DPD_SAMPLE          0x20
135
136 #define PMC_WAKE_STATUS         0x14
137 #define PMC_SW_WAKE_STATUS      0x18
138 #define PMC_COREPWRGOOD_TIMER   0x3c
139 #define PMC_CPUPWRGOOD_TIMER    0xc8
140 #define PMC_CPUPWROFF_TIMER     0xcc
141 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
142
143 #define PMC_PWRGATE_TOGGLE      0x30
144 #define PWRGATE_TOGGLE_START    (1 << 8)
145 #define UN_PWRGATE_CPU          \
146         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
147
148 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
149 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
150 #endif
151
152 #define CLK_RESET_CCLK_BURST    0x20
153 #define CLK_RESET_CCLK_DIVIDER  0x24
154 #define CLK_RESET_PLLC_BASE     0x80
155 #define CLK_RESET_PLLM_BASE     0x90
156 #define CLK_RESET_PLLX_BASE     0xe0
157 #define CLK_RESET_PLLX_MISC     0xe4
158 #define CLK_RESET_PLLP_BASE     0xa0
159 #define CLK_RESET_PLLP_OUTA     0xa4
160 #define CLK_RESET_PLLP_OUTB     0xa8
161 #define CLK_RESET_PLLP_MISC     0xac
162
163 #define CLK_RESET_SOURCE_CSITE  0x1d4
164
165 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
166 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
167 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
168 #define CLK_RESET_CCLK_IDLE_POLICY         1
169 #define CLK_RESET_CCLK_RUN_POLICY          2
170 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
171 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
172
173 #define EMC_MRW_0               0x0e8
174 #define EMC_MRW_DEV_SELECTN     30
175 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
176
177 #define MC_SECURITY_START       0x6c
178 #define MC_SECURITY_SIZE        0x70
179 #define MC_SECURITY_CFG2        0x7c
180
181 #define AWAKE_CPU_FREQ_MIN      51000
182 static struct pm_qos_request_list awake_cpu_freq_req;
183
184 struct dvfs_rail *tegra_cpu_rail;
185 static struct dvfs_rail *tegra_core_rail;
186 static struct clk *tegra_pclk;
187 static const struct tegra_suspend_platform_data *pdata;
188 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
189
190 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
191 enum tegra_cluster_switch_time_id {
192         tegra_cluster_switch_time_id_start = 0,
193         tegra_cluster_switch_time_id_prolog,
194         tegra_cluster_switch_time_id_switch,
195         tegra_cluster_switch_time_id_epilog,
196         tegra_cluster_switch_time_id_max
197 };
198
199 static unsigned long
200                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
201 #define tegra_cluster_switch_time(flags, id) \
202         do { \
203                 barrier(); \
204                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
205                         void __iomem *timer_us = \
206                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
207                         if (id < tegra_cluster_switch_time_id_max) \
208                                 tegra_cluster_switch_times[id] = \
209                                                         readl(timer_us); \
210                                 wmb(); \
211                 } \
212                 barrier(); \
213         } while(0)
214 #else
215 #define tegra_cluster_switch_time(flags, id) do {} while(0)
216 #endif
217
218 #ifdef CONFIG_PM_SLEEP
219 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
220         [TEGRA_SUSPEND_NONE]    = "none",
221         [TEGRA_SUSPEND_LP2]     = "lp2",
222         [TEGRA_SUSPEND_LP1]     = "lp1",
223         [TEGRA_SUSPEND_LP0]     = "lp0",
224 };
225
226 unsigned long tegra_cpu_power_good_time(void)
227 {
228         if (WARN_ON_ONCE(!pdata))
229                 return 5000;
230
231         return pdata->cpu_timer;
232 }
233
234 unsigned long tegra_cpu_power_off_time(void)
235 {
236         if (WARN_ON_ONCE(!pdata))
237                 return 5000;
238
239         return pdata->cpu_off_timer;
240 }
241
242 unsigned long tegra_cpu_lp2_min_residency(void)
243 {
244         if (WARN_ON_ONCE(!pdata))
245                 return 2000;
246
247         return pdata->cpu_lp2_min_residency;
248 }
249
250 /*
251  * create_suspend_pgtable
252  *
253  * Creates a page table with identity mappings of physical memory and IRAM
254  * for use when the MMU is off, in addition to all the regular kernel mappings.
255  */
256 static __init int create_suspend_pgtable(void)
257 {
258         tegra_pgd = pgd_alloc(&init_mm);
259         if (!tegra_pgd)
260                 return -ENOMEM;
261
262         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
263         identity_mapping_add(tegra_pgd, PLAT_PHYS_OFFSET,
264                 PLAT_PHYS_OFFSET + (unsigned long)high_memory - PAGE_OFFSET);
265         identity_mapping_add(tegra_pgd, IO_IRAM_PHYS,
266                 IO_IRAM_PHYS + SECTION_SIZE);
267
268         /* inner/outer write-back/write-allocate, sharable */
269         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
270
271         return 0;
272 }
273
274 /*
275  * alloc_suspend_context
276  *
277  * Allocate a non-cacheable page to hold the CPU contexts.
278  * The standard ARM CPU context save functions don't work if there's
279  * an external L2 cache controller (like a PL310) in system.
280  */
281 static __init int alloc_suspend_context(void)
282 {
283 #if USE_TEGRA_CPU_SUSPEND
284         pgprot_t prot = __pgprot_modify(pgprot_kernel, L_PTE_MT_MASK,
285                                         L_PTE_MT_BUFFERABLE | L_PTE_XN);
286         struct page *ctx_page;
287         unsigned long ctx_virt = 0;
288         pgd_t *pgd;
289         pmd_t *pmd;
290         pte_t *pte;
291
292         ctx_page = alloc_pages(GFP_KERNEL, 0);
293         if (IS_ERR_OR_NULL(ctx_page))
294                 goto fail;
295
296         tegra_cpu_context = vm_map_ram(&ctx_page, 1, -1, prot);
297         if (IS_ERR_OR_NULL(tegra_cpu_context))
298                 goto fail;
299
300         /* Add the context page to our private pgd. */
301         ctx_virt = (unsigned long)tegra_cpu_context;
302
303         pgd = tegra_pgd + pgd_index(ctx_virt);
304         if (!pgd_present(*pgd))
305                 goto fail;
306         pmd = pmd_offset(pgd, ctx_virt);
307         if (!pmd_none(*pmd))
308                 goto fail;
309         pte = pte_alloc_kernel(pmd, ctx_virt);
310         if (!pte)
311                 goto fail;
312
313         set_pte_ext(pte, mk_pte(ctx_page, prot), 0);
314
315         outer_clean_range(__pa(pmd), __pa(pmd + 1));
316
317         return 0;
318
319 fail:
320         if (ctx_page)
321                 __free_page(ctx_page);
322         if (ctx_virt)
323                 vm_unmap_ram((void*)ctx_virt, 1);
324         tegra_cpu_context = NULL;
325         return -ENOMEM;
326 #else
327         return 0;
328 #endif
329 }
330
331 /* ensures that sufficient time is passed for a register write to
332  * serialize into the 32KHz domain */
333 static void pmc_32kwritel(u32 val, unsigned long offs)
334 {
335         writel(val, pmc + offs);
336         udelay(130);
337 }
338
339 static void set_power_timers(unsigned long us_on, unsigned long us_off,
340                              long rate)
341 {
342         static unsigned long last_us_off = 0;
343         unsigned long long ticks;
344         unsigned long long pclk;
345
346         if (WARN_ON_ONCE(rate <= 0))
347                 pclk = 100000000;
348         else
349                 pclk = rate;
350
351         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
352                 ticks = (us_on * pclk) + 999999ull;
353                 do_div(ticks, 1000000);
354                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
355
356                 ticks = (us_off * pclk) + 999999ull;
357                 do_div(ticks, 1000000);
358                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
359                 wmb();
360         }
361         tegra_last_pclk = pclk;
362         last_us_off = us_off;
363 }
364
365 /*
366  * restore_cpu_complex
367  *
368  * restores cpu clock setting, clears flow controller
369  *
370  * Always called on CPU 0.
371  */
372 static void restore_cpu_complex(u32 mode)
373 {
374         int cpu = smp_processor_id();
375         unsigned int reg, policy;
376
377         BUG_ON(cpu != 0);
378
379         /* Is CPU complex already running on PLLX? */
380         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
381         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
382
383         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
384                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
385         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
386                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
387         else
388                 BUG();
389
390         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
391                 /* restore PLLX settings if CPU is on different PLL */
392                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
393                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
394
395                 /* wait for PLL stabilization if PLLX was enabled */
396                 if (tegra_sctx.pllx_base & (1<<30)) {
397 #if USE_PLL_LOCK_BITS
398                         /* Enable lock detector */
399                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
400                         reg |= 1<<18;
401                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
402                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
403                                  (1<<27)))
404                                 cpu_relax();
405
406                         udelay(PLL_POST_LOCK_DELAY);
407 #else
408                         udelay(300);
409 #endif
410                 }
411         }
412
413         /* Restore original burst policy setting for calls resulting from CPU
414            LP2 in idle or system suspend; keep cluster switch prolog setting
415            intact. */
416         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
417                 writel(tegra_sctx.cclk_divider, clk_rst +
418                        CLK_RESET_CCLK_DIVIDER);
419                 writel(tegra_sctx.cpu_burst, clk_rst +
420                        CLK_RESET_CCLK_BURST);
421         }
422
423         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
424
425         /* Do not power-gate CPU 0 when flow controlled */
426         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
427         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
428         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
429         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
430         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
431         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
432         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
433
434         /* If an immedidate cluster switch is being perfomed, restore the
435            local timer registers. For calls resulting from CPU LP2 in
436            idle or system suspend, the local timer was shut down and
437            timekeeping switched over to the global system timer. In this
438            case keep local timer disabled, and restore only periodic load. */
439         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
440                       TEGRA_POWER_CLUSTER_IMMEDIATE)))
441                 tegra_sctx.twd.twd_ctrl = 0;
442         tegra_twd_resume(&tegra_sctx.twd);
443 }
444
445 /*
446  * suspend_cpu_complex
447  *
448  * saves pll state for use by restart_plls, prepares flow controller for
449  * transition to suspend state
450  *
451  * Must always be called on cpu 0.
452  */
453 static void suspend_cpu_complex(u32 mode)
454 {
455         int cpu = smp_processor_id();
456         unsigned int reg;
457         int i;
458
459         BUG_ON(cpu != 0);
460
461         /* switch coresite to clk_m, save off original source */
462         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
463         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
464
465         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
466         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
467         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
468         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
469         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
470         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
471         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
472         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
473
474         tegra_twd_suspend(&tegra_sctx.twd);
475
476         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
477         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
478         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
479         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
480         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
481 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
482         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
483 #else
484         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
485 #endif
486         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
487         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
488
489         for (i = 0; i < num_possible_cpus(); i++) {
490                 if (i == cpu)
491                         continue;
492                 reg = readl(FLOW_CTRL_CPU_CSR(i));
493                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
494                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
495                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
496         }
497
498         tegra_gic_cpu_disable();
499 #ifndef CONFIG_ARCH_TEGRA_2x_SOC
500         /* Tegra3 enters LPx states via WFI - do not propagate legacy IRQs
501            to CPU core to avoid fall through WFI (IRQ-to-flow controller wake
502            path is not affected). */
503         tegra_gic_pass_through_disable();
504 #endif
505 }
506
507 void tegra_clear_cpu_in_lp2(int cpu)
508 {
509         spin_lock(&tegra_lp2_lock);
510         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
511         cpumask_clear_cpu(cpu, &tegra_in_lp2);
512
513         /* Update the IRAM copy used by the reset handler. The IRAM copy
514            can't use used directly by cpumask_clear_cpu() because it uses
515            LDREX/STREX which requires the addressed location to be inner
516            cacheable and sharable which IRAM isn't. */
517         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
518         dsb();
519
520         spin_unlock(&tegra_lp2_lock);
521 }
522
523 bool tegra_set_cpu_in_lp2(int cpu)
524 {
525         bool last_cpu = false;
526
527         spin_lock(&tegra_lp2_lock);
528         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
529         cpumask_set_cpu(cpu, &tegra_in_lp2);
530
531         /* Update the IRAM copy used by the reset handler. The IRAM copy
532            can't use used directly by cpumask_set_cpu() because it uses
533            LDREX/STREX which requires the addressed location to be inner
534            cacheable and sharable which IRAM isn't. */
535         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
536         dsb();
537
538         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
539                 last_cpu = true;
540 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
541         else if (cpu == 1)
542                 tegra2_cpu_set_resettable_soon();
543 #endif
544
545         spin_unlock(&tegra_lp2_lock);
546         return last_cpu;
547 }
548
549 bool tegra_is_cpu_in_lp2(int cpu)
550 {
551         bool in_lp2;
552
553         spin_lock(&tegra_lp2_lock);
554         in_lp2 = cpumask_test_cpu(cpu, &tegra_in_lp2);
555         spin_unlock(&tegra_lp2_lock);
556         return in_lp2;
557 }
558
559 static void tegra_sleep_core(enum tegra_suspend_mode mode,
560                              unsigned long v2p)
561 {
562 #ifdef CONFIG_TRUSTED_FOUNDATIONS
563         if (mode == TEGRA_SUSPEND_LP0) {
564                 tegra_generic_smc_uncached(0xFFFFFFFC, 0xFFFFFFE3,
565                                            virt_to_phys(tegra_resume));
566         } else {
567                 tegra_generic_smc_uncached(0xFFFFFFFC, 0xFFFFFFE6,
568                                            (TEGRA_RESET_HANDLER_BASE +
569                                             tegra_cpu_reset_handler_offset));
570         }
571 #endif
572 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
573         tegra2_sleep_core(v2p);
574 #else
575         tegra3_sleep_core(v2p);
576 #endif
577 }
578
579 static inline void tegra_sleep_cpu(unsigned long v2p)
580 {
581 #ifdef CONFIG_TRUSTED_FOUNDATIONS
582         tegra_generic_smc_uncached(0xFFFFFFFC, 0xFFFFFFE4,
583                                    (TEGRA_RESET_HANDLER_BASE +
584                                     tegra_cpu_reset_handler_offset));
585 #endif
586         tegra_sleep_cpu_save(v2p);
587 }
588
589 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
590 {
591         u32 mode;       /* hardware + software power mode flags */
592         unsigned int remain;
593         pgd_t *pgd;
594
595         /* Only the last cpu down does the final suspend steps */
596         mode = readl(pmc + PMC_CTRL);
597         mode |= TEGRA_POWER_CPU_PWRREQ_OE;
598         if (pdata->combined_req)
599                 mode &= ~TEGRA_POWER_PWRREQ_OE;
600         else
601                 mode |= TEGRA_POWER_PWRREQ_OE;
602         mode &= ~TEGRA_POWER_EFFECT_LP0;
603         writel(mode, pmc + PMC_CTRL);
604
605         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
606
607         /*
608          * We can use clk_get_rate_all_locked() here, because all other cpus
609          * are in LP2 state and irqs are disabled
610          */
611         if (flags & TEGRA_POWER_CLUSTER_MASK) {
612                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
613                 set_power_timers(pdata->cpu_timer, 0,
614                         clk_get_rate_all_locked(tegra_pclk));
615                 if (flags & TEGRA_POWER_CLUSTER_G) {
616                         /*
617                          * To reduce the vdd_cpu up latency when LP->G
618                          * transition. Before the transition, enable
619                          * the vdd_cpu rail.
620                          */
621                         if (is_lp_cluster())
622                                 writel(UN_PWRGATE_CPU,
623                                        pmc + PMC_PWRGATE_TOGGLE);
624                 }
625                 tegra_cluster_switch_prolog(flags);
626         } else {
627                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
628                         clk_get_rate_all_locked(tegra_pclk));
629         }
630
631         if (sleep_time)
632                 tegra_lp2_set_trigger(sleep_time);
633
634         cpu_complex_pm_enter();
635         suspend_cpu_complex(flags);
636         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
637         flush_cache_all();
638         /*
639          * No need to flush complete L2. Cleaning kernel and IO mappings
640          * is enough for the LP code sequence that has L2 disabled but
641          * MMU on.
642          */
643         pgd = cpu_get_pgd();
644         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
645                           __pa(pgd + PTRS_PER_PGD));
646         outer_disable();
647
648         tegra_sleep_cpu(PLAT_PHYS_OFFSET - PAGE_OFFSET);
649
650         tegra_init_cache(false);
651         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
652         restore_cpu_complex(flags);
653         cpu_complex_pm_exit();
654
655         remain = tegra_lp2_timer_remain();
656         if (sleep_time)
657                 tegra_lp2_set_trigger(0);
658
659         if (flags & TEGRA_POWER_CLUSTER_MASK) {
660                 tegra_cluster_switch_epilog(flags);
661                 trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
662         }
663         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
664
665 #if INSTRUMENT_CLUSTER_SWITCH
666         if (flags & TEGRA_POWER_CLUSTER_MASK) {
667                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
668                         is_lp_cluster() ? "G=>LP" : "LP=>G",
669                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
670                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
671                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
672                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
673                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
674                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
675                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
676                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
677         }
678 #endif
679         return remain;
680 }
681
682 static int tegra_common_suspend(void)
683 {
684         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
685
686         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
687         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
688         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
689
690         /* copy the reset vector and SDRAM shutdown code into IRAM */
691         memcpy(iram_save, iram_code, iram_save_size);
692         memcpy(iram_code, tegra_iram_start(), iram_save_size);
693
694         return 0;
695 }
696
697 static void tegra_common_resume(void)
698 {
699         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
700 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
701         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
702 #endif
703
704         /* Clear DPD sample */
705         writel(0x0, pmc + PMC_DPD_SAMPLE);
706
707         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
708         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
709         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
710 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
711         /* trigger emc mode write */
712         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
713 #endif
714         /* clear scratch registers shared by suspend and the reset pen */
715         writel(0x0, pmc + PMC_SCRATCH39);
716         writel(0x0, pmc + PMC_SCRATCH41);
717
718         /* restore IRAM */
719         memcpy(iram_code, iram_save, iram_save_size);
720 }
721
722 static int tegra_suspend_prepare_late(void)
723 {
724 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
725         disable_irq(INT_SYS_STATS_MON);
726 #endif
727         return 0;
728 }
729
730 static void tegra_suspend_wake(void)
731 {
732 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
733         enable_irq(INT_SYS_STATS_MON);
734 #endif
735 }
736
737 static void tegra_pm_set(enum tegra_suspend_mode mode)
738 {
739         u32 reg, boot_flag;
740         unsigned long rate = 32768;
741
742         reg = readl(pmc + PMC_CTRL);
743         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
744         if (pdata->combined_req)
745                 reg &= ~TEGRA_POWER_PWRREQ_OE;
746         else
747                 reg |= TEGRA_POWER_PWRREQ_OE;
748         reg &= ~TEGRA_POWER_EFFECT_LP0;
749
750         switch (mode) {
751         case TEGRA_SUSPEND_LP0:
752 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
753                 rate = clk_get_rate_all_locked(tegra_pclk);
754 #endif
755                 if (pdata->combined_req) {
756                         reg |= TEGRA_POWER_PWRREQ_OE;
757                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
758                 }
759
760                 /*
761                  * LP0 boots through the AVP, which then resumes the AVP to
762                  * the address in scratch 39, and the cpu to the address in
763                  * scratch 41 to tegra_resume
764                  */
765                 writel(0x0, pmc + PMC_SCRATCH39);
766
767                 /* Enable DPD sample to trigger sampling pads data and direction
768                  * in which pad will be driven during lp0 mode*/
769                 writel(0x1, pmc + PMC_DPD_SAMPLE);
770
771                 /* Set warmboot flag */
772                 boot_flag = readl(pmc + PMC_SCRATCH0);
773                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
774
775                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
776
777                 reg |= TEGRA_POWER_EFFECT_LP0;
778                 /* No break here. LP0 code falls through to write SCRATCH41 */
779         case TEGRA_SUSPEND_LP1:
780                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
781                 wmb();
782                 break;
783         case TEGRA_SUSPEND_LP2:
784                 rate = clk_get_rate(tegra_pclk);
785                 break;
786         case TEGRA_SUSPEND_NONE:
787                 return;
788         default:
789                 BUG();
790         }
791
792         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
793
794         pmc_32kwritel(reg, PMC_CTRL);
795 }
796
797 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
798         [TEGRA_SUSPEND_NONE] = "none",
799         [TEGRA_SUSPEND_LP2] = "LP2",
800         [TEGRA_SUSPEND_LP1] = "LP1",
801         [TEGRA_SUSPEND_LP0] = "LP0",
802 };
803
804 static int tegra_suspend_enter(suspend_state_t state)
805 {
806         int ret;
807         ktime_t delta;
808         struct timespec ts_entry, ts_exit;
809
810         if (pdata && pdata->board_suspend)
811                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
812
813         read_persistent_clock(&ts_entry);
814
815         ret = tegra_suspend_dram(current_suspend_mode, 0);
816         if (ret) {
817                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
818                 goto abort_suspend;
819         }
820
821         read_persistent_clock(&ts_exit);
822
823         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
824                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
825
826                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
827                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
828                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
829                 else
830                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
831         }
832
833 abort_suspend:
834         if (pdata && pdata->board_resume)
835                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
836
837         return ret;
838 }
839
840 static void tegra_suspend_check_pwr_stats(void)
841 {
842         /* cpus and l2 are powered off later */
843         unsigned long pwrgate_partid_mask =
844 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
845                 (1 << TEGRA_POWERGATE_HEG)      |
846                 (1 << TEGRA_POWERGATE_SATA)     |
847                 (1 << TEGRA_POWERGATE_3D1)      |
848 #endif
849                 (1 << TEGRA_POWERGATE_3D)       |
850                 (1 << TEGRA_POWERGATE_VENC)     |
851                 (1 << TEGRA_POWERGATE_PCIE)     |
852                 (1 << TEGRA_POWERGATE_VDEC)     |
853                 (1 << TEGRA_POWERGATE_MPE);
854
855         int partid;
856
857         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
858                 if ((1 << partid) & pwrgate_partid_mask)
859                         if (tegra_powergate_is_powered(partid))
860                                 pr_warning("partition %s is left on before suspend\n",
861                                         tegra_powergate_get_name(partid));
862
863         return;
864 }
865
866 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
867 {
868         int err = 0;
869         u32 scratch37 = 0xDEADBEEF;
870
871         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
872                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
873                 err = -ENXIO;
874                 goto fail;
875         }
876
877         if (tegra_is_voice_call_active()) {
878                 u32 reg;
879
880                 /* backup the current value of scratch37 */
881                 scratch37 = readl(pmc + PMC_SCRATCH37);
882
883                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
884                 reg = TEGRA_POWER_LP1_AUDIO;
885                 pmc_32kwritel(reg, PMC_SCRATCH37);
886         }
887
888         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
889                 pr_info("LP0 not used due to unsupported wakeup events\n");
890                 mode = TEGRA_SUSPEND_LP1;
891         }
892
893         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
894                 tegra_suspend_check_pwr_stats();
895
896         tegra_common_suspend();
897
898         tegra_pm_set(mode);
899
900         if (pdata && pdata->board_suspend)
901                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
902
903         local_fiq_disable();
904
905         trace_cpu_suspend(CPU_SUSPEND_START);
906
907         cpu_pm_enter();
908         cpu_complex_pm_enter();
909
910         if (mode == TEGRA_SUSPEND_LP0) {
911 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
912                 u32 reg = readl(pmc + PMC_SCRATCH4);
913                 if (is_lp_cluster())
914                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
915                 else
916                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
917                 pmc_32kwritel(reg, PMC_SCRATCH4);
918 #endif
919                 tegra_lp0_suspend_mc();
920                 tegra_cpu_reset_handler_save();
921
922         }
923         else if (mode == TEGRA_SUSPEND_LP1)
924                 *iram_cpu_lp1_mask = 1;
925
926         suspend_cpu_complex(flags);
927
928         flush_cache_all();
929         outer_flush_all();
930         outer_disable();
931
932         if (mode == TEGRA_SUSPEND_LP2)
933                 tegra_sleep_cpu(PLAT_PHYS_OFFSET - PAGE_OFFSET);
934         else
935                 tegra_sleep_core(mode, PLAT_PHYS_OFFSET - PAGE_OFFSET);
936
937         tegra_init_cache(true);
938
939         if (mode == TEGRA_SUSPEND_LP0) {
940                 tegra_cpu_reset_handler_restore();
941                 tegra_lp0_resume_mc();
942         } else if (mode == TEGRA_SUSPEND_LP1)
943                 *iram_cpu_lp1_mask = 0;
944
945         /* if scratch37 was clobbered during LP1, restore it */
946         if (scratch37 != 0xDEADBEEF)
947                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
948
949         restore_cpu_complex(flags);
950
951         /* for platforms where the core & CPU power requests are
952          * combined as a single request to the PMU, transition out
953          * of LP0 state by temporarily enabling both requests
954          */
955         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
956                 u32 reg;
957                 reg = readl(pmc + PMC_CTRL);
958                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
959                 pmc_32kwritel(reg, PMC_CTRL);
960                 reg &= ~TEGRA_POWER_PWRREQ_OE;
961                 pmc_32kwritel(reg, PMC_CTRL);
962         }
963
964         cpu_complex_pm_exit();
965         cpu_pm_exit();
966
967         if (pdata && pdata->board_resume)
968                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
969
970         trace_cpu_suspend(CPU_SUSPEND_DONE);
971
972         local_fiq_enable();
973
974         tegra_common_resume();
975
976 fail:
977         return err;
978 }
979
980 /*
981  * Function pointers to optional board specific function
982  */
983 void (*tegra_deep_sleep)(int);
984 EXPORT_SYMBOL(tegra_deep_sleep);
985
986 static int tegra_suspend_prepare(void)
987 {
988         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
989                 tegra_deep_sleep(1);
990         return 0;
991 }
992
993 static void tegra_suspend_finish(void)
994 {
995         if (pdata && pdata->cpu_resume_boost) {
996                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
997                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
998                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
999         }
1000
1001         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1002                 tegra_deep_sleep(0);
1003 }
1004
1005 static const struct platform_suspend_ops tegra_suspend_ops = {
1006         .valid          = suspend_valid_only_mem,
1007         .prepare        = tegra_suspend_prepare,
1008         .finish         = tegra_suspend_finish,
1009         .prepare_late   = tegra_suspend_prepare_late,
1010         .wake           = tegra_suspend_wake,
1011         .enter          = tegra_suspend_enter,
1012 };
1013
1014 static ssize_t suspend_mode_show(struct kobject *kobj,
1015                                         struct kobj_attribute *attr, char *buf)
1016 {
1017         char *start = buf;
1018         char *end = buf + PAGE_SIZE;
1019
1020         start += scnprintf(start, end - start, "%s ", \
1021                                 tegra_suspend_name[current_suspend_mode]);
1022         start += scnprintf(start, end - start, "\n");
1023
1024         return start - buf;
1025 }
1026
1027 static ssize_t suspend_mode_store(struct kobject *kobj,
1028                                         struct kobj_attribute *attr,
1029                                         const char *buf, size_t n)
1030 {
1031         int len;
1032         const char *name_ptr;
1033         enum tegra_suspend_mode new_mode;
1034
1035         name_ptr = buf;
1036         while (*name_ptr && !isspace(*name_ptr))
1037                 name_ptr++;
1038         len = name_ptr - buf;
1039         if (!len)
1040                 goto bad_name;
1041         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1042         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1043                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1044                 pr_info("Illegal tegra suspend state: %s\n", buf);
1045                 goto bad_name;
1046         }
1047
1048         for (new_mode = TEGRA_SUSPEND_NONE; \
1049                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1050                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1051                         current_suspend_mode = new_mode;
1052                         break;
1053                 }
1054         }
1055
1056 bad_name:
1057         return n;
1058 }
1059
1060 static struct kobj_attribute suspend_mode_attribute =
1061         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1062
1063 static struct kobject *suspend_kobj;
1064
1065 static int tegra_pm_enter_suspend(void)
1066 {
1067         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1068         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1069                 tegra_lp0_cpu_mode(true);
1070         return 0;
1071 }
1072
1073 static void tegra_pm_enter_resume(void)
1074 {
1075         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1076                 tegra_lp0_cpu_mode(false);
1077         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1078 }
1079
1080 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1081         .suspend = tegra_pm_enter_suspend,
1082         .resume = tegra_pm_enter_resume,
1083 };
1084
1085 static __init int tegra_pm_enter_syscore_init(void)
1086 {
1087         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1088         return 0;
1089 }
1090 subsys_initcall(tegra_pm_enter_syscore_init);
1091 #endif
1092
1093 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1094 {
1095         u32 reg;
1096         u32 mode;
1097
1098         tegra_cpu_rail = tegra_dvfs_get_rail_by_name("vdd_cpu");
1099         tegra_core_rail = tegra_dvfs_get_rail_by_name("vdd_core");
1100         pm_qos_add_request(&awake_cpu_freq_req, PM_QOS_CPU_FREQ_MIN,
1101                            AWAKE_CPU_FREQ_MIN);
1102
1103         tegra_pclk = clk_get_sys(NULL, "pclk");
1104         BUG_ON(IS_ERR(tegra_pclk));
1105         pdata = plat;
1106         (void)reg;
1107         (void)mode;
1108
1109         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1110                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1111
1112 #ifndef CONFIG_PM_SLEEP
1113         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1114                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1115                            "disabling suspend\n", __func__);
1116                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1117         }
1118 #else
1119         if (create_suspend_pgtable() < 0) {
1120                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1121                                 __func__);
1122                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1123                 goto fail;
1124         }
1125
1126         if (alloc_suspend_context() < 0) {
1127                 pr_err("%s: CPU context alloc failed -- LP0/LP1/LP2 unavailable\n",
1128                                 __func__);
1129                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1130                 goto fail;
1131         }
1132
1133         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1134             (tegra_get_revision() == TEGRA_REVISION_A01) &&
1135             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1136                 /* Tegra 3 A01 supports only LP1 */
1137                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1138                            "-- disabling LP0\n", __func__);
1139                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1140         }
1141         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1142                 tegra_lp0_vec_relocate) {
1143                 unsigned char *reloc_lp0;
1144                 unsigned long tmp;
1145                 void __iomem *orig;
1146                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1147                                         GFP_KERNEL);
1148                 WARN_ON(!reloc_lp0);
1149                 if (!reloc_lp0) {
1150                         pr_err("%s: Failed to allocate reloc_lp0\n",
1151                                 __func__);
1152                         goto out;
1153                 }
1154
1155                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1156                 WARN_ON(!orig);
1157                 if (!orig) {
1158                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1159                                 __func__, tegra_lp0_vec_start);
1160                         kfree(reloc_lp0);
1161                         goto out;
1162                 }
1163
1164                 tmp = (unsigned long) reloc_lp0;
1165                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1166                 reloc_lp0 = (unsigned char *)tmp;
1167                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1168                 iounmap(orig);
1169                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1170         }
1171
1172 out:
1173         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1174                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1175                            "provided by bootlader -- disabling LP0\n",
1176                            __func__);
1177                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1178         }
1179
1180         iram_save_size = tegra_iram_end() - tegra_iram_start();
1181
1182         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1183         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1184                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1185                        "-- LP0/LP1 unavailable\n", __func__);
1186                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1187         }
1188
1189         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1190         /* Initialize scratch registers used for CPU LP2 synchronization */
1191         writel(0, pmc + PMC_SCRATCH37);
1192         writel(0, pmc + PMC_SCRATCH38);
1193         writel(0, pmc + PMC_SCRATCH39);
1194         writel(0, pmc + PMC_SCRATCH41);
1195
1196         /* Always enable CPU power request; just normal polarity is supported */
1197         reg = readl(pmc + PMC_CTRL);
1198         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1199         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1200         pmc_32kwritel(reg, PMC_CTRL);
1201
1202         /* Configure core power request and system clock control if LP0
1203            is supported */
1204         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1205         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1206
1207         reg = readl(pmc + PMC_CTRL);
1208
1209         if (!pdata->sysclkreq_high)
1210                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1211         else
1212                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1213
1214         if (!pdata->corereq_high)
1215                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1216         else
1217                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1218
1219         /* configure output inverters while the request is tristated */
1220         pmc_32kwritel(reg, PMC_CTRL);
1221
1222         /* now enable requests */
1223         reg |= TEGRA_POWER_SYSCLK_OE;
1224         if (!pdata->combined_req)
1225                 reg |= TEGRA_POWER_PWRREQ_OE;
1226         pmc_32kwritel(reg, PMC_CTRL);
1227
1228         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1229                 tegra_lp0_suspend_init();
1230
1231         suspend_set_ops(&tegra_suspend_ops);
1232
1233         /* Create /sys/power/suspend/type */
1234         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1235         if (suspend_kobj) {
1236                 if (sysfs_create_file(suspend_kobj, \
1237                                                 &suspend_mode_attribute.attr))
1238                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1239                                                                 __func__);
1240         }
1241
1242         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1243         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1244
1245         /* clear io dpd settings before kernel */
1246         tegra_bl_io_dpd_cleanup();
1247
1248 fail:
1249 #endif
1250         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1251                 tegra_lp2_in_idle(false);
1252
1253         current_suspend_mode = plat->suspend_mode;
1254 }
1255
1256 unsigned long debug_uart_port_base = 0;
1257 EXPORT_SYMBOL(debug_uart_port_base);
1258
1259 static int tegra_debug_uart_suspend(void)
1260 {
1261         void __iomem *uart;
1262         u32 lcr;
1263
1264         if (!debug_uart_port_base)
1265                 return 0;
1266
1267         uart = IO_ADDRESS(debug_uart_port_base);
1268
1269         lcr = readb(uart + UART_LCR * 4);
1270
1271         tegra_sctx.uart[0] = lcr;
1272         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1273
1274         /* DLAB = 0 */
1275         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1276
1277         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1278
1279         /* DLAB = 1 */
1280         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1281
1282         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1283         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1284
1285         writeb(lcr, uart + UART_LCR * 4);
1286
1287         return 0;
1288 }
1289
1290 static void tegra_debug_uart_resume(void)
1291 {
1292         void __iomem *uart;
1293         u32 lcr;
1294
1295         if (!debug_uart_port_base)
1296                 return;
1297
1298         uart = IO_ADDRESS(debug_uart_port_base);
1299
1300         lcr = tegra_sctx.uart[0];
1301
1302         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1303
1304         /* DLAB = 0 */
1305         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1306
1307         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1308                         uart + UART_FCR * 4);
1309
1310         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1311
1312         /* DLAB = 1 */
1313         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1314
1315         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1316         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1317
1318         writeb(lcr, uart + UART_LCR * 4);
1319 }
1320
1321 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1322         .suspend = tegra_debug_uart_suspend,
1323         .resume = tegra_debug_uart_resume,
1324 };
1325
1326 struct clk *debug_uart_clk = NULL;
1327 EXPORT_SYMBOL(debug_uart_clk);
1328
1329 void tegra_console_uart_suspend(void)
1330 {
1331         if (console_suspend_enabled && debug_uart_clk)
1332                 clk_disable(debug_uart_clk);
1333 }
1334
1335 void tegra_console_uart_resume(void)
1336 {
1337         if (console_suspend_enabled && debug_uart_clk)
1338                 clk_enable(debug_uart_clk);
1339 }
1340
1341 static int tegra_debug_uart_syscore_init(void)
1342 {
1343         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1344         return 0;
1345 }
1346 arch_initcall(tegra_debug_uart_syscore_init);
1347
1348 #ifdef CONFIG_HAS_EARLYSUSPEND
1349 static struct clk *clk_wake;
1350 static void pm_early_suspend(struct early_suspend *h)
1351 {
1352         if (clk_wake)
1353                 clk_disable(clk_wake);
1354         pm_qos_update_request(&awake_cpu_freq_req, PM_QOS_DEFAULT_VALUE);
1355 }
1356
1357 static void pm_late_resume(struct early_suspend *h)
1358 {
1359         if (clk_wake)
1360                 clk_enable(clk_wake);
1361         pm_qos_update_request(&awake_cpu_freq_req, (s32)AWAKE_CPU_FREQ_MIN);
1362 }
1363
1364 static struct early_suspend pm_early_suspender = {
1365                 .suspend = pm_early_suspend,
1366                 .resume = pm_late_resume,
1367 };
1368
1369 static int pm_init_wake_behavior(void)
1370 {
1371         clk_wake = tegra_get_clock_by_name("wake.sclk");
1372         register_early_suspend(&pm_early_suspender);
1373         return 0;
1374 }
1375
1376 late_initcall(pm_init_wake_behavior);
1377 #endif