9ad2b8bf6bcadcbb341b6a81aa757228b9407fee
[linux-3.10.git] / arch / arm / mach-tegra / pm.c
1 /*
2  * arch/arm/mach-tegra/pm.c
3  *
4  * CPU complex suspend & resume functions for Tegra SoCs
5  *
6  * Copyright (c) 2009-2013, NVIDIA CORPORATION.  All rights reserved.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/kernel.h>
24 #include <linux/ctype.h>
25 #include <linux/init.h>
26 #include <linux/io.h>
27 #include <linux/sched.h>
28 #include <linux/smp.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/clk.h>
32 #include <linux/err.h>
33 #include <linux/debugfs.h>
34 #include <linux/delay.h>
35 #include <linux/suspend.h>
36 #include <linux/slab.h>
37 #include <linux/serial_reg.h>
38 #include <linux/seq_file.h>
39 #include <linux/uaccess.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/cpu_pm.h>
42 #include <linux/clk/tegra.h>
43 #include <linux/export.h>
44 #include <linux/vmalloc.h>
45 #include <linux/memblock.h>
46 #include <linux/console.h>
47 #include <linux/tegra_audio.h>
48
49 #include <trace/events/power.h>
50 #include <trace/events/nvsecurity.h>
51
52 #include <asm/cacheflush.h>
53 #include <asm/idmap.h>
54 #include <asm/localtimer.h>
55 #include <asm/pgalloc.h>
56 #include <asm/pgtable.h>
57 #include <asm/tlbflush.h>
58 #include <asm/suspend.h>
59 #include <asm/smp_plat.h>
60
61 #include <mach/irqs.h>
62 #include <mach/powergate.h>
63 #include <mach/hardware.h>
64
65 #include "board.h"
66 #include "clock.h"
67 #include "common.h"
68 #include "cpuidle.h"
69 #include "fuse.h"
70 #include "gic.h"
71 #include "iomap.h"
72 #include "pm.h"
73 #include "pm-irq.h"
74 #include "reset.h"
75 #include "sleep.h"
76 #include "timer.h"
77 #include "dvfs.h"
78 #include "cpu-tegra.h"
79 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
80 #include "tegra14_scratch.h"
81 #endif
82
83 #define CREATE_TRACE_POINTS
84 #include <trace/events/nvpower.h>
85
86 struct suspend_context {
87         /*
88          * The next 7 values are referenced by offset in __restart_plls
89          * in headsmp-t2.S, and should not be moved
90          */
91         u32 pllx_misc;
92         u32 pllx_base;
93         u32 pllp_misc;
94         u32 pllp_base;
95         u32 pllp_outa;
96         u32 pllp_outb;
97         u32 pll_timeout;
98
99         u32 cpu_burst;
100         u32 clk_csite_src;
101         u32 cclk_divider;
102
103         u32 mc[3];
104         u8 uart[5];
105
106         struct tegra_twd_context twd;
107 };
108
109 #ifdef CONFIG_PM_SLEEP
110 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
111 static pgd_t *tegra_pgd;
112 static DEFINE_SPINLOCK(tegra_lp2_lock);
113 static cpumask_t tegra_in_lp2;
114 static cpumask_t *iram_cpu_lp2_mask;
115 static unsigned long *iram_cpu_lp1_mask;
116 static unsigned long *iram_mc_clk_mask;
117 static u8 *iram_save;
118 static unsigned long iram_save_size;
119 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
120 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
121 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
122 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
123 static void __iomem *tert_ictlr = IO_ADDRESS(TEGRA_TERTIARY_ICTLR_BASE);
124 #endif
125 static void __iomem *tmrus_reg_base = IO_ADDRESS(TEGRA_TMR1_BASE);
126 static int tegra_last_pclk;
127 static u64 resume_time;
128 static u64 resume_entry_time;
129 static u64 suspend_time;
130 static u64 suspend_entry_time;
131 #endif
132
133 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
134 static void update_pmc_registers(int instance);
135 #endif
136
137 struct suspend_context tegra_sctx;
138 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
139 extern struct device *get_se_device(void);
140 extern int se_suspend(struct device *dev, bool pooling);
141 extern struct device *get_smmu_device(void);
142 extern int tegra_smmu_resume(struct device *dev);
143 extern int tegra_smmu_suspend(struct device *dev);
144 #endif
145
146 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
147 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
148 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
149 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
150 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
151 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
152 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
153 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
154 #define TEGRA_POWER_CPUPWRGOOD_EN       (1 << 19)  /* CPU power good enable */
155
156 #define TEGRA_DPAD_ORIDE_SYS_CLK_REQ    (1 << 21)
157
158 #define PMC_CTRL                0x0
159 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
160 #define PMC_WAKE_MASK           0xc
161 #define PMC_WAKE_LEVEL          0x10
162 #define PMC_DPAD_ORIDE          0x1C
163 #define PMC_WAKE_DELAY          0xe0
164 #define PMC_DPD_SAMPLE          0x20
165 #if defined(CONFIG_ARCH_TEGRA_14x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
166 #define PMC_DPD_ENABLE          0x24
167 #endif
168 #define PMC_IO_DPD_REQ          0x1B8
169 #define PMC_IO_DPD2_REQ         0x1C0
170
171
172 #define PMC_WAKE_STATUS         0x14
173 #define PMC_SW_WAKE_STATUS      0x18
174 #define PMC_COREPWRGOOD_TIMER   0x3c
175 #define PMC_CPUPWRGOOD_TIMER    0xc8
176 #define PMC_CPUPWROFF_TIMER     0xcc
177 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
178
179 #define PMC_PWRGATE_TOGGLE      0x30
180 #define PWRGATE_TOGGLE_START    (1 << 8)
181 #define UN_PWRGATE_CPU          \
182         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
183
184 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
185 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
186 #endif
187
188 #define CLK_RESET_CCLK_BURST    0x20
189 #define CLK_RESET_CCLK_DIVIDER  0x24
190 #define CLK_RESET_PLLC_BASE     0x80
191 #define CLK_RESET_PLLM_BASE     0x90
192 #define CLK_RESET_PLLX_BASE     0xe0
193 #define CLK_RESET_PLLX_MISC     0xe4
194 #define CLK_RESET_PLLP_BASE     0xa0
195 #define CLK_RESET_PLLP_OUTA     0xa4
196 #define CLK_RESET_PLLP_OUTB     0xa8
197 #define CLK_RESET_PLLP_MISC     0xac
198
199 #define CLK_RESET_SOURCE_CSITE  0x1d4
200
201 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
202 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
203 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
204 #define CLK_RESET_CCLK_IDLE_POLICY         1
205 #define CLK_RESET_CCLK_RUN_POLICY          2
206 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
207 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
208
209 #define EMC_MRW_0               0x0e8
210 #define EMC_MRW_DEV_SELECTN     30
211 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
212
213 #define MC_SECURITY_START       0x6c
214 #define MC_SECURITY_SIZE        0x70
215 #define MC_SECURITY_CFG2        0x7c
216
217 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
218 static struct clk *tegra_dfll;
219 #endif
220 static struct clk *tegra_pclk;
221 static const struct tegra_suspend_platform_data *pdata;
222 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
223
224 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
225 enum tegra_cluster_switch_time_id {
226         tegra_cluster_switch_time_id_start = 0,
227         tegra_cluster_switch_time_id_prolog,
228         tegra_cluster_switch_time_id_switch,
229         tegra_cluster_switch_time_id_epilog,
230         tegra_cluster_switch_time_id_max
231 };
232
233 static unsigned long
234                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
235 #define tegra_cluster_switch_time(flags, id) \
236         do { \
237                 barrier(); \
238                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
239                         void __iomem *timer_us = \
240                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
241                         if (id < tegra_cluster_switch_time_id_max) \
242                                 tegra_cluster_switch_times[id] = \
243                                                         readl(timer_us); \
244                                 wmb(); \
245                 } \
246                 barrier(); \
247         } while(0)
248 #else
249 #define tegra_cluster_switch_time(flags, id) do {} while(0)
250 #endif
251
252 #ifdef CONFIG_PM_SLEEP
253 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
254         [TEGRA_SUSPEND_NONE]    = "none",
255         [TEGRA_SUSPEND_LP2]     = "lp2",
256         [TEGRA_SUSPEND_LP1]     = "lp1",
257         [TEGRA_SUSPEND_LP0]     = "lp0",
258 };
259
260 void tegra_log_resume_time(void)
261 {
262         u64 resume_end_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
263
264         if (resume_entry_time > resume_end_time)
265                 resume_end_time |= 1ull<<32;
266         resume_time = resume_end_time - resume_entry_time;
267 }
268
269 void tegra_log_suspend_time(void)
270 {
271         suspend_entry_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
272 }
273
274 static void tegra_get_suspend_time(void)
275 {
276         u64 suspend_end_time;
277         suspend_end_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
278
279         if (suspend_entry_time > suspend_end_time)
280                 suspend_end_time |= 1ull<<32;
281         suspend_time = suspend_end_time - suspend_entry_time;
282 }
283
284 unsigned long tegra_cpu_power_good_time(void)
285 {
286         if (WARN_ON_ONCE(!pdata))
287                 return 5000;
288
289         return pdata->cpu_timer;
290 }
291
292 unsigned long tegra_cpu_power_off_time(void)
293 {
294         if (WARN_ON_ONCE(!pdata))
295                 return 5000;
296
297         return pdata->cpu_off_timer;
298 }
299
300 unsigned long tegra_cpu_lp2_min_residency(void)
301 {
302         if (WARN_ON_ONCE(!pdata))
303                 return 2000;
304
305         return pdata->cpu_lp2_min_residency;
306 }
307
308 unsigned long tegra_mc_clk_stop_min_residency(void)
309 {
310         return 20000;
311 }
312
313 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
314 #define TEGRA_MIN_RESIDENCY_VMIN_FMIN   2000
315 #define TEGRA_MIN_RESIDENCY_NCPU_SLOW   2000
316 #define TEGRA_MIN_RESIDENCY_NCPU_FAST   13000
317 #define TEGRA_MIN_RESIDENCY_CRAIL       20000
318
319 unsigned long tegra_min_residency_vmin_fmin(void)
320 {
321         return pdata && pdata->min_residency_vmin_fmin
322                         ? pdata->min_residency_vmin_fmin
323                         : TEGRA_MIN_RESIDENCY_VMIN_FMIN;
324 }
325
326 unsigned long tegra_min_residency_ncpu()
327 {
328         if (is_lp_cluster()) {
329                 return pdata && pdata->min_residency_ncpu_slow
330                         ? pdata->min_residency_ncpu_slow
331                         : TEGRA_MIN_RESIDENCY_NCPU_SLOW;
332         } else
333                 return pdata && pdata->min_residency_ncpu_fast
334                         ? pdata->min_residency_ncpu_fast
335                         : TEGRA_MIN_RESIDENCY_NCPU_FAST;
336 }
337
338 unsigned long tegra_min_residency_crail(void)
339 {
340         return pdata && pdata->min_residency_crail
341                         ? pdata->min_residency_crail
342                         : TEGRA_MIN_RESIDENCY_CRAIL;
343 }
344 #endif
345
346 static void suspend_cpu_dfll_mode(void)
347 {
348 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
349         /* If DFLL is used as CPU clock source go to open loop mode */
350         if (!is_lp_cluster() && tegra_dfll &&
351             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
352                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 0);
353 #endif
354 }
355
356 static void resume_cpu_dfll_mode(void)
357 {
358 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
359         /* If DFLL is used as CPU clock source restore closed loop mode */
360         if (!is_lp_cluster() && tegra_dfll &&
361             tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail))
362                 tegra_clk_cfg_ex(tegra_dfll, TEGRA_CLK_DFLL_LOCK, 1);
363 #endif
364 }
365
366 /*
367  * create_suspend_pgtable
368  *
369  * Creates a page table with identity mappings of physical memory and IRAM
370  * for use when the MMU is off, in addition to all the regular kernel mappings.
371  */
372 static __init int create_suspend_pgtable(void)
373 {
374         tegra_pgd = pgd_alloc(&init_mm);
375         if (!tegra_pgd)
376                 return -ENOMEM;
377
378         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
379         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
380                 high_memory, 0);
381         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
382                 IO_IRAM_VIRT + SECTION_SIZE, 0);
383
384 #if defined(CONFIG_ARM_LPAE)
385         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK);
386 #else
387         /* inner/outer write-back/write-allocate, sharable */
388         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
389 #endif
390
391         return 0;
392 }
393
394 /* ensures that sufficient time is passed for a register write to
395  * serialize into the 32KHz domain */
396 static void pmc_32kwritel(u32 val, unsigned long offs)
397 {
398         writel(val, pmc + offs);
399         udelay(130);
400 }
401
402 static void set_power_timers(unsigned long us_on, unsigned long us_off,
403                              long rate)
404 {
405         static unsigned long last_us_off = 0;
406         unsigned long long ticks;
407         unsigned long long pclk;
408
409         if (WARN_ON_ONCE(rate <= 0))
410                 pclk = 100000000;
411         else
412                 pclk = rate;
413
414         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
415                 ticks = (us_on * pclk) + 999999ull;
416                 do_div(ticks, 1000000);
417                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
418
419                 ticks = (us_off * pclk) + 999999ull;
420                 do_div(ticks, 1000000);
421                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
422                 wmb();
423         }
424         tegra_last_pclk = pclk;
425         last_us_off = us_off;
426 }
427
428 /*
429  * restore_cpu_complex
430  *
431  * restores cpu clock setting, clears flow controller
432  *
433  * Always called on CPU 0.
434  */
435 static void restore_cpu_complex(u32 mode)
436 {
437         int cpu = cpu_logical_map(smp_processor_id());
438         unsigned int reg;
439 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
440         unsigned int policy;
441 #endif
442
443 /*
444  * On Tegra11x PLLX and CPU burst policy is either preserved across LP2,
445  * or restored by common clock suspend/resume procedures. Hence, we don't
446  * need it here.
447  */
448 #if defined(CONFIG_ARCH_TEGRA_2x_SOC) || defined(CONFIG_ARCH_TEGRA_3x_SOC)
449         /* Is CPU complex already running on PLLX? */
450         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
451         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
452
453         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
454                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
455         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
456                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
457         else
458                 BUG();
459
460         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
461                 /* restore PLLX settings if CPU is on different PLL */
462                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
463                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
464
465                 /* wait for PLL stabilization if PLLX was enabled */
466                 if (tegra_sctx.pllx_base & (1<<30)) {
467 #if USE_PLL_LOCK_BITS
468                         /* Enable lock detector */
469                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
470                         reg |= 1<<18;
471                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
472                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
473                                  (1<<27)))
474                                 cpu_relax();
475
476                         udelay(PLL_POST_LOCK_DELAY);
477 #else
478                         udelay(300);
479 #endif
480                 }
481         }
482
483         /* Restore original burst policy setting for calls resulting from CPU
484            LP2 in idle or system suspend; keep cluster switch prolog setting
485            intact. */
486         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
487                 writel(tegra_sctx.cclk_divider, clk_rst +
488                        CLK_RESET_CCLK_DIVIDER);
489                 writel(tegra_sctx.cpu_burst, clk_rst +
490                        CLK_RESET_CCLK_BURST);
491         }
492 #endif
493         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
494
495         /* Do not power-gate CPU 0 when flow controlled */
496         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
497         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
498         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
499         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
500         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
501         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
502         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
503
504         /* If an immedidate cluster switch is being perfomed, restore the
505            local timer registers. For calls resulting from CPU LP2 in
506            idle or system suspend, the local timer was shut down and
507            timekeeping switched over to the global system timer. In this
508            case keep local timer disabled, and restore only periodic load. */
509 #ifdef CONFIG_HAVE_ARM_TWD
510         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
511                       TEGRA_POWER_CLUSTER_IMMEDIATE))) {
512                 tegra_sctx.twd.twd_ctrl = 0;
513         }
514         tegra_twd_resume(&tegra_sctx.twd);
515 #endif
516 }
517
518 /*
519  * suspend_cpu_complex
520  *
521  * saves pll state for use by restart_plls, prepares flow controller for
522  * transition to suspend state
523  *
524  * Must always be called on cpu 0.
525  */
526 static void suspend_cpu_complex(u32 mode)
527 {
528         int cpu = cpu_logical_map(smp_processor_id());
529         unsigned int reg;
530         int i;
531
532         BUG_ON(cpu != 0);
533
534         /* switch coresite to clk_m, save off original source */
535         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
536         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
537
538         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
539         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
540         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
541         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
542         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
543         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
544         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
545         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
546
547 #ifdef CONFIG_HAVE_ARM_TWD
548         tegra_twd_suspend(&tegra_sctx.twd);
549 #endif
550
551         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
552         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
553         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
554         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
555         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
556 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
557         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
558 #else
559         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
560 #endif
561         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
562         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
563
564         for (i = 0; i < num_possible_cpus(); i++) {
565                 if (i == cpu)
566                         continue;
567                 reg = readl(FLOW_CTRL_CPU_CSR(i));
568                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
569                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
570                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
571         }
572
573         tegra_gic_cpu_disable(true);
574 }
575
576 void tegra_clear_cpu_in_pd(int cpu)
577 {
578         spin_lock(&tegra_lp2_lock);
579         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
580         cpumask_clear_cpu(cpu, &tegra_in_lp2);
581
582         /* Update the IRAM copy used by the reset handler. The IRAM copy
583            can't use used directly by cpumask_clear_cpu() because it uses
584            LDREX/STREX which requires the addressed location to be inner
585            cacheable and sharable which IRAM isn't. */
586         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
587         dsb();
588
589         spin_unlock(&tegra_lp2_lock);
590 }
591
592 bool tegra_set_cpu_in_pd(int cpu)
593 {
594         bool last_cpu = false;
595
596         spin_lock(&tegra_lp2_lock);
597         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
598         cpumask_set_cpu(cpu, &tegra_in_lp2);
599
600         /* Update the IRAM copy used by the reset handler. The IRAM copy
601            can't use used directly by cpumask_set_cpu() because it uses
602            LDREX/STREX which requires the addressed location to be inner
603            cacheable and sharable which IRAM isn't. */
604         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
605         dsb();
606
607         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
608                 last_cpu = true;
609 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
610         else if (cpu == 1)
611                 tegra2_cpu_set_resettable_soon();
612 #endif
613
614         spin_unlock(&tegra_lp2_lock);
615         return last_cpu;
616 }
617
618 static void tegra_sleep_core(enum tegra_suspend_mode mode,
619                              unsigned long v2p)
620 {
621 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
622         outer_flush_range(__pa(&tegra_resume_timestamps_start),
623                           __pa(&tegra_resume_timestamps_end));
624
625         if (mode == TEGRA_SUSPEND_LP0) {
626                 trace_smc_sleep_core(NVSEC_SMC_START);
627
628                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
629                                   virt_to_phys(tegra_resume));
630         } else {
631                 trace_smc_sleep_core(NVSEC_SMC_START);
632
633                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
634                                   (TEGRA_RESET_HANDLER_BASE +
635                                    tegra_cpu_reset_handler_offset));
636         }
637
638         trace_smc_sleep_core(NVSEC_SMC_DONE);
639 #endif
640         tegra_get_suspend_time();
641 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
642         cpu_suspend(v2p, tegra2_sleep_core_finish);
643 #else
644         cpu_suspend(v2p, tegra3_sleep_core_finish);
645 #endif
646 }
647
648 static inline void tegra_sleep_cpu(unsigned long v2p)
649 {
650         cpu_suspend(v2p, tegra_sleep_cpu_finish);
651 }
652
653 static inline void tegra_stop_mc_clk(unsigned long v2p)
654 {
655 #ifdef CONFIG_TEGRA_USE_SECURE_KERNEL
656         outer_flush_range(__pa(&tegra_resume_timestamps_start),
657                           __pa(&tegra_resume_timestamps_end));
658         trace_smc_sleep_core(NVSEC_SMC_START);
659
660         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE5,
661                           (TEGRA_RESET_HANDLER_BASE +
662                            tegra_cpu_reset_handler_offset));
663
664         trace_smc_sleep_core(NVSEC_SMC_DONE);
665 #endif
666         cpu_suspend(v2p, tegra3_stop_mc_clk_finish);
667 }
668
669 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
670                                         unsigned int flags)
671 {
672         u32 reg;
673         unsigned int remain;
674
675         /* Only the last cpu down does the final suspend steps */
676         reg = readl(pmc + PMC_CTRL);
677         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
678         if (pdata->combined_req)
679                 reg &= ~TEGRA_POWER_PWRREQ_OE;
680         else
681                 reg |= TEGRA_POWER_PWRREQ_OE;
682
683         reg &= ~TEGRA_POWER_EFFECT_LP0;
684         writel(reg, pmc + PMC_CTRL);
685
686         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
687
688         /*
689          * We can use clk_get_rate_all_locked() here, because all other cpus
690          * are in LP2 state and irqs are disabled
691          */
692         if (flags & TEGRA_POWER_CLUSTER_MASK) {
693                 if (is_idle_task(current))
694                         trace_nvcpu_cluster_rcuidle(NVPOWER_CPU_CLUSTER_START);
695                 else
696                         trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_START);
697                 set_power_timers(pdata->cpu_timer, 2,
698                         clk_get_rate_all_locked(tegra_pclk));
699                 if (flags & TEGRA_POWER_CLUSTER_G) {
700                         /*
701                          * To reduce the vdd_cpu up latency when LP->G
702                          * transition. Before the transition, enable
703                          * the vdd_cpu rail.
704                          */
705                         if (is_lp_cluster()) {
706 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
707                                 reg = readl(FLOW_CTRL_CPU_PWR_CSR);
708                                 reg |= FLOW_CTRL_CPU_PWR_CSR_RAIL_ENABLE;
709                                 writel(reg, FLOW_CTRL_CPU_PWR_CSR);
710 #else
711                                 writel(UN_PWRGATE_CPU,
712                                        pmc + PMC_PWRGATE_TOGGLE);
713 #endif
714                         }
715                 }
716                 tegra_cluster_switch_prolog(flags);
717         } else {
718                 suspend_cpu_dfll_mode();
719                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
720                         clk_get_rate_all_locked(tegra_pclk));
721 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
722                 reg = readl(FLOW_CTRL_CPU_CSR(0));
723                 reg &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
724                 if (is_lp_cluster()) {
725                         /* for LP cluster, there is no option for rail gating */
726                         if ((flags & TEGRA_POWER_CLUSTER_PART_MASK) ==
727                                                 TEGRA_POWER_CLUSTER_PART_MASK)
728                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_EMU;
729                         else if (flags)
730                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
731                 }
732                 else {
733                         if (flags & TEGRA_POWER_CLUSTER_PART_CRAIL)
734                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
735                         if (flags & TEGRA_POWER_CLUSTER_PART_NONCPU)
736                                 reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU;
737                 }
738                 writel(reg, FLOW_CTRL_CPU_CSR(0));
739 #endif
740         }
741
742         if (sleep_time)
743                 tegra_pd_set_trigger(sleep_time);
744
745         cpu_cluster_pm_enter();
746         suspend_cpu_complex(flags);
747         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
748 #if defined(CONFIG_CACHE_L2X0)
749 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
750         flush_cache_all();
751         outer_disable();
752 #elif !defined(CONFIG_ARCH_TEGRA_14x_SOC)
753         tegra_resume_l2_init = 1;
754         __cpuc_flush_dcache_area(&tegra_resume_l2_init, sizeof(unsigned long));
755         outer_flush_range(__pa(&tegra_resume_l2_init),
756                           __pa(&tegra_resume_l2_init) + sizeof(unsigned long));
757 #endif
758 #endif
759
760         /* T148: Check for mem_req and mem_req_soon only if it is
761          * MC clock stop state.
762          */
763         if (flags & TEGRA_POWER_STOP_MC_CLK) {
764 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
765                 u32 val;
766
767                 /* Check if mem_req or mem_req_soon is asserted or if voice
768                  * call is active call, if yes then we skip SDRAM
769                  * self-refresh and just do CPU power-gating.
770                  */
771                 val = readl(pmc + PMC_IPC_STS);
772                 if ((val & (PMC_IPC_STS_MEM_REQ | PMC_IPC_STS_MEM_REQ_SOON)) ||
773                         tegra_is_voice_call_active()) {
774
775                         /* Reset LP1 and MC clock mask if we skipping SDRAM
776                          * self-refresh.
777                          */
778                         *iram_cpu_lp1_mask = 0;
779                         *iram_mc_clk_mask = 0;
780                         writel(0, pmc + PMC_SCRATCH41);
781
782                         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
783                 } else {
784                         /* Clear mem_sts since SDRAM will not be accessible
785                          * to BBC in this state.
786                          */
787                         val = PMC_IPC_CLR_MEM_STS;
788                         writel(val, pmc + PMC_IPC_CLR);
789
790                         tegra_stop_mc_clk(PHYS_OFFSET - PAGE_OFFSET);
791                 }
792 #else
793                 /* If it is not T148 then we do not have to
794                  * check mem_req and mem_req_soon.
795                  */
796                 tegra_stop_mc_clk(PHYS_OFFSET - PAGE_OFFSET);
797 #endif
798         } else {
799                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
800         }
801
802 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
803         tegra_init_cache(true);
804 #elif defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
805         tegra_init_cache(false);
806 #endif
807
808 #if defined(CONFIG_TRUSTED_FOUNDATIONS)
809 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
810         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
811         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
812 #endif
813 #endif
814
815         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
816         restore_cpu_complex(flags);
817         cpu_cluster_pm_exit();
818
819         remain = tegra_pd_timer_remain();
820         if (sleep_time)
821                 tegra_pd_set_trigger(0);
822
823         if (flags & TEGRA_POWER_CLUSTER_MASK) {
824                 tegra_cluster_switch_epilog(flags);
825                 if (is_idle_task(current))
826                         trace_nvcpu_cluster_rcuidle(NVPOWER_CPU_CLUSTER_DONE);
827                 else
828                         trace_nvcpu_cluster(NVPOWER_CPU_CLUSTER_DONE);
829         } else {
830                 resume_cpu_dfll_mode();
831         }
832         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
833
834 #if INSTRUMENT_CLUSTER_SWITCH
835         if (flags & TEGRA_POWER_CLUSTER_MASK) {
836                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
837                         is_lp_cluster() ? "G=>LP" : "LP=>G",
838                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
839                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
840                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
841                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
842                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
843                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
844                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
845                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
846         }
847 #endif
848         return remain;
849 }
850
851 void tegra_mc_clk_prepare(void)
852 {
853         /* copy the reset vector and SDRAM shutdown code into IRAM */
854         memcpy(iram_save, iram_code, iram_save_size);
855         memcpy(iram_code, tegra_iram_start(), iram_save_size);
856
857         *iram_cpu_lp1_mask = 1;
858         *iram_mc_clk_mask = 1;
859
860         __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
861         wmb();
862 }
863
864 void tegra_mc_clk_finish(void)
865 {
866         /* restore IRAM */
867         memcpy(iram_code, iram_save, iram_save_size);
868         *iram_cpu_lp1_mask = 0;
869         *iram_mc_clk_mask = 0;
870         writel(0, pmc + PMC_SCRATCH41);
871 }
872
873 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
874 int tegra_is_lp1_suspend_mode(void)
875 {
876         return (current_suspend_mode == TEGRA_SUSPEND_LP1);
877 }
878 #endif
879
880 static int tegra_common_suspend(void)
881 {
882         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
883
884         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
885         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
886         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
887
888 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
889         if (pdata && pdata->lp1_lowvolt_support) {
890                 u32 lp1_core_lowvolt =
891                         (tegra_is_voice_call_active() ||
892                         tegra_dvfs_rail_get_thermal_floor(tegra_core_rail)) ?
893                         pdata->lp1_core_volt_low_cold << 8 :
894                         pdata->lp1_core_volt_low << 8;
895
896                 lp1_core_lowvolt |= pdata->core_reg_addr;
897                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
898         }
899 #endif
900
901         /* copy the reset vector and SDRAM shutdown code into IRAM */
902         memcpy(iram_save, iram_code, iram_save_size);
903         memcpy(iram_code, tegra_iram_start(), iram_save_size);
904
905         return 0;
906 }
907
908 static void tegra_common_resume(void)
909 {
910         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
911 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
912         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
913 #endif
914
915 #if defined(CONFIG_ARCH_TEGRA_14x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
916         /* Clear DPD Enable */
917         writel(0x0, pmc + PMC_DPD_ENABLE);
918 #endif
919
920         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
921         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
922         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
923 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
924         /* trigger emc mode write */
925         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
926         /* clear scratch registers shared by suspend and the reset pen */
927         writel(0x0, pmc + PMC_SCRATCH39);
928 #endif
929         writel(0x0, pmc + PMC_SCRATCH41);
930
931         /* restore IRAM */
932         memcpy(iram_code, iram_save, iram_save_size);
933 }
934
935 static int tegra_suspend_prepare_late(void)
936 {
937 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
938         disable_irq(INT_SYS_STATS_MON);
939 #endif
940         return 0;
941 }
942
943 static void tegra_suspend_wake(void)
944 {
945 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
946         enable_irq(INT_SYS_STATS_MON);
947 #endif
948 }
949
950 static void tegra_pm_set(enum tegra_suspend_mode mode)
951 {
952         u32 reg, boot_flag;
953         unsigned long rate = 32768;
954
955         reg = readl(pmc + PMC_CTRL);
956         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
957         if (pdata->combined_req)
958                 reg &= ~TEGRA_POWER_PWRREQ_OE;
959         else
960                 reg |= TEGRA_POWER_PWRREQ_OE;
961         reg &= ~TEGRA_POWER_EFFECT_LP0;
962
963         switch (mode) {
964         case TEGRA_SUSPEND_LP0:
965 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
966                 rate = clk_get_rate_all_locked(tegra_pclk);
967 #endif
968                 if (pdata->combined_req) {
969                         reg |= TEGRA_POWER_PWRREQ_OE;
970                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
971                 }
972
973 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
974                 /*
975                  * LP0 boots through the AVP, which then resumes the AVP to
976                  * the address in scratch 39, and the cpu to the address in
977                  * scratch 41 to tegra_resume
978                  */
979                 writel(0x0, pmc + PMC_SCRATCH39);
980 #endif
981
982                 /* Enable DPD sample to trigger sampling pads data and direction
983                  * in which pad will be driven during lp0 mode*/
984                 writel(0x1, pmc + PMC_DPD_SAMPLE);
985 #if !defined(CONFIG_ARCH_TEGRA_3x_SOC) && !defined(CONFIG_ARCH_TEGRA_2x_SOC)
986 #if defined(CONFIG_ARCH_TEGRA_11x_SOC) || defined(CONFIG_ARCH_TEGRA_12x_SOC)
987                 writel(0x800fdfff, pmc + PMC_IO_DPD_REQ);
988 #else
989                 writel(0x800fffff, pmc + PMC_IO_DPD_REQ);
990 #endif
991                 writel(0x80001fff, pmc + PMC_IO_DPD2_REQ);
992 #endif
993
994 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
995                 /* this is needed only for T11x, not for other chips */
996                 reg &= ~TEGRA_POWER_CPUPWRGOOD_EN;
997 #endif
998
999                 /* Set warmboot flag */
1000                 boot_flag = readl(pmc + PMC_SCRATCH0);
1001                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
1002
1003                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
1004
1005                 reg |= TEGRA_POWER_EFFECT_LP0;
1006                 /* No break here. LP0 code falls through to write SCRATCH41 */
1007         case TEGRA_SUSPEND_LP1:
1008                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
1009                 wmb();
1010                 break;
1011         case TEGRA_SUSPEND_LP2:
1012                 rate = clk_get_rate(tegra_pclk);
1013                 break;
1014         case TEGRA_SUSPEND_NONE:
1015                 return;
1016         default:
1017                 BUG();
1018         }
1019
1020         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
1021
1022         pmc_32kwritel(reg, PMC_CTRL);
1023 }
1024
1025 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
1026         [TEGRA_SUSPEND_NONE] = "none",
1027         [TEGRA_SUSPEND_LP2] = "LP2",
1028         [TEGRA_SUSPEND_LP1] = "LP1",
1029         [TEGRA_SUSPEND_LP0] = "LP0",
1030 };
1031
1032 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
1033 static int save_se_context(void)
1034 {
1035         struct device *smmu_dev, *se_dev;
1036         int ret = 0;
1037
1038         smmu_dev = get_smmu_device();
1039         if (!smmu_dev) {
1040                 pr_info("Failed to get smmu device\n");
1041                 goto save_fail;
1042         }
1043
1044         se_dev = get_se_device();
1045         if (!se_dev) {
1046                 pr_info("Failed to get SE device \n");
1047                 goto save_fail;
1048         }
1049
1050         /* smmu resume needs to be called
1051          * for se_suspend() operation */
1052         ret = tegra_smmu_resume(smmu_dev);
1053         if (ret) {
1054                 pr_info("Failed to resume smmu device\n");
1055                 goto save_fail;
1056         }
1057
1058         ret = se_suspend(se_dev, true);
1059         if (ret) {
1060                 pr_info("Failed to suspend SE device\n");
1061                 goto save_fail;
1062         }
1063
1064         ret = tegra_smmu_suspend(smmu_dev);
1065         if (ret) {
1066                 pr_info("Failed to suspend smmu device\n");
1067                 goto save_fail;
1068         }
1069
1070 save_fail:
1071         return ret;
1072 }
1073 #endif
1074
1075 static int tegra_suspend_enter(suspend_state_t state)
1076 {
1077         int ret = 0;
1078         ktime_t delta;
1079         struct timespec ts_entry, ts_exit;
1080
1081         if (pdata && pdata->board_suspend)
1082                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
1083
1084         read_persistent_clock(&ts_entry);
1085
1086         ret = tegra_suspend_dram(current_suspend_mode, 0);
1087         if (ret) {
1088                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
1089                 goto abort_suspend;
1090         }
1091
1092 #if defined(CONFIG_CRYPTO_DEV_TEGRA_SE) && defined(CONFIG_ARCH_TEGRA_14x_SOC)
1093         ret = save_se_context();
1094         if (ret) {
1095                 pr_info("Failed to save SE context\n");
1096                 goto abort_suspend;
1097         }
1098 #endif
1099
1100         read_persistent_clock(&ts_exit);
1101
1102         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
1103                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
1104
1105                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
1106                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1107                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
1108                 else
1109                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
1110         }
1111
1112 abort_suspend:
1113         if (pdata && pdata->board_resume)
1114                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
1115
1116         return ret;
1117 }
1118
1119 static void tegra_suspend_check_pwr_stats(void)
1120 {
1121         /* cpus and l2 are powered off later */
1122         unsigned long pwrgate_partid_mask =
1123 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
1124                 (1 << TEGRA_POWERGATE_HEG)      |
1125                 (1 << TEGRA_POWERGATE_SATA)     |
1126                 (1 << TEGRA_POWERGATE_3D1)      |
1127 #endif
1128                 (1 << TEGRA_POWERGATE_3D)       |
1129                 (1 << TEGRA_POWERGATE_VENC)     |
1130                 (1 << TEGRA_POWERGATE_PCIE)     |
1131                 (1 << TEGRA_POWERGATE_VDEC)     |
1132                 (1 << TEGRA_POWERGATE_MPE);
1133
1134         int partid;
1135
1136         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
1137                 if ((1 << partid) & pwrgate_partid_mask)
1138                         if (tegra_powergate_is_powered(partid))
1139                                 pr_debug("partition %s is left on before suspend\n",
1140                                         tegra_powergate_get_name(partid));
1141
1142         return;
1143 }
1144
1145 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1146 /* This is the opposite of the LP1BB related PMC setup that occurs
1147  * during suspend.
1148  */
1149 static void tegra_disable_lp1bb_interrupt(void)
1150 {
1151         unsigned reg;
1152         /* mem_req = 0 was set as an interrupt during LP1BB entry.
1153          * It has to be disabled now
1154          */
1155         reg = readl(pmc + PMC_CTRL2);
1156         reg &= ~(PMC_CTRL2_WAKE_DET_EN);
1157         pmc_32kwritel(reg, PMC_CTRL2);
1158
1159         /* Program mem_req NOT to be a wake event */
1160         reg = readl(pmc + PMC_WAKE2_MASK);
1161         reg &= ~(PMC_WAKE2_BB_MEM_REQ);
1162         pmc_32kwritel(reg, PMC_WAKE2_MASK);
1163
1164         reg = PMC_WAKE2_BB_MEM_REQ;
1165         pmc_32kwritel(reg, PMC_WAKE2_STATUS);
1166
1167         /* Set up the LIC to NOT accept pmc_wake events as interrupts */
1168         reg = TRI_ICTLR_PMC_WAKE_INT;
1169         writel(reg, tert_ictlr + TRI_ICTLR_CPU_IER_CLR);
1170 }
1171 #endif
1172
1173 static void tegra_suspend_powergate_control(int partid, bool turn_off)
1174 {
1175         if (turn_off)
1176                 tegra_powergate_partition(partid);
1177         else
1178                 tegra_unpowergate_partition(partid);
1179 }
1180
1181 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
1182 {
1183         int err = 0;
1184         u32 scratch37 = 0xDEADBEEF;
1185         u32 reg;
1186
1187 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1188         u32 enter_state = 0;
1189 #endif
1190         bool tegra_suspend_vde_powergated = false;
1191
1192         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
1193                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
1194                 err = -ENXIO;
1195                 goto fail;
1196         }
1197
1198 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1199         update_pmc_registers(1);
1200 #endif
1201
1202         if (tegra_is_voice_call_active()) {
1203                 /* backup the current value of scratch37 */
1204                 scratch37 = readl(pmc + PMC_SCRATCH37);
1205
1206                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
1207                 reg = TEGRA_POWER_LP1_AUDIO;
1208                 pmc_32kwritel(reg, PMC_SCRATCH37);
1209         }
1210
1211         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
1212                 pr_info("LP0 not used due to unsupported wakeup events\n");
1213                 mode = TEGRA_SUSPEND_LP1;
1214         }
1215
1216         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
1217                 tegra_suspend_check_pwr_stats();
1218
1219         /* turn off VDE partition in LP1 */
1220         if (mode == TEGRA_SUSPEND_LP1 &&
1221                 tegra_powergate_is_powered(TEGRA_POWERGATE_VDEC)) {
1222                 pr_info("turning off partition %s in LP1\n",
1223                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
1224                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, true);
1225                 tegra_suspend_vde_powergated = true;
1226         }
1227
1228         tegra_common_suspend();
1229
1230         tegra_pm_set(mode);
1231
1232         if (pdata && pdata->board_suspend)
1233                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
1234
1235         local_fiq_disable();
1236
1237 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1238         tegra_smp_save_power_mask();
1239 #endif
1240
1241         trace_cpu_suspend(CPU_SUSPEND_START, tegra_rtc_read_ms());
1242
1243         if (mode == TEGRA_SUSPEND_LP0) {
1244 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
1245                 reg = readl(pmc + PMC_SCRATCH4);
1246                 if (is_lp_cluster())
1247                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
1248                 else
1249                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
1250                 pmc_32kwritel(reg, PMC_SCRATCH4);
1251 #endif
1252                 tegra_tsc_suspend();
1253                 tegra_lp0_suspend_mc();
1254                 tegra_cpu_reset_handler_save();
1255                 tegra_tsc_wait_for_suspend();
1256 #ifndef CONFIG_TEGRA_SIMULATION_PLATFORM
1257                 tegra_smp_clear_power_mask();
1258 #endif
1259         }
1260
1261 #if !defined(CONFIG_ARCH_TEGRA_14x_SOC)
1262         if (mode == TEGRA_SUSPEND_LP1)
1263 #endif
1264                 *iram_cpu_lp1_mask = 1;
1265
1266         suspend_cpu_complex(flags);
1267
1268 #if defined(CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE)
1269         /* In case of LP0/1, program external power gating accordinly */
1270         if (mode == TEGRA_SUSPEND_LP0 || mode == TEGRA_SUSPEND_LP1) {
1271                 reg = readl(FLOW_CTRL_CPU_CSR(0));
1272                 if (is_lp_cluster())
1273                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_NCPU; /* Non CPU */
1274                 else
1275                         reg |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;  /* CRAIL */
1276                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(0));
1277         }
1278 #endif
1279
1280         flush_cache_all();
1281         outer_disable();
1282
1283         if (mode == TEGRA_SUSPEND_LP2)
1284                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
1285         else
1286                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
1287
1288         resume_entry_time = 0;
1289         if (mode != TEGRA_SUSPEND_LP0)
1290                 resume_entry_time = readl(tmrus_reg_base + TIMERUS_CNTR_1US);
1291
1292         tegra_init_cache(true);
1293
1294 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1295         reg = readl(pmc + PMC_LP_STATE_SCRATCH_REG);
1296         enter_state = (reg >> PMC_LP_STATE_BIT_OFFSET) & PMC_LP_STATE_BIT_MASK;
1297         /* If we actually had entered in either LP1 or LP1BB,
1298          * restore power mask and disable mem_req interrupt PMC
1299          */
1300         if (enter_state) {
1301                 pr_info("Exited state is LP1/LP1BB\n");
1302                 tegra_disable_lp1bb_interrupt();
1303                 tegra_smp_restore_power_mask();
1304         }
1305 #endif
1306
1307 #if defined(CONFIG_TEGRA_USE_SECURE_KERNEL)
1308 #ifndef CONFIG_ARCH_TEGRA_11x_SOC
1309         trace_smc_wake(tegra_resume_smc_entry_time, NVSEC_SMC_START);
1310         trace_smc_wake(tegra_resume_smc_exit_time, NVSEC_SMC_DONE);
1311 #endif
1312
1313         if (mode == TEGRA_SUSPEND_LP0) {
1314                 trace_secureos_init(tegra_resume_entry_time,
1315                         NVSEC_SUSPEND_EXIT_DONE);
1316         }
1317 #endif
1318
1319         if (mode == TEGRA_SUSPEND_LP0) {
1320
1321                 /* CPUPWRGOOD_EN is not enabled in HW so disabling this, *
1322                 * Otherwise it is creating issue in cluster switch after LP0 *
1323 #ifdef CONFIG_ARCH_TEGRA_11x_SOC
1324                 reg = readl(pmc+PMC_CTRL);
1325                 reg |= TEGRA_POWER_CPUPWRGOOD_EN;
1326                 pmc_32kwritel(reg, PMC_CTRL);
1327 #endif
1328                 */
1329
1330                 tegra_tsc_resume();
1331                 tegra_cpu_reset_handler_restore();
1332                 tegra_lp0_resume_mc();
1333                 tegra_tsc_wait_for_resume();
1334         }
1335
1336 #if !defined(CONFIG_ARCH_TEGRA_14x_SOC)
1337         if (mode == TEGRA_SUSPEND_LP1)
1338 #endif
1339                 *iram_cpu_lp1_mask = 0;
1340
1341         /* if scratch37 was clobbered during LP1, restore it */
1342         if (scratch37 != 0xDEADBEEF)
1343                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
1344
1345         restore_cpu_complex(flags);
1346
1347         /* for platforms where the core & CPU power requests are
1348          * combined as a single request to the PMU, transition out
1349          * of LP0 state by temporarily enabling both requests
1350          */
1351         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
1352                 reg = readl(pmc + PMC_CTRL);
1353                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1354                 pmc_32kwritel(reg, PMC_CTRL);
1355                 reg &= ~TEGRA_POWER_PWRREQ_OE;
1356                 pmc_32kwritel(reg, PMC_CTRL);
1357         }
1358
1359         if (pdata && pdata->board_resume)
1360                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
1361
1362         trace_cpu_suspend(CPU_SUSPEND_DONE, tegra_rtc_read_ms());
1363
1364         local_fiq_enable();
1365
1366         tegra_common_resume();
1367
1368         /* turn on VDE partition in LP1 */
1369         if (mode == TEGRA_SUSPEND_LP1 && tegra_suspend_vde_powergated) {
1370                 pr_info("turning on partition %s in LP1\n",
1371                         tegra_powergate_get_name(TEGRA_POWERGATE_VDEC));
1372                 tegra_suspend_powergate_control(TEGRA_POWERGATE_VDEC, false);
1373         }
1374
1375 fail:
1376         return err;
1377 }
1378
1379 /*
1380  * Function pointers to optional board specific function
1381  */
1382 void (*tegra_deep_sleep)(int);
1383 EXPORT_SYMBOL(tegra_deep_sleep);
1384
1385 static int tegra_suspend_prepare(void)
1386 {
1387         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1388                 tegra_deep_sleep(1);
1389         return 0;
1390 }
1391
1392 static void tegra_suspend_finish(void)
1393 {
1394         if (pdata && pdata->cpu_resume_boost) {
1395                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
1396                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
1397                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
1398         }
1399
1400         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
1401                 tegra_deep_sleep(0);
1402 }
1403
1404 static const struct platform_suspend_ops tegra_suspend_ops = {
1405         .valid          = suspend_valid_only_mem,
1406         .prepare        = tegra_suspend_prepare,
1407         .finish         = tegra_suspend_finish,
1408         .prepare_late   = tegra_suspend_prepare_late,
1409         .wake           = tegra_suspend_wake,
1410         .enter          = tegra_suspend_enter,
1411 };
1412
1413 static ssize_t suspend_mode_show(struct kobject *kobj,
1414                                         struct kobj_attribute *attr, char *buf)
1415 {
1416         char *start = buf;
1417         char *end = buf + PAGE_SIZE;
1418
1419         start += scnprintf(start, end - start, "%s ", \
1420                                 tegra_suspend_name[current_suspend_mode]);
1421         start += scnprintf(start, end - start, "\n");
1422
1423         return start - buf;
1424 }
1425
1426 static ssize_t suspend_mode_store(struct kobject *kobj,
1427                                         struct kobj_attribute *attr,
1428                                         const char *buf, size_t n)
1429 {
1430         int len;
1431         const char *name_ptr;
1432         enum tegra_suspend_mode new_mode;
1433
1434         name_ptr = buf;
1435         while (*name_ptr && !isspace(*name_ptr))
1436                 name_ptr++;
1437         len = name_ptr - buf;
1438         if (!len)
1439                 goto bad_name;
1440         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
1441         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
1442                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
1443                 pr_info("Illegal tegra suspend state: %s\n", buf);
1444                 goto bad_name;
1445         }
1446
1447         for (new_mode = TEGRA_SUSPEND_NONE; \
1448                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
1449                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
1450                         current_suspend_mode = new_mode;
1451                         break;
1452                 }
1453         }
1454
1455 bad_name:
1456         return n;
1457 }
1458
1459 static struct kobj_attribute suspend_mode_attribute =
1460         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
1461
1462 static ssize_t suspend_resume_time_show(struct kobject *kobj,
1463                                         struct kobj_attribute *attr,
1464                                         char *buf)
1465 {
1466         return sprintf(buf, "%ums\n", ((u32)resume_time / 1000));
1467 }
1468
1469 static struct kobj_attribute suspend_resume_time_attribute =
1470         __ATTR(resume_time, 0444, suspend_resume_time_show, 0);
1471
1472 static ssize_t suspend_time_show(struct kobject *kobj,
1473                                         struct kobj_attribute *attr,
1474                                         char *buf)
1475 {
1476         return sprintf(buf, "%ums\n", ((u32)suspend_time / 1000));
1477 }
1478
1479 static struct kobj_attribute suspend_time_attribute =
1480         __ATTR(suspend_time, 0444, suspend_time_show, 0);
1481
1482 static struct kobject *suspend_kobj;
1483
1484 static int tegra_pm_enter_suspend(void)
1485 {
1486         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
1487         suspend_cpu_dfll_mode();
1488         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1489                 tegra_lp0_cpu_mode(true);
1490         return 0;
1491 }
1492
1493 static void tegra_pm_enter_resume(void)
1494 {
1495         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1496                 tegra_lp0_cpu_mode(false);
1497         resume_cpu_dfll_mode();
1498         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1499 }
1500
1501 static void tegra_pm_enter_shutdown(void)
1502 {
1503         suspend_cpu_dfll_mode();
1504         pr_info("Shutting down tegra ...\n");
1505 }
1506
1507 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1508         .suspend = tegra_pm_enter_suspend,
1509         .resume = tegra_pm_enter_resume,
1510         .shutdown = tegra_pm_enter_shutdown,
1511 };
1512
1513 static __init int tegra_pm_enter_syscore_init(void)
1514 {
1515         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1516         return 0;
1517 }
1518 subsys_initcall(tegra_pm_enter_syscore_init);
1519 #endif
1520
1521 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1522 {
1523         u32 reg;
1524         u32 mode;
1525
1526 #ifdef CONFIG_ARCH_TEGRA_HAS_CL_DVFS
1527         tegra_dfll = clk_get_sys(NULL, "dfll_cpu");
1528         BUG_ON(IS_ERR(tegra_dfll));
1529 #endif
1530         tegra_pclk = clk_get_sys(NULL, "pclk");
1531         BUG_ON(IS_ERR(tegra_pclk));
1532         pdata = plat;
1533         (void)reg;
1534         (void)mode;
1535
1536         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1537                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1538
1539 #ifndef CONFIG_PM_SLEEP
1540         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1541                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1542                            "disabling suspend\n", __func__);
1543                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1544         }
1545 #else
1546         if (create_suspend_pgtable() < 0) {
1547                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1548                                 __func__);
1549                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1550                 goto fail;
1551         }
1552
1553         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1554             (tegra_revision == TEGRA_REVISION_A01) &&
1555             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1556                 /* Tegra 3 A01 supports only LP1 */
1557                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1558                            "-- disabling LP0\n", __func__);
1559                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1560         }
1561         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1562                 tegra_lp0_vec_relocate) {
1563                 unsigned char *reloc_lp0;
1564                 unsigned long tmp;
1565                 void __iomem *orig;
1566                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1567                                         GFP_KERNEL);
1568                 WARN_ON(!reloc_lp0);
1569                 if (!reloc_lp0) {
1570                         pr_err("%s: Failed to allocate reloc_lp0\n",
1571                                 __func__);
1572                         goto out;
1573                 }
1574
1575                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1576                 WARN_ON(!orig);
1577                 if (!orig) {
1578                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1579                                 __func__, tegra_lp0_vec_start);
1580                         kfree(reloc_lp0);
1581                         goto out;
1582                 }
1583
1584                 tmp = (unsigned long) reloc_lp0;
1585                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1586                 reloc_lp0 = (unsigned char *)tmp;
1587                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1588                 iounmap(orig);
1589                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1590         }
1591
1592 out:
1593         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1594                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1595                            "provided by bootlader -- disabling LP0\n",
1596                            __func__);
1597                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1598         }
1599
1600         iram_save_size = tegra_iram_end() - tegra_iram_start();
1601
1602         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1603         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1604                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1605                        "-- LP0/LP1 unavailable\n", __func__);
1606                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1607         }
1608
1609 #ifdef CONFIG_TEGRA_LP1_LOW_COREVOLTAGE
1610         if (pdata->lp1_lowvolt_support) {
1611                 u32 lp1_core_lowvolt, lp1_core_highvolt;
1612                 memcpy(tegra_lp1_register_pmuslave_addr(), &pdata->pmuslave_addr, 4);
1613                 memcpy(tegra_lp1_register_i2c_base_addr(), &pdata->i2c_base_addr, 4);
1614
1615                 lp1_core_lowvolt = 0;
1616                 lp1_core_lowvolt = (pdata->lp1_core_volt_low << 8) | pdata->core_reg_addr;
1617                 memcpy(tegra_lp1_register_core_lowvolt(), &lp1_core_lowvolt, 4);
1618
1619                 lp1_core_highvolt = 0;
1620                 lp1_core_highvolt = (pdata->lp1_core_volt_high << 8) | pdata->core_reg_addr;
1621                 memcpy(tegra_lp1_register_core_highvolt(), &lp1_core_highvolt, 4);
1622         }
1623 #endif
1624         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1625         /* Initialize scratch registers used for CPU LP2 synchronization */
1626         writel(0, pmc + PMC_SCRATCH37);
1627         writel(0, pmc + PMC_SCRATCH38);
1628 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
1629         writel(0, pmc + PMC_SCRATCH39);
1630 #endif
1631         writel(0, pmc + PMC_SCRATCH41);
1632
1633         /* Always enable CPU power request; just normal polarity is supported */
1634         reg = readl(pmc + PMC_CTRL);
1635         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1636         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1637         pmc_32kwritel(reg, PMC_CTRL);
1638
1639         /* Configure core power request and system clock control if LP0
1640            is supported */
1641         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1642         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1643
1644         reg = readl(pmc + PMC_CTRL);
1645
1646         if (!pdata->sysclkreq_high)
1647                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1648         else
1649                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1650
1651         if (!pdata->corereq_high)
1652                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1653         else
1654                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1655
1656         /* configure output inverters while the request is tristated */
1657         pmc_32kwritel(reg, PMC_CTRL);
1658
1659         /* now enable requests */
1660         reg |= TEGRA_POWER_SYSCLK_OE;
1661         if (!pdata->combined_req)
1662                 reg |= TEGRA_POWER_PWRREQ_OE;
1663         pmc_32kwritel(reg, PMC_CTRL);
1664
1665         if (pdata->sysclkreq_gpio) {
1666                 reg = readl(pmc + PMC_DPAD_ORIDE);
1667                 reg &= ~TEGRA_DPAD_ORIDE_SYS_CLK_REQ;
1668                 pmc_32kwritel(reg, PMC_DPAD_ORIDE);
1669         }
1670
1671         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1672                 tegra_lp0_suspend_init();
1673
1674         suspend_set_ops(&tegra_suspend_ops);
1675
1676         /* Create /sys/power/suspend/type */
1677         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1678         if (suspend_kobj) {
1679                 if (sysfs_create_file(suspend_kobj, \
1680                                                 &suspend_mode_attribute.attr))
1681                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1682                                                                 __func__);
1683                 if (sysfs_create_file(suspend_kobj, \
1684                                         &suspend_resume_time_attribute.attr))
1685                         pr_err("%s: sysfs_create_file resume_time failed!\n",
1686                                                                 __func__);
1687                 if (sysfs_create_file(suspend_kobj, \
1688                                         &suspend_time_attribute.attr))
1689                         pr_err("%s: sysfs_create_file suspend_time failed!\n",
1690                                                                 __func__);
1691         }
1692
1693         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1694         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1695         iram_mc_clk_mask = tegra_mc_clk_mask;
1696
1697         /* clear io dpd settings before kernel */
1698         tegra_bl_io_dpd_cleanup();
1699
1700 fail:
1701 #endif
1702         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1703                 tegra_pd_in_idle(false);
1704
1705         current_suspend_mode = plat->suspend_mode;
1706 }
1707
1708 unsigned long debug_uart_port_base = 0;
1709 EXPORT_SYMBOL(debug_uart_port_base);
1710
1711 static int tegra_debug_uart_suspend(void)
1712 {
1713         void __iomem *uart;
1714         u32 lcr;
1715
1716         if (!debug_uart_port_base)
1717                 return 0;
1718
1719         uart = IO_ADDRESS(debug_uart_port_base);
1720
1721         lcr = readb(uart + UART_LCR * 4);
1722
1723         tegra_sctx.uart[0] = lcr;
1724         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1725
1726         /* DLAB = 0 */
1727         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1728
1729         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1730
1731         /* DLAB = 1 */
1732         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1733
1734         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1735         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1736
1737         writeb(lcr, uart + UART_LCR * 4);
1738
1739         return 0;
1740 }
1741
1742 static void tegra_debug_uart_resume(void)
1743 {
1744         void __iomem *uart;
1745         u32 lcr;
1746
1747         if (!debug_uart_port_base)
1748                 return;
1749
1750         uart = IO_ADDRESS(debug_uart_port_base);
1751
1752         lcr = tegra_sctx.uart[0];
1753
1754         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1755
1756         /* DLAB = 0 */
1757         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1758
1759         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1760                         uart + UART_FCR * 4);
1761
1762         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1763
1764         /* DLAB = 1 */
1765         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1766
1767         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1768         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1769
1770         writeb(lcr, uart + UART_LCR * 4);
1771 }
1772
1773 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1774         .suspend = tegra_debug_uart_suspend,
1775         .resume = tegra_debug_uart_resume,
1776 };
1777
1778 struct clk *debug_uart_clk = NULL;
1779 EXPORT_SYMBOL(debug_uart_clk);
1780
1781 void tegra_console_uart_suspend(void)
1782 {
1783         if (console_suspend_enabled && debug_uart_clk)
1784                 tegra_clk_disable_unprepare(debug_uart_clk);
1785 }
1786
1787 void tegra_console_uart_resume(void)
1788 {
1789         if (console_suspend_enabled && debug_uart_clk)
1790                 tegra_clk_prepare_enable(debug_uart_clk);
1791 }
1792
1793 static int tegra_debug_uart_syscore_init(void)
1794 {
1795         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1796         return 0;
1797 }
1798 arch_initcall(tegra_debug_uart_syscore_init);
1799
1800 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
1801 static inline bool pmc_write_check(int index, int bit_position)
1802 {
1803         if (pmc_write_bitmap[index] & (1 << bit_position))
1804                 return true;
1805         else
1806                 return false;
1807 }
1808
1809 static void update_pmc_registers(int instance)
1810 {
1811         u32 i, j;
1812
1813         /* Based on index, we select that block of scratches */
1814         u32 base2 = (tegra_wb0_params_address + (instance - 1) *
1815                 tegra_wb0_params_block_size);
1816         void __iomem *base = ioremap(base2, tegra_wb0_params_block_size);
1817
1818 #define copy_dram_to_pmc(index, bit)    \
1819         pmc_32kwritel(readl(base + PMC_REGISTER_OFFSET(index, bit)), \
1820                 PMC_REGISTER_OFFSET(index, bit) + PMC_SCRATCH0)
1821
1822
1823         /* Iterate through the bitmap, and copy those registers
1824          * which are marked in the bitmap
1825          */
1826         for (i = 0, j = 0; j < ARRAY_SIZE(pmc_write_bitmap);) {
1827                 if (pmc_write_bitmap[j] == 0) {
1828                         j++;
1829                         i = 0;
1830                         continue;
1831                 }
1832
1833                 if (pmc_write_check(j, i))
1834                         copy_dram_to_pmc(j, i);
1835
1836                 if (++i > (sizeof(pmc_write_bitmap[0]) * 8)) {
1837                         i = 0;
1838                         j++;
1839                 }
1840         }
1841
1842 #undef copy_dram_to_pmc
1843         iounmap(base);
1844 }
1845 #endif
1846
1847 #ifdef CONFIG_ARM_ARCH_TIMER
1848
1849 static u32 tsc_suspend_start;
1850 static u32 tsc_resume_start;
1851
1852 #define pmc_writel(value, reg) \
1853                 writel(value, (uintptr_t)pmc + (reg))
1854 #define pmc_readl(reg) \
1855                 readl((uintptr_t)pmc + (reg))
1856
1857 #define PMC_DPD_ENABLE                  0x24
1858 #define PMC_DPD_ENABLE_TSC_MULT_ENABLE  (1 << 1)
1859
1860 #define PMC_TSC_MULT                    0x2b4
1861 #define PMC_TSC_MULT_FREQ_STS           (1 << 16)
1862
1863 #define TSC_TIMEOUT_US                  32
1864
1865 void tegra_tsc_suspend(void)
1866 {
1867         if (arch_timer_initialized) {
1868                 u32 reg = pmc_readl(PMC_DPD_ENABLE);
1869                 BUG_ON(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE);
1870                 reg |= PMC_DPD_ENABLE_TSC_MULT_ENABLE;
1871                 pmc_writel(reg, PMC_DPD_ENABLE);
1872                 tsc_suspend_start = timer_readl(TIMERUS_CNTR_1US);
1873         }
1874 }
1875
1876 void tegra_tsc_resume(void)
1877 {
1878         if (arch_timer_initialized) {
1879                 u32 reg = pmc_readl(PMC_DPD_ENABLE);
1880                 BUG_ON(!(reg & PMC_DPD_ENABLE_TSC_MULT_ENABLE));
1881                 reg &= ~PMC_DPD_ENABLE_TSC_MULT_ENABLE;
1882                 pmc_writel(reg, PMC_DPD_ENABLE);
1883                 tsc_resume_start = timer_readl(TIMERUS_CNTR_1US);
1884         }
1885 }
1886
1887 void tegra_tsc_wait_for_suspend(void)
1888 {
1889         if (arch_timer_initialized) {
1890                 while ((timer_readl(TIMERUS_CNTR_1US) - tsc_suspend_start) <
1891                         TSC_TIMEOUT_US) {
1892                         if (pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS)
1893                                 break;
1894                         cpu_relax();
1895                 }
1896         }
1897 }
1898
1899 void tegra_tsc_wait_for_resume(void)
1900 {
1901         if (arch_timer_initialized) {
1902                 while ((timer_readl(TIMERUS_CNTR_1US) - tsc_resume_start) <
1903                         TSC_TIMEOUT_US) {
1904                         if (!(pmc_readl(PMC_TSC_MULT) & PMC_TSC_MULT_FREQ_STS))
1905                                 break;
1906                         cpu_relax();
1907                 }
1908         }
1909 }
1910 #endif