arch/arm/mach-tegra/pm.c

   1 /*
   2  * arch/arm/mach-tegra/pm.c
   3  *
   4  * CPU complex suspend & resume functions for Tegra SoCs
   5  *
   6  * Copyright (c) 2009-2012, NVIDIA Corporation.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  21  */
  22
  23 #include <linux/kernel.h>
  24 #include <linux/ctype.h>
  25 #include <linux/init.h>
  26 #include <linux/io.h>
  27 #include <linux/sched.h>
  28 #include <linux/smp.h>
  29 #include <linux/irq.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/clk.h>
  32 #include <linux/err.h>
  33 #include <linux/debugfs.h>
  34 #include <linux/delay.h>
  35 #include <linux/suspend.h>
  36 #include <linux/slab.h>
  37 #include <linux/serial_reg.h>
  38 #include <linux/seq_file.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/syscore_ops.h>
  41 #include <linux/cpu_pm.h>
  42 #include <linux/clk/tegra.h>
  43 #include <linux/export.h>
  44 #include <linux/vmalloc.h>
  45 #include <linux/memblock.h>
  46 #include <linux/console.h>
  47 #include <linux/tegra_audio.h>
  48
  49 #include <trace/events/power.h>
  50
  51 #include <asm/cacheflush.h>
  52 #include <asm/idmap.h>
  53 #include <asm/localtimer.h>
  54 #include <asm/pgalloc.h>
  55 #include <asm/pgtable.h>
  56 #include <asm/tlbflush.h>
  57 #include <asm/suspend.h>
  58 #include <asm/smp_plat.h>
  59
  60 #include <mach/irqs.h>
  61 #include <mach/powergate.h>
  62 #include <mach/hardware.h>
  63
  64 #include "board.h"
  65 #include "clock.h"
  66 #include "common.h"
  67 #include "cpuidle.h"
  68 #include "fuse.h"
  69 #include "gic.h"
  70 #include "iomap.h"
  71 #include "pm.h"
  72 #include "pm-irq.h"
  73 #include "reset.h"
  74 #include "sleep.h"
  75 #include "timer.h"
  76 #include "dvfs.h"
  77 #include "cpu-tegra.h"
  78
  79 struct suspend_context {
  80         /*
  81          * The next 7 values are referenced by offset in __restart_plls
  82          * in headsmp-t2.S, and should not be moved
  83          */
  84         u32 pllx_misc;
  85         u32 pllx_base;
  86         u32 pllp_misc;
  87         u32 pllp_base;
  88         u32 pllp_outa;
  89         u32 pllp_outb;
  90         u32 pll_timeout;
  91
  92         u32 cpu_burst;
  93         u32 clk_csite_src;
  94         u32 cclk_divider;
  95
  96         u32 mc[3];
  97         u8 uart[5];
  98
  99         struct tegra_twd_context twd;
 100 };
 101
 102 #ifdef CONFIG_PM_SLEEP
 103 phys_addr_t tegra_pgd_phys;     /* pgd used by hotplug & LP2 bootup */
 104 static pgd_t *tegra_pgd;
 105 static DEFINE_SPINLOCK(tegra_lp2_lock);
 106 static cpumask_t tegra_in_lp2;
 107 static cpumask_t *iram_cpu_lp2_mask;
 108 static unsigned long *iram_cpu_lp1_mask;
 109 static u8 *iram_save;
 110 static unsigned long iram_save_size;
 111 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
 112 static void __iomem *clk_rst = IO_ADDRESS(TEGRA_CLK_RESET_BASE);
 113 static void __iomem *pmc = IO_ADDRESS(TEGRA_PMC_BASE);
 114 static int tegra_last_pclk;
 115 #endif
 116
 117 struct suspend_context tegra_sctx;
 118
 119 #define TEGRA_POWER_PWRREQ_POLARITY     (1 << 8)   /* core power request polarity */
 120 #define TEGRA_POWER_PWRREQ_OE           (1 << 9)   /* core power request enable */
 121 #define TEGRA_POWER_SYSCLK_POLARITY     (1 << 10)  /* sys clk polarity */
 122 #define TEGRA_POWER_SYSCLK_OE           (1 << 11)  /* system clock enable */
 123 #define TEGRA_POWER_PWRGATE_DIS         (1 << 12)  /* power gate disabled */
 124 #define TEGRA_POWER_EFFECT_LP0          (1 << 14)  /* enter LP0 when CPU pwr gated */
 125 #define TEGRA_POWER_CPU_PWRREQ_POLARITY (1 << 15)  /* CPU power request polarity */
 126 #define TEGRA_POWER_CPU_PWRREQ_OE       (1 << 16)  /* CPU power request enable */
 127
 128 #define PMC_CTRL                0x0
 129 #define PMC_CTRL_LATCH_WAKEUPS  (1 << 5)
 130 #define PMC_WAKE_MASK           0xc
 131 #define PMC_WAKE_LEVEL          0x10
 132 #define PMC_DPAD_ORIDE          0x1C
 133 #define PMC_WAKE_DELAY          0xe0
 134 #define PMC_DPD_SAMPLE          0x20
 135
 136 #define PMC_WAKE_STATUS         0x14
 137 #define PMC_SW_WAKE_STATUS      0x18
 138 #define PMC_COREPWRGOOD_TIMER   0x3c
 139 #define PMC_CPUPWRGOOD_TIMER    0xc8
 140 #define PMC_CPUPWROFF_TIMER     0xcc
 141 #define PMC_COREPWROFF_TIMER    PMC_WAKE_DELAY
 142
 143 #define PMC_PWRGATE_TOGGLE      0x30
 144 #define PWRGATE_TOGGLE_START    (1 << 8)
 145 #define UN_PWRGATE_CPU          \
 146         (PWRGATE_TOGGLE_START | TEGRA_CPU_POWERGATE_ID(TEGRA_POWERGATE_CPU))
 147
 148 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
 149 #define PMC_SCRATCH4_WAKE_CLUSTER_MASK  (1<<31)
 150 #endif
 151
 152 #define CLK_RESET_CCLK_BURST    0x20
 153 #define CLK_RESET_CCLK_DIVIDER  0x24
 154 #define CLK_RESET_PLLC_BASE     0x80
 155 #define CLK_RESET_PLLM_BASE     0x90
 156 #define CLK_RESET_PLLX_BASE     0xe0
 157 #define CLK_RESET_PLLX_MISC     0xe4
 158 #define CLK_RESET_PLLP_BASE     0xa0
 159 #define CLK_RESET_PLLP_OUTA     0xa4
 160 #define CLK_RESET_PLLP_OUTB     0xa8
 161 #define CLK_RESET_PLLP_MISC     0xac
 162
 163 #define CLK_RESET_SOURCE_CSITE  0x1d4
 164
 165 #define CLK_RESET_CCLK_BURST_POLICY_SHIFT 28
 166 #define CLK_RESET_CCLK_RUN_POLICY_SHIFT    4
 167 #define CLK_RESET_CCLK_IDLE_POLICY_SHIFT   0
 168 #define CLK_RESET_CCLK_IDLE_POLICY         1
 169 #define CLK_RESET_CCLK_RUN_POLICY          2
 170 #define CLK_RESET_CCLK_BURST_POLICY_PLLM   3
 171 #define CLK_RESET_CCLK_BURST_POLICY_PLLX   8
 172
 173 #define EMC_MRW_0               0x0e8
 174 #define EMC_MRW_DEV_SELECTN     30
 175 #define EMC_MRW_DEV_NONE        (3 << EMC_MRW_DEV_SELECTN)
 176
 177 #define MC_SECURITY_START       0x6c
 178 #define MC_SECURITY_SIZE        0x70
 179 #define MC_SECURITY_CFG2        0x7c
 180
 181 struct dvfs_rail *tegra_cpu_rail;
 182 static struct dvfs_rail *tegra_core_rail;
 183 static struct clk *tegra_pclk;
 184 static const struct tegra_suspend_platform_data *pdata;
 185 static enum tegra_suspend_mode current_suspend_mode = TEGRA_SUSPEND_NONE;
 186
 187 #if defined(CONFIG_TEGRA_CLUSTER_CONTROL) && INSTRUMENT_CLUSTER_SWITCH
 188 enum tegra_cluster_switch_time_id {
 189         tegra_cluster_switch_time_id_start = 0,
 190         tegra_cluster_switch_time_id_prolog,
 191         tegra_cluster_switch_time_id_switch,
 192         tegra_cluster_switch_time_id_epilog,
 193         tegra_cluster_switch_time_id_max
 194 };
 195
 196 static unsigned long
 197                 tegra_cluster_switch_times[tegra_cluster_switch_time_id_max];
 198 #define tegra_cluster_switch_time(flags, id) \
 199         do { \
 200                 barrier(); \
 201                 if (flags & TEGRA_POWER_CLUSTER_MASK) { \
 202                         void __iomem *timer_us = \
 203                                                 IO_ADDRESS(TEGRA_TMRUS_BASE); \
 204                         if (id < tegra_cluster_switch_time_id_max) \
 205                                 tegra_cluster_switch_times[id] = \
 206                                                         readl(timer_us); \
 207                                 wmb(); \
 208                 } \
 209                 barrier(); \
 210         } while(0)
 211 #else
 212 #define tegra_cluster_switch_time(flags, id) do {} while(0)
 213 #endif
 214
 215 #ifdef CONFIG_PM_SLEEP
 216 static const char *tegra_suspend_name[TEGRA_MAX_SUSPEND_MODE] = {
 217         [TEGRA_SUSPEND_NONE]    = "none",
 218         [TEGRA_SUSPEND_LP2]     = "lp2",
 219         [TEGRA_SUSPEND_LP1]     = "lp1",
 220         [TEGRA_SUSPEND_LP0]     = "lp0",
 221 };
 222
 223 unsigned long tegra_cpu_power_good_time(void)
 224 {
 225         if (WARN_ON_ONCE(!pdata))
 226                 return 5000;
 227
 228         return pdata->cpu_timer;
 229 }
 230
 231 unsigned long tegra_cpu_power_off_time(void)
 232 {
 233         if (WARN_ON_ONCE(!pdata))
 234                 return 5000;
 235
 236         return pdata->cpu_off_timer;
 237 }
 238
 239 unsigned long tegra_cpu_lp2_min_residency(void)
 240 {
 241         if (WARN_ON_ONCE(!pdata))
 242                 return 2000;
 243
 244         return pdata->cpu_lp2_min_residency;
 245 }
 246
 247 /*
 248  * create_suspend_pgtable
 249  *
 250  * Creates a page table with identity mappings of physical memory and IRAM
 251  * for use when the MMU is off, in addition to all the regular kernel mappings.
 252  */
 253 static __init int create_suspend_pgtable(void)
 254 {
 255         tegra_pgd = pgd_alloc(&init_mm);
 256         if (!tegra_pgd)
 257                 return -ENOMEM;
 258
 259         /* Only identity-map size of lowmem (high_memory - PAGE_OFFSET) */
 260         identity_mapping_add(tegra_pgd, phys_to_virt(PHYS_OFFSET),
 261                 high_memory, 0);
 262         identity_mapping_add(tegra_pgd, IO_IRAM_VIRT,
 263                 IO_IRAM_VIRT + SECTION_SIZE, 0);
 264
 265         /* inner/outer write-back/write-allocate, sharable */
 266         tegra_pgd_phys = (virt_to_phys(tegra_pgd) & PAGE_MASK) | 0x4A;
 267
 268         return 0;
 269 }
 270
 271 /* ensures that sufficient time is passed for a register write to
 272  * serialize into the 32KHz domain */
 273 static void pmc_32kwritel(u32 val, unsigned long offs)
 274 {
 275         writel(val, pmc + offs);
 276         udelay(130);
 277 }
 278
 279 static void set_power_timers(unsigned long us_on, unsigned long us_off,
 280                              long rate)
 281 {
 282         static unsigned long last_us_off = 0;
 283         unsigned long long ticks;
 284         unsigned long long pclk;
 285
 286         if (WARN_ON_ONCE(rate <= 0))
 287                 pclk = 100000000;
 288         else
 289                 pclk = rate;
 290
 291         if ((rate != tegra_last_pclk) || (us_off != last_us_off)) {
 292                 ticks = (us_on * pclk) + 999999ull;
 293                 do_div(ticks, 1000000);
 294                 writel((unsigned long)ticks, pmc + PMC_CPUPWRGOOD_TIMER);
 295
 296                 ticks = (us_off * pclk) + 999999ull;
 297                 do_div(ticks, 1000000);
 298                 writel((unsigned long)ticks, pmc + PMC_CPUPWROFF_TIMER);
 299                 wmb();
 300         }
 301         tegra_last_pclk = pclk;
 302         last_us_off = us_off;
 303 }
 304
 305 /*
 306  * restore_cpu_complex
 307  *
 308  * restores cpu clock setting, clears flow controller
 309  *
 310  * Always called on CPU 0.
 311  */
 312 static void restore_cpu_complex(u32 mode)
 313 {
 314         int cpu = smp_processor_id();
 315         unsigned int reg, policy;
 316
 317         BUG_ON(cpu != 0);
 318
 319 #ifdef CONFIG_SMP
 320         cpu = cpu_logical_map(cpu);
 321 #endif
 322         /* Is CPU complex already running on PLLX? */
 323         reg = readl(clk_rst + CLK_RESET_CCLK_BURST);
 324         policy = (reg >> CLK_RESET_CCLK_BURST_POLICY_SHIFT) & 0xF;
 325
 326         if (policy == CLK_RESET_CCLK_IDLE_POLICY)
 327                 reg = (reg >> CLK_RESET_CCLK_IDLE_POLICY_SHIFT) & 0xF;
 328         else if (policy == CLK_RESET_CCLK_RUN_POLICY)
 329                 reg = (reg >> CLK_RESET_CCLK_RUN_POLICY_SHIFT) & 0xF;
 330         else
 331                 BUG();
 332
 333         if (reg != CLK_RESET_CCLK_BURST_POLICY_PLLX) {
 334                 /* restore PLLX settings if CPU is on different PLL */
 335                 writel(tegra_sctx.pllx_misc, clk_rst + CLK_RESET_PLLX_MISC);
 336                 writel(tegra_sctx.pllx_base, clk_rst + CLK_RESET_PLLX_BASE);
 337
 338                 /* wait for PLL stabilization if PLLX was enabled */
 339                 if (tegra_sctx.pllx_base & (1<<30)) {
 340 #if USE_PLL_LOCK_BITS
 341                         /* Enable lock detector */
 342                         reg = readl(clk_rst + CLK_RESET_PLLX_MISC);
 343                         reg |= 1<<18;
 344                         writel(reg, clk_rst + CLK_RESET_PLLX_MISC);
 345                         while (!(readl(clk_rst + CLK_RESET_PLLX_BASE) &
 346                                  (1<<27)))
 347                                 cpu_relax();
 348
 349                         udelay(PLL_POST_LOCK_DELAY);
 350 #else
 351                         udelay(300);
 352 #endif
 353                 }
 354         }
 355
 356         /* Restore original burst policy setting for calls resulting from CPU
 357            LP2 in idle or system suspend; keep cluster switch prolog setting
 358            intact. */
 359         if (!(mode & TEGRA_POWER_CLUSTER_MASK)) {
 360                 writel(tegra_sctx.cclk_divider, clk_rst +
 361                        CLK_RESET_CCLK_DIVIDER);
 362                 writel(tegra_sctx.cpu_burst, clk_rst +
 363                        CLK_RESET_CCLK_BURST);
 364         }
 365
 366         writel(tegra_sctx.clk_csite_src, clk_rst + CLK_RESET_SOURCE_CSITE);
 367
 368         /* Do not power-gate CPU 0 when flow controlled */
 369         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
 370         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
 371         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
 372         reg &= ~FLOW_CTRL_CSR_ENABLE;           /* clear enable */
 373         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr */
 374         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event */
 375         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
 376
 377         /* If an immedidate cluster switch is being perfomed, restore the
 378            local timer registers. For calls resulting from CPU LP2 in
 379            idle or system suspend, the local timer was shut down and
 380            timekeeping switched over to the global system timer. In this
 381            case keep local timer disabled, and restore only periodic load. */
 382         if (!(mode & (TEGRA_POWER_CLUSTER_MASK |
 383                       TEGRA_POWER_CLUSTER_IMMEDIATE)))
 384                 tegra_sctx.twd.twd_ctrl = 0;
 385         tegra_twd_resume(&tegra_sctx.twd);
 386 }
 387
 388 /*
 389  * suspend_cpu_complex
 390  *
 391  * saves pll state for use by restart_plls, prepares flow controller for
 392  * transition to suspend state
 393  *
 394  * Must always be called on cpu 0.
 395  */
 396 static void suspend_cpu_complex(u32 mode)
 397 {
 398         int cpu = smp_processor_id();
 399         unsigned int reg;
 400         int i;
 401
 402         BUG_ON(cpu != 0);
 403
 404 #ifdef CONFIG_SMP
 405         cpu = cpu_logical_map(cpu);
 406 #endif
 407         /* switch coresite to clk_m, save off original source */
 408         tegra_sctx.clk_csite_src = readl(clk_rst + CLK_RESET_SOURCE_CSITE);
 409         writel(3<<30, clk_rst + CLK_RESET_SOURCE_CSITE);
 410
 411         tegra_sctx.cpu_burst = readl(clk_rst + CLK_RESET_CCLK_BURST);
 412         tegra_sctx.pllx_base = readl(clk_rst + CLK_RESET_PLLX_BASE);
 413         tegra_sctx.pllx_misc = readl(clk_rst + CLK_RESET_PLLX_MISC);
 414         tegra_sctx.pllp_base = readl(clk_rst + CLK_RESET_PLLP_BASE);
 415         tegra_sctx.pllp_outa = readl(clk_rst + CLK_RESET_PLLP_OUTA);
 416         tegra_sctx.pllp_outb = readl(clk_rst + CLK_RESET_PLLP_OUTB);
 417         tegra_sctx.pllp_misc = readl(clk_rst + CLK_RESET_PLLP_MISC);
 418         tegra_sctx.cclk_divider = readl(clk_rst + CLK_RESET_CCLK_DIVIDER);
 419
 420         tegra_twd_suspend(&tegra_sctx.twd);
 421
 422         reg = readl(FLOW_CTRL_CPU_CSR(cpu));
 423         reg &= ~FLOW_CTRL_CSR_WFE_BITMAP;       /* clear wfe bitmap */
 424         reg &= ~FLOW_CTRL_CSR_WFI_BITMAP;       /* clear wfi bitmap */
 425         reg |= FLOW_CTRL_CSR_INTR_FLAG;         /* clear intr flag */
 426         reg |= FLOW_CTRL_CSR_EVENT_FLAG;        /* clear event flag */
 427 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 428         reg |= FLOW_CTRL_CSR_WFE_CPU0 << cpu;   /* enable power gating on wfe */
 429 #else
 430         reg |= FLOW_CTRL_CSR_WFI_CPU0 << cpu;   /* enable power gating on wfi */
 431 #endif
 432         reg |= FLOW_CTRL_CSR_ENABLE;            /* enable power gating */
 433         flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(cpu));
 434
 435         for (i = 0; i < num_possible_cpus(); i++) {
 436                 if (i == cpu)
 437                         continue;
 438                 reg = readl(FLOW_CTRL_CPU_CSR(i));
 439                 reg |= FLOW_CTRL_CSR_EVENT_FLAG;
 440                 reg |= FLOW_CTRL_CSR_INTR_FLAG;
 441                 flowctrl_writel(reg, FLOW_CTRL_CPU_CSR(i));
 442         }
 443
 444         tegra_gic_cpu_disable(true);
 445 }
 446
 447 void tegra_clear_cpu_in_lp2(int cpu)
 448 {
 449         spin_lock(&tegra_lp2_lock);
 450         BUG_ON(!cpumask_test_cpu(cpu, &tegra_in_lp2));
 451         cpumask_clear_cpu(cpu, &tegra_in_lp2);
 452
 453         /* Update the IRAM copy used by the reset handler. The IRAM copy
 454            can't use used directly by cpumask_clear_cpu() because it uses
 455            LDREX/STREX which requires the addressed location to be inner
 456            cacheable and sharable which IRAM isn't. */
 457         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
 458         dsb();
 459
 460         spin_unlock(&tegra_lp2_lock);
 461 }
 462
 463 bool tegra_set_cpu_in_lp2(int cpu)
 464 {
 465         bool last_cpu = false;
 466
 467         spin_lock(&tegra_lp2_lock);
 468         BUG_ON(cpumask_test_cpu(cpu, &tegra_in_lp2));
 469         cpumask_set_cpu(cpu, &tegra_in_lp2);
 470
 471         /* Update the IRAM copy used by the reset handler. The IRAM copy
 472            can't use used directly by cpumask_set_cpu() because it uses
 473            LDREX/STREX which requires the addressed location to be inner
 474            cacheable and sharable which IRAM isn't. */
 475         writel(tegra_in_lp2.bits[0], iram_cpu_lp2_mask);
 476         dsb();
 477
 478         if ((cpu == 0) && cpumask_equal(&tegra_in_lp2, cpu_online_mask))
 479                 last_cpu = true;
 480 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 481         else if (cpu == 1)
 482                 tegra2_cpu_set_resettable_soon();
 483 #endif
 484
 485         spin_unlock(&tegra_lp2_lock);
 486         return last_cpu;
 487 }
 488
 489 static void tegra_sleep_core(enum tegra_suspend_mode mode,
 490                              unsigned long v2p)
 491 {
 492 #ifdef CONFIG_TRUSTED_FOUNDATIONS
 493         if (mode == TEGRA_SUSPEND_LP0) {
 494                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE3,
 495                                   virt_to_phys(tegra_resume));
 496         } else {
 497                 tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
 498                                   (TEGRA_RESET_HANDLER_BASE +
 499                                    tegra_cpu_reset_handler_offset));
 500         }
 501 #endif
 502 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 503         cpu_suspend(v2p, tegra2_sleep_core_finish);
 504 #else
 505         cpu_suspend(v2p, tegra3_sleep_core_finish);
 506 #endif
 507 }
 508
 509 static inline void tegra_sleep_cpu(unsigned long v2p)
 510 {
 511 #ifdef CONFIG_TRUSTED_FOUNDATIONS
 512         tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE4,
 513                           (TEGRA_RESET_HANDLER_BASE +
 514                            tegra_cpu_reset_handler_offset));
 515 #endif
 516         cpu_suspend(v2p, tegra_sleep_cpu_finish);
 517 }
 518
 519 unsigned int tegra_idle_lp2_last(unsigned int sleep_time, unsigned int flags)
 520 {
 521         u32 mode;       /* hardware + software power mode flags */
 522         unsigned int remain;
 523         pgd_t *pgd;
 524
 525         /* Only the last cpu down does the final suspend steps */
 526         mode = readl(pmc + PMC_CTRL);
 527         mode |= TEGRA_POWER_CPU_PWRREQ_OE;
 528         if (pdata->combined_req)
 529                 mode &= ~TEGRA_POWER_PWRREQ_OE;
 530         else
 531                 mode |= TEGRA_POWER_PWRREQ_OE;
 532         mode &= ~TEGRA_POWER_EFFECT_LP0;
 533         writel(mode, pmc + PMC_CTRL);
 534         mode |= flags;
 535
 536         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_start);
 537
 538         /*
 539          * We can use clk_get_rate_all_locked() here, because all other cpus
 540          * are in LP2 state and irqs are disabled
 541          */
 542         if (flags & TEGRA_POWER_CLUSTER_MASK) {
 543                 trace_cpu_cluster(POWER_CPU_CLUSTER_START);
 544                 set_power_timers(pdata->cpu_timer, 0,
 545                         clk_get_rate_all_locked(tegra_pclk));
 546                 if (flags & TEGRA_POWER_CLUSTER_G) {
 547                         /*
 548                          * To reduce the vdd_cpu up latency when LP->G
 549                          * transition. Before the transition, enable
 550                          * the vdd_cpu rail.
 551                          */
 552                         if (is_lp_cluster())
 553                                 writel(UN_PWRGATE_CPU,
 554                                        pmc + PMC_PWRGATE_TOGGLE);
 555                 }
 556                 tegra_cluster_switch_prolog(mode);
 557         } else {
 558                 set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer,
 559                         clk_get_rate_all_locked(tegra_pclk));
 560         }
 561
 562         if (sleep_time)
 563                 tegra_lp2_set_trigger(sleep_time);
 564
 565         cpu_cluster_pm_enter();
 566         suspend_cpu_complex(mode);
 567         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_prolog);
 568         flush_cache_all();
 569         /*
 570          * No need to flush complete L2. Cleaning kernel and IO mappings
 571          * is enough for the LP code sequence that has L2 disabled but
 572          * MMU on.
 573          */
 574         pgd = cpu_get_pgd();
 575         outer_clean_range(__pa(pgd + USER_PTRS_PER_PGD),
 576                           __pa(pgd + PTRS_PER_PGD));
 577         outer_disable();
 578
 579         tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
 580
 581         tegra_init_cache(false);
 582         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_switch);
 583         restore_cpu_complex(mode);
 584         cpu_cluster_pm_exit();
 585
 586         remain = tegra_lp2_timer_remain();
 587         if (sleep_time)
 588                 tegra_lp2_set_trigger(0);
 589
 590         if (flags & TEGRA_POWER_CLUSTER_MASK) {
 591                 tegra_cluster_switch_epilog(mode);
 592                 trace_cpu_cluster(POWER_CPU_CLUSTER_DONE);
 593         }
 594         tegra_cluster_switch_time(flags, tegra_cluster_switch_time_id_epilog);
 595
 596 #if INSTRUMENT_CLUSTER_SWITCH
 597         if (flags & TEGRA_POWER_CLUSTER_MASK) {
 598                 pr_err("%s: prolog %lu us, switch %lu us, epilog %lu us, total %lu us\n",
 599                         is_lp_cluster() ? "G=>LP" : "LP=>G",
 600                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog] -
 601                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start],
 602                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch] -
 603                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_prolog],
 604                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
 605                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_switch],
 606                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_epilog] -
 607                         tegra_cluster_switch_times[tegra_cluster_switch_time_id_start]);
 608         }
 609 #endif
 610         return remain;
 611 }
 612
 613 static int tegra_common_suspend(void)
 614 {
 615         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
 616
 617         tegra_sctx.mc[0] = readl(mc + MC_SECURITY_START);
 618         tegra_sctx.mc[1] = readl(mc + MC_SECURITY_SIZE);
 619         tegra_sctx.mc[2] = readl(mc + MC_SECURITY_CFG2);
 620
 621         /* copy the reset vector and SDRAM shutdown code into IRAM */
 622         memcpy(iram_save, iram_code, iram_save_size);
 623         memcpy(iram_code, tegra_iram_start(), iram_save_size);
 624
 625         return 0;
 626 }
 627
 628 static void tegra_common_resume(void)
 629 {
 630         void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
 631 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 632         void __iomem *emc = IO_ADDRESS(TEGRA_EMC_BASE);
 633 #endif
 634
 635         /* Clear DPD sample */
 636         writel(0x0, pmc + PMC_DPD_SAMPLE);
 637
 638         writel(tegra_sctx.mc[0], mc + MC_SECURITY_START);
 639         writel(tegra_sctx.mc[1], mc + MC_SECURITY_SIZE);
 640         writel(tegra_sctx.mc[2], mc + MC_SECURITY_CFG2);
 641 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 642         /* trigger emc mode write */
 643         writel(EMC_MRW_DEV_NONE, emc + EMC_MRW_0);
 644 #endif
 645         /* clear scratch registers shared by suspend and the reset pen */
 646         writel(0x0, pmc + PMC_SCRATCH39);
 647         writel(0x0, pmc + PMC_SCRATCH41);
 648
 649         /* restore IRAM */
 650         memcpy(iram_code, iram_save, iram_save_size);
 651 }
 652
 653 static int tegra_suspend_prepare_late(void)
 654 {
 655 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 656         disable_irq(INT_SYS_STATS_MON);
 657 #endif
 658         return 0;
 659 }
 660
 661 static void tegra_suspend_wake(void)
 662 {
 663 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 664         enable_irq(INT_SYS_STATS_MON);
 665 #endif
 666 }
 667
 668 static void tegra_pm_set(enum tegra_suspend_mode mode)
 669 {
 670         u32 reg, boot_flag;
 671         unsigned long rate = 32768;
 672
 673         reg = readl(pmc + PMC_CTRL);
 674         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
 675         if (pdata->combined_req)
 676                 reg &= ~TEGRA_POWER_PWRREQ_OE;
 677         else
 678                 reg |= TEGRA_POWER_PWRREQ_OE;
 679         reg &= ~TEGRA_POWER_EFFECT_LP0;
 680
 681         switch (mode) {
 682         case TEGRA_SUSPEND_LP0:
 683 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
 684                 rate = clk_get_rate_all_locked(tegra_pclk);
 685 #endif
 686                 if (pdata->combined_req) {
 687                         reg |= TEGRA_POWER_PWRREQ_OE;
 688                         reg &= ~TEGRA_POWER_CPU_PWRREQ_OE;
 689                 }
 690
 691                 /*
 692                  * LP0 boots through the AVP, which then resumes the AVP to
 693                  * the address in scratch 39, and the cpu to the address in
 694                  * scratch 41 to tegra_resume
 695                  */
 696                 writel(0x0, pmc + PMC_SCRATCH39);
 697
 698                 /* Enable DPD sample to trigger sampling pads data and direction
 699                  * in which pad will be driven during lp0 mode*/
 700                 writel(0x1, pmc + PMC_DPD_SAMPLE);
 701
 702                 /* Set warmboot flag */
 703                 boot_flag = readl(pmc + PMC_SCRATCH0);
 704                 pmc_32kwritel(boot_flag | 1, PMC_SCRATCH0);
 705
 706                 pmc_32kwritel(tegra_lp0_vec_start, PMC_SCRATCH1);
 707
 708                 reg |= TEGRA_POWER_EFFECT_LP0;
 709                 /* No break here. LP0 code falls through to write SCRATCH41 */
 710         case TEGRA_SUSPEND_LP1:
 711                 __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
 712                 wmb();
 713                 break;
 714         case TEGRA_SUSPEND_LP2:
 715                 rate = clk_get_rate(tegra_pclk);
 716                 break;
 717         case TEGRA_SUSPEND_NONE:
 718                 return;
 719         default:
 720                 BUG();
 721         }
 722
 723         set_power_timers(pdata->cpu_timer, pdata->cpu_off_timer, rate);
 724
 725         pmc_32kwritel(reg, PMC_CTRL);
 726 }
 727
 728 static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
 729         [TEGRA_SUSPEND_NONE] = "none",
 730         [TEGRA_SUSPEND_LP2] = "LP2",
 731         [TEGRA_SUSPEND_LP1] = "LP1",
 732         [TEGRA_SUSPEND_LP0] = "LP0",
 733 };
 734
 735 static int tegra_suspend_enter(suspend_state_t state)
 736 {
 737         int ret;
 738         ktime_t delta;
 739         struct timespec ts_entry, ts_exit;
 740
 741         if (pdata && pdata->board_suspend)
 742                 pdata->board_suspend(current_suspend_mode, TEGRA_SUSPEND_BEFORE_PERIPHERAL);
 743
 744         read_persistent_clock(&ts_entry);
 745
 746         ret = tegra_suspend_dram(current_suspend_mode, 0);
 747         if (ret) {
 748                 pr_info("Aborting suspend, tegra_suspend_dram error=%d\n", ret);
 749                 goto abort_suspend;
 750         }
 751
 752         read_persistent_clock(&ts_exit);
 753
 754         if (timespec_compare(&ts_exit, &ts_entry) > 0) {
 755                 delta = timespec_to_ktime(timespec_sub(ts_exit, ts_entry));
 756
 757                 tegra_dvfs_rail_pause(tegra_cpu_rail, delta, false);
 758                 if (current_suspend_mode == TEGRA_SUSPEND_LP0)
 759                         tegra_dvfs_rail_pause(tegra_core_rail, delta, false);
 760                 else
 761                         tegra_dvfs_rail_pause(tegra_core_rail, delta, true);
 762         }
 763
 764 abort_suspend:
 765         if (pdata && pdata->board_resume)
 766                 pdata->board_resume(current_suspend_mode, TEGRA_RESUME_AFTER_PERIPHERAL);
 767
 768         return ret;
 769 }
 770
 771 static void tegra_suspend_check_pwr_stats(void)
 772 {
 773         /* cpus and l2 are powered off later */
 774         unsigned long pwrgate_partid_mask =
 775 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
 776                 (1 << TEGRA_POWERGATE_HEG)      |
 777                 (1 << TEGRA_POWERGATE_SATA)     |
 778                 (1 << TEGRA_POWERGATE_3D1)      |
 779 #endif
 780                 (1 << TEGRA_POWERGATE_3D)       |
 781                 (1 << TEGRA_POWERGATE_VENC)     |
 782                 (1 << TEGRA_POWERGATE_PCIE)     |
 783                 (1 << TEGRA_POWERGATE_VDEC)     |
 784                 (1 << TEGRA_POWERGATE_MPE);
 785
 786         int partid;
 787
 788         for (partid = 0; partid < TEGRA_NUM_POWERGATE; partid++)
 789                 if ((1 << partid) & pwrgate_partid_mask)
 790                         if (tegra_powergate_is_powered(partid))
 791                                 pr_warning("partition %s is left on before suspend\n",
 792                                         tegra_powergate_get_name(partid));
 793
 794         return;
 795 }
 796
 797 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags)
 798 {
 799         int err = 0;
 800         u32 scratch37 = 0xDEADBEEF;
 801
 802         if (WARN_ON(mode <= TEGRA_SUSPEND_NONE ||
 803                 mode >= TEGRA_MAX_SUSPEND_MODE)) {
 804                 err = -ENXIO;
 805                 goto fail;
 806         }
 807
 808         if (tegra_is_voice_call_active()) {
 809                 u32 reg;
 810
 811                 /* backup the current value of scratch37 */
 812                 scratch37 = readl(pmc + PMC_SCRATCH37);
 813
 814                 /* If voice call is active, set a flag in PMC_SCRATCH37 */
 815                 reg = TEGRA_POWER_LP1_AUDIO;
 816                 pmc_32kwritel(reg, PMC_SCRATCH37);
 817         }
 818
 819         if ((mode == TEGRA_SUSPEND_LP0) && !tegra_pm_irq_lp0_allowed()) {
 820                 pr_info("LP0 not used due to unsupported wakeup events\n");
 821                 mode = TEGRA_SUSPEND_LP1;
 822         }
 823
 824         if ((mode == TEGRA_SUSPEND_LP0) || (mode == TEGRA_SUSPEND_LP1))
 825                 tegra_suspend_check_pwr_stats();
 826
 827         tegra_common_suspend();
 828
 829         tegra_pm_set(mode);
 830
 831         if (pdata && pdata->board_suspend)
 832                 pdata->board_suspend(mode, TEGRA_SUSPEND_BEFORE_CPU);
 833
 834         local_fiq_disable();
 835
 836         trace_cpu_suspend(CPU_SUSPEND_START);
 837
 838         if (mode == TEGRA_SUSPEND_LP0) {
 839 #ifdef CONFIG_TEGRA_CLUSTER_CONTROL
 840                 u32 reg = readl(pmc + PMC_SCRATCH4);
 841                 if (is_lp_cluster())
 842                         reg |= PMC_SCRATCH4_WAKE_CLUSTER_MASK;
 843                 else
 844                         reg &= (~PMC_SCRATCH4_WAKE_CLUSTER_MASK);
 845                 pmc_32kwritel(reg, PMC_SCRATCH4);
 846 #endif
 847                 tegra_tsc_suspend();
 848                 tegra_lp0_suspend_mc();
 849                 tegra_cpu_reset_handler_save();
 850                 tegra_tsc_wait_for_suspend();
 851         }
 852         else if (mode == TEGRA_SUSPEND_LP1)
 853                 *iram_cpu_lp1_mask = 1;
 854
 855         suspend_cpu_complex(flags);
 856
 857         flush_cache_all();
 858         outer_flush_all();
 859         outer_disable();
 860
 861         if (mode == TEGRA_SUSPEND_LP2)
 862                 tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
 863         else
 864                 tegra_sleep_core(mode, PHYS_OFFSET - PAGE_OFFSET);
 865
 866         tegra_init_cache(true);
 867
 868         if (mode == TEGRA_SUSPEND_LP0) {
 869                 tegra_tsc_resume();
 870                 tegra_cpu_reset_handler_restore();
 871                 tegra_lp0_resume_mc();
 872                 tegra_tsc_wait_for_resume();
 873         } else if (mode == TEGRA_SUSPEND_LP1)
 874                 *iram_cpu_lp1_mask = 0;
 875
 876         /* if scratch37 was clobbered during LP1, restore it */
 877         if (scratch37 != 0xDEADBEEF)
 878                 pmc_32kwritel(scratch37, PMC_SCRATCH37);
 879
 880         restore_cpu_complex(flags);
 881
 882         /* for platforms where the core & CPU power requests are
 883          * combined as a single request to the PMU, transition out
 884          * of LP0 state by temporarily enabling both requests
 885          */
 886         if (mode == TEGRA_SUSPEND_LP0 && pdata->combined_req) {
 887                 u32 reg;
 888                 reg = readl(pmc + PMC_CTRL);
 889                 reg |= TEGRA_POWER_CPU_PWRREQ_OE;
 890                 pmc_32kwritel(reg, PMC_CTRL);
 891                 reg &= ~TEGRA_POWER_PWRREQ_OE;
 892                 pmc_32kwritel(reg, PMC_CTRL);
 893         }
 894
 895         if (pdata && pdata->board_resume)
 896                 pdata->board_resume(mode, TEGRA_RESUME_AFTER_CPU);
 897
 898         trace_cpu_suspend(CPU_SUSPEND_DONE);
 899
 900         local_fiq_enable();
 901
 902         tegra_common_resume();
 903
 904 fail:
 905         return err;
 906 }
 907
 908 /*
 909  * Function pointers to optional board specific function
 910  */
 911 void (*tegra_deep_sleep)(int);
 912 EXPORT_SYMBOL(tegra_deep_sleep);
 913
 914 static int tegra_suspend_prepare(void)
 915 {
 916         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
 917                 tegra_deep_sleep(1);
 918         return 0;
 919 }
 920
 921 static void tegra_suspend_finish(void)
 922 {
 923         if (pdata && pdata->cpu_resume_boost) {
 924                 int ret = tegra_suspended_target(pdata->cpu_resume_boost);
 925                 pr_info("Tegra: resume CPU boost to %u KHz: %s (%d)\n",
 926                         pdata->cpu_resume_boost, ret ? "Failed" : "OK", ret);
 927         }
 928
 929         if ((current_suspend_mode == TEGRA_SUSPEND_LP0) && tegra_deep_sleep)
 930                 tegra_deep_sleep(0);
 931 }
 932
 933 static const struct platform_suspend_ops tegra_suspend_ops = {
 934         .valid          = suspend_valid_only_mem,
 935         .prepare        = tegra_suspend_prepare,
 936         .finish         = tegra_suspend_finish,
 937         .prepare_late   = tegra_suspend_prepare_late,
 938         .wake           = tegra_suspend_wake,
 939         .enter          = tegra_suspend_enter,
 940 };
 941
 942 static ssize_t suspend_mode_show(struct kobject *kobj,
 943                                         struct kobj_attribute *attr, char *buf)
 944 {
 945         char *start = buf;
 946         char *end = buf + PAGE_SIZE;
 947
 948         start += scnprintf(start, end - start, "%s ", \
 949                                 tegra_suspend_name[current_suspend_mode]);
 950         start += scnprintf(start, end - start, "\n");
 951
 952         return start - buf;
 953 }
 954
 955 static ssize_t suspend_mode_store(struct kobject *kobj,
 956                                         struct kobj_attribute *attr,
 957                                         const char *buf, size_t n)
 958 {
 959         int len;
 960         const char *name_ptr;
 961         enum tegra_suspend_mode new_mode;
 962
 963         name_ptr = buf;
 964         while (*name_ptr && !isspace(*name_ptr))
 965                 name_ptr++;
 966         len = name_ptr - buf;
 967         if (!len)
 968                 goto bad_name;
 969         /* TEGRA_SUSPEND_NONE not allowed as suspend state */
 970         if (!(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_NONE], len))
 971                 || !(strncmp(buf, tegra_suspend_name[TEGRA_SUSPEND_LP2], len))) {
 972                 pr_info("Illegal tegra suspend state: %s\n", buf);
 973                 goto bad_name;
 974         }
 975
 976         for (new_mode = TEGRA_SUSPEND_NONE; \
 977                         new_mode < TEGRA_MAX_SUSPEND_MODE; ++new_mode) {
 978                 if (!strncmp(buf, tegra_suspend_name[new_mode], len)) {
 979                         current_suspend_mode = new_mode;
 980                         break;
 981                 }
 982         }
 983
 984 bad_name:
 985         return n;
 986 }
 987
 988 static struct kobj_attribute suspend_mode_attribute =
 989         __ATTR(mode, 0644, suspend_mode_show, suspend_mode_store);
 990
 991 static struct kobject *suspend_kobj;
 992
 993 static int tegra_pm_enter_suspend(void)
 994 {
 995         pr_info("Entering suspend state %s\n", lp_state[current_suspend_mode]);
 996         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
 997                 tegra_lp0_cpu_mode(true);
 998         return 0;
 999 }
1000
1001 static void tegra_pm_enter_resume(void)
1002 {
1003         if (current_suspend_mode == TEGRA_SUSPEND_LP0)
1004                 tegra_lp0_cpu_mode(false);
1005         pr_info("Exited suspend state %s\n", lp_state[current_suspend_mode]);
1006 }
1007
1008 static struct syscore_ops tegra_pm_enter_syscore_ops = {
1009         .suspend = tegra_pm_enter_suspend,
1010         .resume = tegra_pm_enter_resume,
1011 };
1012
1013 static __init int tegra_pm_enter_syscore_init(void)
1014 {
1015         register_syscore_ops(&tegra_pm_enter_syscore_ops);
1016         return 0;
1017 }
1018 subsys_initcall(tegra_pm_enter_syscore_init);
1019 #endif
1020
1021 void __init tegra_init_suspend(struct tegra_suspend_platform_data *plat)
1022 {
1023         u32 reg;
1024         u32 mode;
1025
1026         tegra_cpu_rail = tegra_dvfs_get_rail_by_name("vdd_cpu");
1027         tegra_core_rail = tegra_dvfs_get_rail_by_name("vdd_core");
1028         tegra_pclk = clk_get_sys(NULL, "pclk");
1029         BUG_ON(IS_ERR(tegra_pclk));
1030         pdata = plat;
1031         (void)reg;
1032         (void)mode;
1033
1034         if (plat->suspend_mode == TEGRA_SUSPEND_LP2)
1035                 plat->suspend_mode = TEGRA_SUSPEND_LP0;
1036
1037 #ifndef CONFIG_PM_SLEEP
1038         if (plat->suspend_mode != TEGRA_SUSPEND_NONE) {
1039                 pr_warning("%s: Suspend requires CONFIG_PM_SLEEP -- "
1040                            "disabling suspend\n", __func__);
1041                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1042         }
1043 #else
1044         if (create_suspend_pgtable() < 0) {
1045                 pr_err("%s: PGD memory alloc failed -- LP0/LP1/LP2 unavailable\n",
1046                                 __func__);
1047                 plat->suspend_mode = TEGRA_SUSPEND_NONE;
1048                 goto fail;
1049         }
1050
1051         if ((tegra_get_chipid() == TEGRA_CHIPID_TEGRA3) &&
1052             (tegra_revision == TEGRA_REVISION_A01) &&
1053             (plat->suspend_mode == TEGRA_SUSPEND_LP0)) {
1054                 /* Tegra 3 A01 supports only LP1 */
1055                 pr_warning("%s: Suspend mode LP0 is not supported on A01 "
1056                            "-- disabling LP0\n", __func__);
1057                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1058         }
1059         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && tegra_lp0_vec_size &&
1060                 tegra_lp0_vec_relocate) {
1061                 unsigned char *reloc_lp0;
1062                 unsigned long tmp;
1063                 void __iomem *orig;
1064                 reloc_lp0 = kmalloc(tegra_lp0_vec_size + L1_CACHE_BYTES - 1,
1065                                         GFP_KERNEL);
1066                 WARN_ON(!reloc_lp0);
1067                 if (!reloc_lp0) {
1068                         pr_err("%s: Failed to allocate reloc_lp0\n",
1069                                 __func__);
1070                         goto out;
1071                 }
1072
1073                 orig = ioremap(tegra_lp0_vec_start, tegra_lp0_vec_size);
1074                 WARN_ON(!orig);
1075                 if (!orig) {
1076                         pr_err("%s: Failed to map tegra_lp0_vec_start %08lx\n",
1077                                 __func__, tegra_lp0_vec_start);
1078                         kfree(reloc_lp0);
1079                         goto out;
1080                 }
1081
1082                 tmp = (unsigned long) reloc_lp0;
1083                 tmp = (tmp + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
1084                 reloc_lp0 = (unsigned char *)tmp;
1085                 memcpy(reloc_lp0, orig, tegra_lp0_vec_size);
1086                 iounmap(orig);
1087                 tegra_lp0_vec_start = virt_to_phys(reloc_lp0);
1088         }
1089
1090 out:
1091         if (plat->suspend_mode == TEGRA_SUSPEND_LP0 && !tegra_lp0_vec_size) {
1092                 pr_warning("%s: Suspend mode LP0 requested, no lp0_vec "
1093                            "provided by bootlader -- disabling LP0\n",
1094                            __func__);
1095                 plat->suspend_mode = TEGRA_SUSPEND_LP1;
1096         }
1097
1098         iram_save_size = tegra_iram_end() - tegra_iram_start();
1099
1100         iram_save = kmalloc(iram_save_size, GFP_KERNEL);
1101         if (!iram_save && (plat->suspend_mode >= TEGRA_SUSPEND_LP1)) {
1102                 pr_err("%s: unable to allocate memory for SDRAM self-refresh "
1103                        "-- LP0/LP1 unavailable\n", __func__);
1104                 plat->suspend_mode = TEGRA_SUSPEND_LP2;
1105         }
1106
1107         /* !!!FIXME!!! THIS IS TEGRA2 ONLY */
1108         /* Initialize scratch registers used for CPU LP2 synchronization */
1109         writel(0, pmc + PMC_SCRATCH37);
1110         writel(0, pmc + PMC_SCRATCH38);
1111         writel(0, pmc + PMC_SCRATCH39);
1112         writel(0, pmc + PMC_SCRATCH41);
1113
1114         /* Always enable CPU power request; just normal polarity is supported */
1115         reg = readl(pmc + PMC_CTRL);
1116         BUG_ON(reg & TEGRA_POWER_CPU_PWRREQ_POLARITY);
1117         reg |= TEGRA_POWER_CPU_PWRREQ_OE;
1118         pmc_32kwritel(reg, PMC_CTRL);
1119
1120         /* Configure core power request and system clock control if LP0
1121            is supported */
1122         __raw_writel(pdata->core_timer, pmc + PMC_COREPWRGOOD_TIMER);
1123         __raw_writel(pdata->core_off_timer, pmc + PMC_COREPWROFF_TIMER);
1124
1125         reg = readl(pmc + PMC_CTRL);
1126
1127         if (!pdata->sysclkreq_high)
1128                 reg |= TEGRA_POWER_SYSCLK_POLARITY;
1129         else
1130                 reg &= ~TEGRA_POWER_SYSCLK_POLARITY;
1131
1132         if (!pdata->corereq_high)
1133                 reg |= TEGRA_POWER_PWRREQ_POLARITY;
1134         else
1135                 reg &= ~TEGRA_POWER_PWRREQ_POLARITY;
1136
1137         /* configure output inverters while the request is tristated */
1138         pmc_32kwritel(reg, PMC_CTRL);
1139
1140         /* now enable requests */
1141         reg |= TEGRA_POWER_SYSCLK_OE;
1142         if (!pdata->combined_req)
1143                 reg |= TEGRA_POWER_PWRREQ_OE;
1144         pmc_32kwritel(reg, PMC_CTRL);
1145
1146         if (pdata->suspend_mode == TEGRA_SUSPEND_LP0)
1147                 tegra_lp0_suspend_init();
1148
1149         suspend_set_ops(&tegra_suspend_ops);
1150
1151         /* Create /sys/power/suspend/type */
1152         suspend_kobj = kobject_create_and_add("suspend", power_kobj);
1153         if (suspend_kobj) {
1154                 if (sysfs_create_file(suspend_kobj, \
1155                                                 &suspend_mode_attribute.attr))
1156                         pr_err("%s: sysfs_create_file suspend type failed!\n",
1157                                                                 __func__);
1158         }
1159
1160         iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
1161         iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
1162 fail:
1163 #endif
1164         if (plat->suspend_mode == TEGRA_SUSPEND_NONE)
1165                 tegra_lp2_in_idle(false);
1166
1167         current_suspend_mode = plat->suspend_mode;
1168 }
1169
1170 unsigned long debug_uart_port_base = 0;
1171 EXPORT_SYMBOL(debug_uart_port_base);
1172
1173 static int tegra_debug_uart_suspend(void)
1174 {
1175         void __iomem *uart;
1176         u32 lcr;
1177
1178         if (!debug_uart_port_base)
1179                 return 0;
1180
1181         uart = IO_ADDRESS(debug_uart_port_base);
1182
1183         lcr = readb(uart + UART_LCR * 4);
1184
1185         tegra_sctx.uart[0] = lcr;
1186         tegra_sctx.uart[1] = readb(uart + UART_MCR * 4);
1187
1188         /* DLAB = 0 */
1189         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1190
1191         tegra_sctx.uart[2] = readb(uart + UART_IER * 4);
1192
1193         /* DLAB = 1 */
1194         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1195
1196         tegra_sctx.uart[3] = readb(uart + UART_DLL * 4);
1197         tegra_sctx.uart[4] = readb(uart + UART_DLM * 4);
1198
1199         writeb(lcr, uart + UART_LCR * 4);
1200
1201         return 0;
1202 }
1203
1204 static void tegra_debug_uart_resume(void)
1205 {
1206         void __iomem *uart;
1207         u32 lcr;
1208
1209         if (!debug_uart_port_base)
1210                 return;
1211
1212         uart = IO_ADDRESS(debug_uart_port_base);
1213
1214         lcr = tegra_sctx.uart[0];
1215
1216         writeb(tegra_sctx.uart[1], uart + UART_MCR * 4);
1217
1218         /* DLAB = 0 */
1219         writeb(lcr & ~UART_LCR_DLAB, uart + UART_LCR * 4);
1220
1221         writeb(UART_FCR_ENABLE_FIFO | UART_FCR_T_TRIG_01 | UART_FCR_R_TRIG_01,
1222                         uart + UART_FCR * 4);
1223
1224         writeb(tegra_sctx.uart[2], uart + UART_IER * 4);
1225
1226         /* DLAB = 1 */
1227         writeb(lcr | UART_LCR_DLAB, uart + UART_LCR * 4);
1228
1229         writeb(tegra_sctx.uart[3], uart + UART_DLL * 4);
1230         writeb(tegra_sctx.uart[4], uart + UART_DLM * 4);
1231
1232         writeb(lcr, uart + UART_LCR * 4);
1233 }
1234
1235 static struct syscore_ops tegra_debug_uart_syscore_ops = {
1236         .suspend = tegra_debug_uart_suspend,
1237         .resume = tegra_debug_uart_resume,
1238 };
1239
1240 struct clk *debug_uart_clk = NULL;
1241 EXPORT_SYMBOL(debug_uart_clk);
1242
1243 void tegra_console_uart_suspend(void)
1244 {
1245         if (console_suspend_enabled && debug_uart_clk)
1246                 clk_disable(debug_uart_clk);
1247 }
1248
1249 void tegra_console_uart_resume(void)
1250 {
1251         if (console_suspend_enabled && debug_uart_clk)
1252                 clk_enable(debug_uart_clk);
1253 }
1254
1255 static int tegra_debug_uart_syscore_init(void)
1256 {
1257         register_syscore_ops(&tegra_debug_uart_syscore_ops);
1258         return 0;
1259 }
1260 arch_initcall(tegra_debug_uart_syscore_init);