ARM: Tegra: Add idle state for MC clock
Prashant Gaikwad [Fri, 22 Mar 2013 05:27:40 +0000 (10:27 +0530)]
This state puts DRAM in self-refresh. It is attached to MC clock
domain, disabled initiallly and will get enabled automatically when
MC clock domain is turned off.

/sys/module/cpuidle_t11x/parameters/stop_mc_clk_in_idle can be used
to control this state.

Bug 1010971

Change-Id: Ia7d70ba1e5a4cdd8ac9cc722de20ed0cd4dabf1a
Signed-off-by: Prashant Gaikwad <pgaikwad@nvidia.com>
Reviewed-on: http://git-master/r/197386
Reviewed-by: Bo Yan <byan@nvidia.com>
Reviewed-by: Bharat Nihalani <bnihalani@nvidia.com>

arch/arm/mach-tegra/cpuidle-t11x.c
arch/arm/mach-tegra/cpuidle.c
arch/arm/mach-tegra/headsmp.S
arch/arm/mach-tegra/pm.c
arch/arm/mach-tegra/pm.h
arch/arm/mach-tegra/reset.h
arch/arm/mach-tegra/sleep-t30.S
arch/arm/mach-tegra/sleep.h

index 53938bf..0878854 100644 (file)
@@ -86,6 +86,9 @@ module_param(slow_cluster_power_gating_noncpu, bool, 0644);
 static uint fast_cluster_power_down_mode __read_mostly;
 module_param(fast_cluster_power_down_mode, uint, 0644);
 
+static bool stop_mc_clk_in_idle __read_mostly = false;
+module_param(stop_mc_clk_in_idle, bool, 0644);
+
 static struct clk *cpu_clk_for_dvfs;
 
 static int pd_exit_latencies[5];
@@ -98,6 +101,7 @@ static struct {
        unsigned long long rail_pd_time;
        unsigned long long c0nc_pg_time;
        unsigned long long c1nc_pg_time;
+       unsigned long long mc_clk_stop_time;
        unsigned int rail_gating_count;
        unsigned int rail_gating_bin[32];
        unsigned int rail_gating_done_count;
@@ -110,6 +114,10 @@ static struct {
        unsigned int c1nc_gating_bin[32];
        unsigned int c1nc_gating_done_count;
        unsigned int c1nc_gating_done_count_bin[32];
+       unsigned int mc_clk_stop_count;
+       unsigned int mc_clk_stop_bin[32];
+       unsigned int mc_clk_stop_done_count;
+       unsigned int mc_clk_stop_done_count_bin[32];
        unsigned int pd_int_count[NR_IRQS];
        unsigned int last_pd_int_count[NR_IRQS];
        unsigned int clk_gating_vmin;
@@ -303,6 +311,16 @@ static bool tegra_cpu_cluster_power_down(struct cpuidle_device *dev,
                }
        }
 
+       if (stop_mc_clk_in_idle && (state->power_usage == 0) &&
+           (request > tegra_mc_clk_stop_min_residency())) {
+               flag |= TEGRA_POWER_STOP_MC_CLK;
+
+               idle_stats.mc_clk_stop_count++;
+               idle_stats.mc_clk_stop_bin[bin]++;
+
+               tegra_mc_clk_prepare();
+       }
+
        if (tegra_idle_power_down_last(sleep_time, flag) == 0)
                sleep_completed = true;
        else {
@@ -312,13 +330,20 @@ static bool tegra_cpu_cluster_power_down(struct cpuidle_device *dev,
 
        clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
        exit_time = ktime_get();
+
+       if (flag & TEGRA_POWER_STOP_MC_CLK)
+               tegra_mc_clk_finish();
+
        if (!is_lp_cluster())
                tegra_dvfs_rail_on(tegra_cpu_rail, exit_time);
 
-       if (flag == TEGRA_POWER_CLUSTER_PART_CRAIL)
+       if (flag & TEGRA_POWER_STOP_MC_CLK)
+               idle_stats.mc_clk_stop_time +=
+                       ktime_to_us(ktime_sub(exit_time, entry_time));
+       else if (flag & TEGRA_POWER_CLUSTER_PART_CRAIL)
                idle_stats.rail_pd_time +=
                        ktime_to_us(ktime_sub(exit_time, entry_time));
-       else if (flag == TEGRA_POWER_CLUSTER_PART_NONCPU) {
+       else if (flag & TEGRA_POWER_CLUSTER_PART_NONCPU) {
                if (is_lp_cluster())
                        idle_stats.c1nc_pg_time +=
                                ktime_to_us(ktime_sub(exit_time, entry_time));
@@ -344,10 +369,13 @@ static bool tegra_cpu_cluster_power_down(struct cpuidle_device *dev,
                state->exit_latency = latency;          /* for idle governor */
                smp_wmb();
 
-               if (flag == TEGRA_POWER_CLUSTER_PART_CRAIL) {
+               if (flag & TEGRA_POWER_STOP_MC_CLK) {
+                       idle_stats.mc_clk_stop_done_count++;
+                       idle_stats.mc_clk_stop_done_count_bin[bin]++;
+               } else if (flag & TEGRA_POWER_CLUSTER_PART_CRAIL) {
                        idle_stats.rail_gating_done_count++;
                        idle_stats.rail_gating_done_count_bin[bin]++;
-               } else if (flag == TEGRA_POWER_CLUSTER_PART_NONCPU) {
+               } else if (flag & TEGRA_POWER_CLUSTER_PART_NONCPU) {
                        if (is_lp_cluster()) {
                                idle_stats.c1nc_gating_done_count++;
                                idle_stats.c1nc_gating_done_count_bin[bin]++;
@@ -675,6 +703,20 @@ int tegra11x_pd_debug_show(struct seq_file *s, void *data)
                        idle_stats.c1nc_gating_done_count_bin[bin] * 100 /
                                idle_stats.c1nc_gating_bin[bin]);
        }
+       seq_printf(s, "\n");
+
+       seq_printf(s, "%19s %8s %8s %8s\n", "", "mc clk stop", "comp", "%");
+       seq_printf(s, "-------------------------------------------------\n");
+       for (bin = 0; bin < 32; bin++) {
+               if (idle_stats.mc_clk_stop_bin[bin] == 0)
+                       continue;
+               seq_printf(s, "%6u - %6u ms: %8u %8u %7u%%\n",
+                       1 << (bin - 1), 1 << bin,
+                       idle_stats.mc_clk_stop_bin[bin],
+                       idle_stats.mc_clk_stop_done_count_bin[bin],
+                       idle_stats.mc_clk_stop_done_count_bin[bin] * 100 /
+                               idle_stats.mc_clk_stop_bin[bin]);
+       }
 
        seq_printf(s, "\n");
        seq_printf(s, "%3s %20s %6s %10s\n",
index be1d022..4af96cd 100644 (file)
@@ -52,6 +52,8 @@
 #include "sleep.h"
 #include "timer.h"
 
+#define DRAM_SELF_REFRESH_EXIT_LATENCY 10000
+
 int tegra_pg_exit_latency;
 static int tegra_pd_power_off_time;
 static unsigned int tegra_pd_min_residency;
@@ -64,7 +66,7 @@ struct cpuidle_driver tegra_idle_driver = {
        .owner = THIS_MODULE,
        .en_core_tk_irqen = 1,
 #ifdef CONFIG_PM_SLEEP
-       .state_count = 2,
+       .state_count = 3,
 #else
        .state_count = 1,
 #endif
@@ -73,11 +75,19 @@ struct cpuidle_driver tegra_idle_driver = {
 #ifdef CONFIG_PM_SLEEP
                [1] = {
                        .enter          = tegra_idle_enter_pd,
-                       .power_usage    = 0,
+                       .power_usage    = 100,
                        .flags          = CPUIDLE_FLAG_TIME_VALID,
                        .name           = "powered-down",
                        .desc           = "CPU power gated",
                },
+               [2] = {
+                       .enter          = tegra_idle_enter_pd,
+                       .power_usage    = 0,
+                       .flags          = CPUIDLE_FLAG_TIME_VALID,
+                       .name           = "mc-clock",
+                       .desc           = "MC clock stop",
+                       .disabled       = true,
+               },
 #endif
        },
 };
@@ -173,6 +183,10 @@ static int tegra_cpuidle_register_device(struct cpuidle_driver *drv,
        dev->cpu = cpu;
        dev->state_count = drv->state_count;
 
+       /* MC clock stop is only for CPU 0 */
+       if (cpu != 0)
+               dev->state_count -= 1;
+
        if (cpuidle_register_device(dev)) {
                pr_err("CPU%u: failed to register idle device\n", cpu);
                kfree(dev);
@@ -217,6 +231,14 @@ static int __init tegra_cpuidle_init(void)
        if (state->target_residency < tegra_pd_min_residency)
                state->target_residency = tegra_pd_min_residency;
 
+       state = &tegra_idle_driver.states[2];
+       state->exit_latency = tegra_cpu_power_good_time() +
+               DRAM_SELF_REFRESH_EXIT_LATENCY;
+       state->target_residency = tegra_cpu_power_off_time() +
+               tegra_cpu_power_good_time() + DRAM_SELF_REFRESH_EXIT_LATENCY;
+       if (state->target_residency < tegra_mc_clk_stop_min_residency())
+               state->target_residency = tegra_mc_clk_stop_min_residency();
+
        tegra_pd_min_residency = tegra_cpu_lp2_min_residency();
        tegra_pg_exit_latency = tegra_cpu_power_good_time();
        tegra_pd_power_off_time = tegra_cpu_power_off_time();
@@ -229,6 +251,9 @@ static int __init tegra_cpuidle_init(void)
 
                per_cpu(tegra_idle_drivers, cpu) = drv;
 
+               if (cpu != 0)
+                       drv->state_count -= 1;
+
                ret = cpuidle_register_cpu_driver(drv, cpu);
                if (ret) {
                        pr_err("CPU%u: CPUidle driver registration failed\n",
index 4a39cc6..aee8270 100644 (file)
@@ -39,8 +39,6 @@
 #define RESET_DATA_PHYS (TEGRA_RESET_HANDLER_BASE \
        + __tegra_cpu_reset_handler_data - __tegra_cpu_reset_handler_start)
 
-#define RESET_DATA(x)  ((TEGRA_RESET_##x)*4)
-
 #ifdef CONFIG_SMP
 /*
  *     tegra_secondary_startup
index f49a8ac..db48da4 100644 (file)
@@ -110,6 +110,7 @@ static DEFINE_SPINLOCK(tegra_lp2_lock);
 static cpumask_t tegra_in_lp2;
 static cpumask_t *iram_cpu_lp2_mask;
 static unsigned long *iram_cpu_lp1_mask;
+static unsigned long *iram_mc_clk_mask;
 static u8 *iram_save;
 static unsigned long iram_save_size;
 static void __iomem *iram_code = IO_ADDRESS(TEGRA_IRAM_CODE_AREA);
@@ -259,6 +260,11 @@ unsigned long tegra_cpu_lp2_min_residency(void)
        return pdata->cpu_lp2_min_residency;
 }
 
+unsigned long tegra_mc_clk_stop_min_residency(void)
+{
+       return 20000;
+}
+
 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
 #define TEGRA_MIN_RESIDENCY_VMIN_FMIN  2000
 #define TEGRA_MIN_RESIDENCY_NCPU_SLOW  2000
@@ -594,6 +600,22 @@ static inline void tegra_sleep_cpu(unsigned long v2p)
        cpu_suspend(v2p, tegra_sleep_cpu_finish);
 }
 
+static inline void tegra_stop_mc_clk(unsigned long v2p)
+{
+#ifdef CONFIG_TRUSTED_FOUNDATIONS
+       outer_flush_range(__pa(&tegra_resume_timestamps_start),
+                         __pa(&tegra_resume_timestamps_end));
+       trace_smc_sleep_core(NVSEC_SMC_START);
+
+       tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE6,
+                         (TEGRA_RESET_HANDLER_BASE +
+                          tegra_cpu_reset_handler_offset));
+
+       trace_smc_sleep_core(NVSEC_SMC_DONE);
+#endif
+       cpu_suspend(v2p, tegra3_stop_mc_clk_finish);
+}
+
 unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
                                        unsigned int flags)
 {
@@ -682,7 +704,10 @@ unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
 #endif
 #endif
 
-       tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
+       if (flags & TEGRA_POWER_STOP_MC_CLK)
+               tegra_stop_mc_clk(PHYS_OFFSET - PAGE_OFFSET);
+       else
+               tegra_sleep_cpu(PHYS_OFFSET - PAGE_OFFSET);
 
 #if defined(CONFIG_ARCH_TEGRA_14x_SOC)
        tegra_init_cache(true);
@@ -733,6 +758,28 @@ unsigned int tegra_idle_power_down_last(unsigned int sleep_time,
        return remain;
 }
 
+void tegra_mc_clk_prepare(void)
+{
+       /* copy the reset vector and SDRAM shutdown code into IRAM */
+       memcpy(iram_save, iram_code, iram_save_size);
+       memcpy(iram_code, tegra_iram_start(), iram_save_size);
+
+       *iram_cpu_lp1_mask = 1;
+       *iram_mc_clk_mask = 1;
+
+       __raw_writel(virt_to_phys(tegra_resume), pmc + PMC_SCRATCH41);
+       wmb();
+}
+
+void tegra_mc_clk_finish(void)
+{
+       /* restore IRAM */
+       memcpy(iram_code, iram_save, iram_save_size);
+       *iram_cpu_lp1_mask = 0;
+       *iram_mc_clk_mask = 0;
+       writel(0, pmc + PMC_SCRATCH41);
+}
+
 static int tegra_common_suspend(void)
 {
        void __iomem *mc = IO_ADDRESS(TEGRA_MC_BASE);
@@ -1410,6 +1457,7 @@ out:
 
        iram_cpu_lp2_mask = tegra_cpu_lp2_mask;
        iram_cpu_lp1_mask = tegra_cpu_lp1_mask;
+       iram_mc_clk_mask = tegra_mc_clk_mask;
 
        /* clear io dpd settings before kernel */
        tegra_bl_io_dpd_cleanup();
index c515e6f..a1da6ba 100644 (file)
@@ -79,6 +79,7 @@ struct tegra_suspend_platform_data {
        unsigned long min_residency_ncpu_fast;
        unsigned long min_residency_crail;
 #endif
+       unsigned long min_residency_mc_clk;
 };
 
 /* clears io dpd settings before kernel code */
@@ -87,6 +88,7 @@ void tegra_bl_io_dpd_cleanup(void);
 unsigned long tegra_cpu_power_good_time(void);
 unsigned long tegra_cpu_power_off_time(void);
 unsigned long tegra_cpu_lp2_min_residency(void);
+unsigned long tegra_mc_clk_stop_min_residency(void);
 #ifdef CONFIG_ARCH_TEGRA_HAS_SYMMETRIC_CPU_PWR_GATE
 unsigned long tegra_min_residency_vmin_fmin(void);
 unsigned long tegra_min_residency_ncpu(void);
@@ -95,6 +97,8 @@ unsigned long tegra_min_residency_crail(void);
 void tegra_clear_cpu_in_pd(int cpu);
 bool tegra_set_cpu_in_pd(int cpu);
 
+void tegra_mc_clk_prepare(void);
+void tegra_mc_clk_finish(void);
 int tegra_suspend_dram(enum tegra_suspend_mode mode, unsigned int flags);
 
 #ifdef CONFIG_ARCH_TEGRA_14x_SOC
index 413fdfb..98cdd36 100644 (file)
 #define TEGRA_RESET_C0_L2_DATA_LATENCY 7
 #define TEGRA_RESET_C1_L2_TAG_LATENCY  8
 #define TEGRA_RESET_C1_L2_DATA_LATENCY 9
-#define TEGRA_RESET_DATA_SIZE          10
+#define TEGRA_RESET_MASK_MC_CLK                10
+#define TEGRA_RESET_DATA_SIZE          11
+
+#define RESET_DATA(x)  ((TEGRA_RESET_##x)*4)
 
 #ifndef __ASSEMBLY__
 
@@ -47,6 +50,10 @@ void tegra_secondary_startup(void);
                ((u32)&__tegra_cpu_reset_handler_data[TEGRA_RESET_MASK_LP1] - \
                 (u32)__tegra_cpu_reset_handler_start))))
 
+#define tegra_mc_clk_mask ((unsigned long *)(IO_ADDRESS(TEGRA_RESET_HANDLER_BASE + \
+               ((u32)&__tegra_cpu_reset_handler_data[TEGRA_RESET_MASK_MC_CLK] - \
+                (u32)__tegra_cpu_reset_handler_start))))
+
 #define tegra_cpu_reset_handler_ptr ((u32 *)(IO_ADDRESS(TEGRA_RESET_HANDLER_BASE + \
                ((u32)__tegra_cpu_reset_handler_data - \
                 (u32)__tegra_cpu_reset_handler_start))))
index 41c524e..385d1de 100644 (file)
@@ -31,6 +31,7 @@
 
 #include "sleep.h"
 #include "clock.h"
+#include "reset.h"
 
 #define EMC_CFG                                0xc
 #define EMC_ADR_CFG                    0x10
@@ -270,6 +271,31 @@ ENTRY(tegra3_sleep_core_finish)
        b       tegra_turn_off_mmu
 ENDPROC(tegra3_sleep_core_finish)
 
+ENTRY(tegra3_stop_mc_clk_finish)
+       mov     r4, r0
+       bl      tegra_flush_cache
+       mov     r0, r4
+       bl      tegra_cpu_exit_coherency
+
+       /* preload all the address literals that are needed for the
+        * CPU power-gating process, to avoid loads from SDRAM (which are
+        * not supported once SDRAM is put into self-refresh.
+        * LP0 / LP1 use physical address, since the MMU needs to be
+        * disabled before putting SDRAM into self-refresh to avoid
+        * memory access due to page table walks */
+       mov32   r4, TEGRA_PMC_BASE
+       mov32   r5, TEGRA_CLK_RESET_BASE
+       mov32   r6, TEGRA_FLOW_CTRL_BASE
+       mov32   r7, TEGRA_TMRUS_BASE
+
+       mov32   r1, tegra3_stop_mc_clk
+       mov32   r2, tegra3_iram_start
+       sub     r1, r1, r2
+       mov32   r2, TEGRA_IRAM_CODE_AREA
+       add     r1, r1, r2
+       b       tegra_turn_off_mmu
+ENDPROC(tegra3_stop_mc_clk_finish)
+
 /*
  * tegra3_sleep_cpu_secondary_finish(unsigned long v2p)
  *
@@ -365,6 +391,12 @@ ENTRY(tegra3_lp1_reset)
        /* the CPU and system bus are running at 32KHz and executing from
         * IRAM when this code is executed; immediately switch to CLKM and
         * enable PLLP, PLLM, PLLC, and PLLX. */
+
+       ldr     r8, [r12, #RESET_DATA(MASK_MC_CLK)]
+       tst     r8, r11         @ if memory clock stopped
+       mov32   r2, TEGRA_PMC_BASE
+       bne     emc_exit_selfrefresh
+
        mov32   r0, TEGRA_CLK_RESET_BASE
 #ifndef CONFIG_TRUSTED_FOUNDATIONS
        /* secure code handles 32KHz to CLKM/OSC clock switch */
@@ -573,11 +605,14 @@ powerup_l2_done:
 lp1_exit_resume:
 #endif
 
+emc_exit_selfrefresh:
 #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || defined(CONFIG_ARCH_TEGRA_14x_SOC)
        mov32   r0, TEGRA_EMC_BASE              @ r0 reserved for emc base
+       add     r5, pc, #tegra3_sdram_pad_save-(.+8)    @ r5 --> saved data
 #endif
 #if defined(CONFIG_ARCH_TEGRA_11x_SOC)
        mov32   r0, TEGRA_EMC0_BASE             @ r0 reserved for emc base
+       add     r5, pc, #tegra11_sdram_pad_save-(.+8)   @ r5 --> saved data
 #endif
 
        ldr     r1, [r5, #0x14] @ PMC_IO_DPD_STATUS
@@ -924,6 +959,9 @@ tegra3_lp0_tear_down_core:
        bl      tegra3_cpu_clk32k
        b       tegra3_enter_sleep
 
+tegra3_stop_mc_clk:
+       bl      tegra3_sdram_self_refresh
+       b       tegra3_enter_sleep
 
 /*
  * tegra3_cpu_clk32k
index 1311acf..a21ea0e 100644 (file)
@@ -47,6 +47,9 @@
 #define TEGRA_POWER_CLUSTER_FORCE_SHIFT        2
 #define TEGRA_POWER_CLUSTER_FORCE_MASK (1 << TEGRA_POWER_CLUSTER_FORCE_SHIFT)
 
+/* Stop MC clock */
+#define TEGRA_POWER_STOP_MC_CLK                (1 << 23)
+
 #define TEGRA_POWER_SDRAM_SELFREFRESH  (1 << 26) /* SDRAM is in self-refresh */
 #define TEGRA_POWER_HOTPLUG_SHUTDOWN   (1 << 27) /* Hotplug shutdown */
 #define TEGRA_POWER_CLUSTER_G          (1 << 28) /* G CPU */
@@ -317,6 +320,7 @@ extern unsigned int lp1_register_core_lowvolt;
 extern unsigned int lp1_register_core_highvolt;
 int tegra3_sleep_core_finish(unsigned long int);
 int tegra3_sleep_cpu_secondary_finish(unsigned long int);
+int tegra3_stop_mc_clk_finish(unsigned long int);
 #endif
 
 #ifdef CONFIG_TRUSTED_FOUNDATIONS