*
* CPU idle driver for Tegra11x CPUs
*
- * Copyright (c) 2012, NVIDIA Corporation.
+ * Copyright (c) 2012-2014, NVIDIA CORPORATION. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
#include <linux/clk.h>
#include <linux/cpu_pm.h>
#include <linux/module.h>
+#include <linux/tegra-soc.h>
+#include <linux/tegra-timer.h>
+#include <linux/tegra-cpuidle.h>
+#include <linux/irqchip/tegra.h>
#include <asm/cacheflush.h>
-#include <asm/hardware/gic.h>
#include <asm/localtimer.h>
#include <asm/suspend.h>
#include <asm/cputype.h>
+#include <asm/psci.h>
-#include <mach/iomap.h>
#include <mach/irqs.h>
-#include <mach/hardware.h>
-#include <trace/events/power.h>
+#include <trace/events/nvpower.h>
-#include "clock.h"
-#include "cpuidle.h"
-#include "dvfs.h"
-#include "fuse.h"
-#include "gic.h"
+#include <linux/platform/tegra/clock.h>
+#include <linux/platform/tegra/dvfs.h>
+#include "iomap.h"
#include "pm.h"
-#include "reset.h"
+#include <linux/platform/tegra/reset.h>
#include "sleep.h"
-#include "timer.h"
-#include "fuse.h"
+#include <linux/tegra_ptm.h>
#define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS \
(IO_ADDRESS(TEGRA_CLK_RESET_BASE) + 0x470)
static uint fast_cluster_power_down_mode __read_mostly;
module_param(fast_cluster_power_down_mode, uint, 0644);
+static bool stop_mc_clk_in_idle __read_mostly = false;
+module_param(stop_mc_clk_in_idle, bool, 0644);
+
static struct clk *cpu_clk_for_dvfs;
+static DEFINE_SPINLOCK(vmin_lock);
+static unsigned long cpu_min_rate;
+
static int pd_exit_latencies[5];
static struct {
unsigned long long rail_pd_time;
unsigned long long c0nc_pg_time;
unsigned long long c1nc_pg_time;
+ unsigned long long mc_clk_stop_time;
unsigned int rail_gating_count;
unsigned int rail_gating_bin[32];
unsigned int rail_gating_done_count;
unsigned int c1nc_gating_bin[32];
unsigned int c1nc_gating_done_count;
unsigned int c1nc_gating_done_count_bin[32];
+ unsigned int mc_clk_stop_count;
+ unsigned int mc_clk_stop_bin[32];
+ unsigned int mc_clk_stop_done_count;
+ unsigned int mc_clk_stop_done_count_bin[32];
unsigned int pd_int_count[NR_IRQS];
unsigned int last_pd_int_count[NR_IRQS];
+ unsigned int clk_gating_vmin;
} idle_stats;
static inline unsigned int time_to_bin(unsigned int time)
to_cpumask(&cpu_power_gating_in_idle)))
return false;
- request = ktime_to_us(tick_nohz_get_sleep_length());
+ if (tegra_cpu_timer_get_remain(&request))
+ return false;
+
if (state->exit_latency != pd_exit_latencies[cpu_number(dev->cpu)]) {
/* possible on the 1st entry after cluster switch*/
state->exit_latency = pd_exit_latencies[cpu_number(dev->cpu)];
#endif
}
-static bool tegra_cpu_cluster_power_down(struct cpuidle_device *dev,
+static int tegra_cpu_cluster_power_down(struct cpuidle_device *dev,
struct cpuidle_state *state, s64 request)
{
ktime_t entry_time;
int bin;
unsigned int flag = 0;
s64 sleep_time;
+ int ret = CPUIDLE_STATE_POWERGATING;
/* LP2 entry time */
entry_time = ktime_get();
}
}
+ if (stop_mc_clk_in_idle && (state->power_usage == 0) &&
+ (request > tegra_mc_clk_stop_min_residency())) {
+ flag |= TEGRA_POWER_STOP_MC_CLK;
+
+ trace_nvmc_clk_stop_rcuidle(NVPOWER_MC_CLK_STOP_ENTRY,
+ sleep_time);
+ idle_stats.mc_clk_stop_count++;
+ idle_stats.mc_clk_stop_bin[bin]++;
+
+ tegra_mc_clk_prepare();
+ }
+
if (tegra_idle_power_down_last(sleep_time, flag) == 0)
sleep_completed = true;
else {
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
exit_time = ktime_get();
+
+ if (flag & TEGRA_POWER_STOP_MC_CLK)
+ tegra_mc_clk_finish();
+
if (!is_lp_cluster())
tegra_dvfs_rail_on(tegra_cpu_rail, exit_time);
- if (flag == TEGRA_POWER_CLUSTER_PART_CRAIL)
+ if (flag & TEGRA_POWER_STOP_MC_CLK) {
+ idle_stats.mc_clk_stop_time +=
+ ktime_to_us(ktime_sub(exit_time, entry_time));
+ ret = CPUIDLE_STATE_MC_CLK_STOP;
+ } else if (flag & TEGRA_POWER_CLUSTER_PART_CRAIL)
idle_stats.rail_pd_time +=
ktime_to_us(ktime_sub(exit_time, entry_time));
- else if (flag == TEGRA_POWER_CLUSTER_PART_NONCPU) {
+ else if (flag & TEGRA_POWER_CLUSTER_PART_NONCPU) {
if (is_lp_cluster())
idle_stats.c1nc_pg_time +=
ktime_to_us(ktime_sub(exit_time, entry_time));
state->exit_latency = latency; /* for idle governor */
smp_wmb();
- if (flag == TEGRA_POWER_CLUSTER_PART_CRAIL) {
+ if (flag & TEGRA_POWER_STOP_MC_CLK) {
+ trace_nvmc_clk_stop_rcuidle(NVPOWER_MC_CLK_STOP_EXIT,
+ sleep_time);
+ idle_stats.mc_clk_stop_done_count++;
+ idle_stats.mc_clk_stop_done_count_bin[bin]++;
+ } else if (flag & TEGRA_POWER_CLUSTER_PART_CRAIL) {
idle_stats.rail_gating_done_count++;
idle_stats.rail_gating_done_count_bin[bin]++;
- } else if (flag == TEGRA_POWER_CLUSTER_PART_NONCPU) {
+ } else if (flag & TEGRA_POWER_CLUSTER_PART_NONCPU) {
if (is_lp_cluster()) {
idle_stats.c1nc_gating_done_count++;
idle_stats.c1nc_gating_done_count_bin[bin]++;
cpu_pm_exit();
- return true;
+ return ret;
+}
+
+static void tegra11x_restore_vmin(void)
+{
+ spin_lock(&vmin_lock);
+
+ if (cpu_min_rate) {
+ idle_stats.clk_gating_vmin++;
+ tegra_cpu_g_idle_rate_exchange(&cpu_min_rate);
+ cpu_min_rate = 0;
+ }
+
+ spin_unlock(&vmin_lock);
}
-static bool tegra_cpu_core_power_down(struct cpuidle_device *dev,
+static int tegra_cpu_core_power_down(struct cpuidle_device *dev,
struct cpuidle_state *state, s64 request)
{
#ifdef CONFIG_SMP
s64 sleep_time;
+ u32 cntp_tval;
+ u32 cntfrq;
ktime_t entry_time;
- struct arch_timer_context timer_context;
bool sleep_completed = false;
struct tick_sched *ts = tick_get_tick_sched(dev->cpu);
-#ifdef CONFIG_TRUSTED_FOUNDATIONS
unsigned int cpu = cpu_number(dev->cpu);
+#if defined(CONFIG_ARM_PSCI)
+ int psci_ret = -EPERM;
+ unsigned long entry_point = TEGRA_RESET_HANDLER_BASE +
+ tegra_cpu_reset_handler_offset;
+ struct psci_power_state pps = {
+ TEGRA_ID_CPU_SUSPEND_STDBY,
+ PSCI_POWER_STATE_TYPE_POWER_DOWN
+ };
#endif
-
- if (!arch_timer_get_state(&timer_context)) {
- if ((timer_context.cntp_ctl & ARCH_TIMER_CTRL_ENABLE) &&
- ~(timer_context.cntp_ctl & ARCH_TIMER_CTRL_IT_MASK)) {
- if (timer_context.cntp_tval <= 0) {
- cpu_do_idle();
- return false;
- }
- request = div_u64((u64)timer_context.cntp_tval *
- 1000000, timer_context.cntfrq);
-#ifdef CONFIG_TEGRA_LP2_CPU_TIMER
- if (request >= state->target_residency) {
- timer_context.cntp_tval -= state->exit_latency *
- (timer_context.cntfrq / 1000000);
- __asm__("mcr p15, 0, %0, c14, c2, 0\n"
- :
- :
- "r"(timer_context.cntp_tval));
- }
-#endif
- }
- }
-
- if (!tegra_is_cpu_wake_timer_ready(dev->cpu) ||
- (request < state->target_residency) ||
- (!ts) || (ts->nohz_mode == NOHZ_MODE_INACTIVE)) {
+ if ((tegra_cpu_timer_get_remain(&request) == -ETIME) ||
+ (request <= state->target_residency) || (!ts) ||
+ (ts->nohz_mode == NOHZ_MODE_INACTIVE) ||
+ !tegra_is_cpu_wake_timer_ready(dev->cpu)) {
/*
* Not enough time left to enter LP2, or wake timer not ready
*/
cpu_do_idle();
- return false;
+ return CPUIDLE_STATE_CLKGATING;
}
+#ifdef CONFIG_TEGRA_LP2_CPU_TIMER
+ cntfrq = tegra_clk_measure_input_freq();
+ cntp_tval = (request - state->exit_latency) * (cntfrq / 1000000);
+ asm volatile("mcr p15, 0, %0, c14, c2, 0" : : "r"(cntp_tval));
+#endif
cpu_pm_enter();
#if !defined(CONFIG_TEGRA_LP2_CPU_TIMER)
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
tegra_pd_set_trigger(sleep_time);
#endif
- idle_stats.tear_down_count[cpu_number(dev->cpu)]++;
+ idle_stats.tear_down_count[cpu]++;
entry_time = ktime_get();
tegra_cpu_wake_by_time[dev->cpu] = ktime_to_us(entry_time) + request;
smp_wmb();
-#ifdef CONFIG_TRUSTED_FOUNDATIONS
+#if defined(CONFIG_ARM_PSCI)
if ((cpu == 0) || (cpu == 4)) {
- tegra_generic_smc(0xFFFFFFFC, 0xFFFFFFE7,
- (TEGRA_RESET_HANDLER_BASE +
- tegra_cpu_reset_handler_offset));
+ if (psci_ops.cpu_suspend) {
+ psci_ret = psci_ops.cpu_suspend(pps, entry_point);
+ while (psci_ret == -EPERM)
+ psci_ret = tegra_restart_prev_smc();
+ }
}
#endif
+
cpu_suspend(0, tegra3_sleep_cpu_secondary_finish);
+ tegra11x_restore_vmin();
+
tegra_cpu_wake_by_time[dev->cpu] = LLONG_MAX;
#ifdef CONFIG_TEGRA_LP2_CPU_TIMER
- if (!arch_timer_get_state(&timer_context))
- sleep_completed = (timer_context.cntp_tval <= 0);
+ asm volatile("mrc p15, 0, %0, c14, c2, 0" : "=r" (cntp_tval));
+ if ((s32)cntp_tval <= 0)
+ sleep_completed = true;
#else
sleep_completed = !tegra_pd_timer_remain();
tegra_pd_set_trigger(0);
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
#endif
sleep_time = ktime_to_us(ktime_sub(ktime_get(), entry_time));
- idle_stats.cpu_pg_time[cpu_number(dev->cpu)] += sleep_time;
+ idle_stats.cpu_pg_time[cpu] += sleep_time;
if (sleep_completed) {
/*
* Stayed in LP2 for the full time until timer expires,
* adjust the exit latency based on measurement
*/
int offset = sleep_time - request;
- int latency = pd_exit_latencies[cpu_number(dev->cpu)] +
+ int latency = pd_exit_latencies[cpu] +
offset / 16;
latency = clamp(latency, 0, 10000);
- pd_exit_latencies[cpu_number(dev->cpu)] = latency;
+ pd_exit_latencies[cpu] = latency;
state->exit_latency = latency; /* for idle governor */
smp_wmb();
}
#endif
cpu_pm_exit();
+ return CPUIDLE_STATE_POWERGATING;
+}
+
+static bool tegra11x_idle_enter_vmin(struct cpuidle_device *dev,
+ struct cpuidle_state *state)
+{
+ int status = -1;
+
+ spin_lock(&vmin_lock);
+
+ cpu_min_rate = 0;
+
+ if (!tegra_rail_off_is_allowed()) {
+ spin_unlock(&vmin_lock);
+ return false;
+ }
+
+ status = tegra_cpu_g_idle_rate_exchange(&cpu_min_rate);
+
+ if (status) {
+ cpu_min_rate = 0;
+ spin_unlock(&vmin_lock);
+ return false;
+ }
+
+ spin_unlock(&vmin_lock);
+
+ cpu_do_idle();
+
+ tegra11x_restore_vmin();
+
return true;
}
-bool tegra11x_idle_power_down(struct cpuidle_device *dev,
+int tegra11x_idle_power_down(struct cpuidle_device *dev,
struct cpuidle_state *state)
{
- bool power_down;
+ int power_down = 0;
bool cpu_gating_only = false;
+ bool clkgt_at_vmin = false;
bool power_gating_cpu_only = true;
- s64 request = ktime_to_us(tick_nohz_get_sleep_length());
+ int status = -1;
+ unsigned long rate;
+ s64 request;
+
+ if (tegra_cpu_timer_get_remain(&request)) {
+ cpu_do_idle();
+ return false;
+ }
tegra_set_cpu_in_pd(dev->cpu);
cpu_gating_only = (((fast_cluster_power_down_mode
<< TEGRA_POWER_CLUSTER_PART_SHIFT)
- & TEGRA_POWER_CLUSTER_PART_MASK) == 0);
+ & TEGRA_POWER_CLUSTER_PART_MASK) == 0) ||
+ (tegra_dvfs_is_dfll_bypass() &&
+ !tegra_dvfs_rail_is_dfll_mode(tegra_cpu_rail));
if (is_lp_cluster()) {
if (slow_cluster_power_gating_noncpu &&
- (request > tegra_min_residency_noncpu()))
+ (request > tegra_min_residency_ncpu()))
power_gating_cpu_only = false;
else
power_gating_cpu_only = true;
- } else if (!cpu_gating_only &&
- (num_online_cpus() == 1) &&
- tegra_rail_off_is_allowed()) {
- if (fast_cluster_power_down_mode &&
- TEGRA_POWER_CLUSTER_FORCE_MASK)
- power_gating_cpu_only = cpu_gating_only;
- else if (request > tegra_min_residency_noncpu())
- power_gating_cpu_only = false;
- else
- power_gating_cpu_only = true;
- } else
- power_gating_cpu_only = true;
+ } else {
+ if (tegra_dvfs_rail_updating(cpu_clk_for_dvfs))
+ clkgt_at_vmin = false;
+ else if (tegra_force_clkgt_at_vmin ==
+ TEGRA_CPUIDLE_FORCE_DO_CLKGT_VMIN)
+ clkgt_at_vmin = true;
+ else if (tegra_force_clkgt_at_vmin ==
+ TEGRA_CPUIDLE_FORCE_NO_CLKGT_VMIN)
+ clkgt_at_vmin = false;
+ else if ((request >= tegra_min_residency_vmin_fmin()) &&
+ ((request < tegra_min_residency_ncpu()) ||
+ cpu_gating_only))
+ clkgt_at_vmin = true;
+
+ if (clkgt_at_vmin)
+ clkgt_at_vmin = tegra11x_idle_enter_vmin(dev, state);
+
+ if (!clkgt_at_vmin && (num_online_cpus() == 1)) {
+ if (!cpu_gating_only && tegra_rail_off_is_allowed()) {
+ if (fast_cluster_power_down_mode &
+ TEGRA_POWER_CLUSTER_FORCE_MASK)
+ power_gating_cpu_only = false;
+ else if (request >
+ tegra_min_residency_ncpu())
+ power_gating_cpu_only = false;
+ else
+ power_gating_cpu_only = true;
+ } else
+ power_gating_cpu_only = true;
+ }
+ }
+
+ if (clkgt_at_vmin) {
+ power_down = true;
+ } else if (!power_gating_cpu_only) {
+ if (is_lp_cluster()) {
+ rate = ULONG_MAX;
+ status = tegra_cpu_lp_idle_rate_exchange(&rate);
+ }
- if (power_gating_cpu_only)
- power_down = tegra_cpu_core_power_down(dev, state, request);
- else
power_down = tegra_cpu_cluster_power_down(dev, state, request);
+ /* restore cpu clock after cluster power ungating */
+ if (status == 0)
+ tegra_cpu_lp_idle_rate_exchange(&rate);
+ } else
+ power_down = tegra_cpu_core_power_down(dev, state, request);
+
+ ptm_power_idle_resume(dev->cpu);
tegra_clear_cpu_in_pd(dev->cpu);
return power_down;
idle_stats.tear_down_count[2],
idle_stats.tear_down_count[3],
idle_stats.tear_down_count[4]);
+ seq_printf(s, "clk gating @ Vmin count: %8u\n",
+ idle_stats.clk_gating_vmin);
seq_printf(s, "rail gating count: %8u\n",
idle_stats.rail_gating_count);
seq_printf(s, "rail gating completed: %8u %7u%%\n",
idle_stats.cpu_wants_pd_time[4]) : 0));
seq_printf(s, "\n");
- seq_printf(s, "rail gating time c0nc gating time c1nc gating time\n");
- seq_printf(s, "%8llu ms %8llu ms %8llu ms\n",
+ seq_printf(s, "rail gating time c0nc gating time " \
+ "c1nc gating time mc_clk gating time\n");
+ seq_printf(s, "%8llu ms %8llu ms " \
+ "%8llu ms %8llu ms\n",
div64_u64(idle_stats.rail_pd_time, 1000),
div64_u64(idle_stats.c0nc_pg_time, 1000),
- div64_u64(idle_stats.c1nc_pg_time, 1000));
- seq_printf(s, "%8d%% %8d%% %8d%%\n",
+ div64_u64(idle_stats.c1nc_pg_time, 1000),
+ div64_u64(idle_stats.mc_clk_stop_time, 1000));
+ seq_printf(s, "%8d%% %8d%% " \
+ "%8d%% %8d%%\n",
(int)(idle_stats.cpu_wants_pd_time[0] ?
div64_u64(idle_stats.rail_pd_time * 100,
idle_stats.cpu_wants_pd_time[0]) : 0),
idle_stats.cpu_wants_pd_time[0]) : 0),
(int)(idle_stats.cpu_wants_pd_time[4] ?
div64_u64(idle_stats.c1nc_pg_time * 100,
+ idle_stats.cpu_wants_pd_time[4]) : 0),
+ (int)(idle_stats.cpu_wants_pd_time[4] ?
+ div64_u64(idle_stats.mc_clk_stop_time * 100,
idle_stats.cpu_wants_pd_time[4]) : 0));
seq_printf(s, "\n");
idle_stats.c1nc_gating_done_count_bin[bin] * 100 /
idle_stats.c1nc_gating_bin[bin]);
}
+ seq_printf(s, "\n");
+
+ seq_printf(s, "%19s %8s %8s %8s\n", "", "mc clk stop", "comp", "%");
+ seq_printf(s, "-------------------------------------------------\n");
+ for (bin = 0; bin < 32; bin++) {
+ if (idle_stats.mc_clk_stop_bin[bin] == 0)
+ continue;
+ seq_printf(s, "%6u - %6u ms: %8u %8u %7u%%\n",
+ 1 << (bin - 1), 1 << bin,
+ idle_stats.mc_clk_stop_bin[bin],
+ idle_stats.mc_clk_stop_done_count_bin[bin],
+ idle_stats.mc_clk_stop_done_count_bin[bin] * 100 /
+ idle_stats.mc_clk_stop_bin[bin]);
+ }
seq_printf(s, "\n");
seq_printf(s, "%3s %20s %6s %10s\n",
#endif
};
+ cpu_min_rate = 0;
cpu_clk_for_dvfs = tegra_get_clock_by_name("cpu_g");
for (i = 0; i < ARRAY_SIZE(pd_exit_latencies); i++)
Code Review / linux-3.10.git / blobdiff