Merge branch 'core-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6.git] / kernel / sched.c
index 854ab41..c535cc4 100644 (file)
@@ -141,7 +141,7 @@ struct rt_prio_array {
 
 struct rt_bandwidth {
        /* nests inside the rq lock: */
-       spinlock_t              rt_runtime_lock;
+       raw_spinlock_t          rt_runtime_lock;
        ktime_t                 rt_period;
        u64                     rt_runtime;
        struct hrtimer          rt_period_timer;
@@ -178,7 +178,7 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
        rt_b->rt_period = ns_to_ktime(period);
        rt_b->rt_runtime = runtime;
 
-       spin_lock_init(&rt_b->rt_runtime_lock);
+       raw_spin_lock_init(&rt_b->rt_runtime_lock);
 
        hrtimer_init(&rt_b->rt_period_timer,
                        CLOCK_MONOTONIC, HRTIMER_MODE_REL);
@@ -200,7 +200,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
        if (hrtimer_active(&rt_b->rt_period_timer))
                return;
 
-       spin_lock(&rt_b->rt_runtime_lock);
+       raw_spin_lock(&rt_b->rt_runtime_lock);
        for (;;) {
                unsigned long delta;
                ktime_t soft, hard;
@@ -217,7 +217,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
                __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
                                HRTIMER_MODE_ABS_PINNED, 0);
        }
-       spin_unlock(&rt_b->rt_runtime_lock);
+       raw_spin_unlock(&rt_b->rt_runtime_lock);
 }
 
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -309,6 +309,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq_var);
  */
 static DEFINE_SPINLOCK(task_group_lock);
 
+#ifdef CONFIG_FAIR_GROUP_SCHED
+
 #ifdef CONFIG_SMP
 static int root_task_group_empty(void)
 {
@@ -316,7 +318,6 @@ static int root_task_group_empty(void)
 }
 #endif
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
 #ifdef CONFIG_USER_SCHED
 # define INIT_TASK_GROUP_LOAD  (2*NICE_0_LOAD)
 #else /* !CONFIG_USER_SCHED */
@@ -469,7 +470,7 @@ struct rt_rq {
        u64 rt_time;
        u64 rt_runtime;
        /* Nests inside the rq lock: */
-       spinlock_t rt_runtime_lock;
+       raw_spinlock_t rt_runtime_lock;
 
 #ifdef CONFIG_RT_GROUP_SCHED
        unsigned long rt_nr_boosted;
@@ -524,7 +525,7 @@ static struct root_domain def_root_domain;
  */
 struct rq {
        /* runqueue lock: */
-       spinlock_t lock;
+       raw_spinlock_t lock;
 
        /*
         * nr_running and cpu_load should be in the same cacheline because
@@ -534,14 +535,12 @@ struct rq {
        #define CPU_LOAD_IDX_MAX 5
        unsigned long cpu_load[CPU_LOAD_IDX_MAX];
 #ifdef CONFIG_NO_HZ
-       unsigned long last_tick_seen;
        unsigned char in_nohz_recently;
 #endif
        /* capture load from *all* tasks on this cpu: */
        struct load_weight load;
        unsigned long nr_load_updates;
        u64 nr_switches;
-       u64 nr_migrations_in;
 
        struct cfs_rq cfs;
        struct rt_rq rt;
@@ -590,6 +589,8 @@ struct rq {
 
        u64 rt_avg;
        u64 age_stamp;
+       u64 idle_stamp;
+       u64 avg_idle;
 #endif
 
        /* calc_load related fields */
@@ -676,6 +677,7 @@ inline void update_rq_clock(struct rq *rq)
 
 /**
  * runqueue_is_locked
+ * @cpu: the processor in question.
  *
  * Returns true if the current cpu runqueue is locked.
  * This interface allows printk to be called with the runqueue lock
@@ -683,7 +685,7 @@ inline void update_rq_clock(struct rq *rq)
  */
 int runqueue_is_locked(int cpu)
 {
-       return spin_is_locked(&cpu_rq(cpu)->lock);
+       return raw_spin_is_locked(&cpu_rq(cpu)->lock);
 }
 
 /*
@@ -770,7 +772,7 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
        if (!sched_feat_names[i])
                return -EINVAL;
 
-       filp->f_pos += cnt;
+       *ppos += cnt;
 
        return cnt;
 }
@@ -812,6 +814,7 @@ const_debug unsigned int sysctl_sched_nr_migrate = 32;
  * default: 0.25ms
  */
 unsigned int sysctl_sched_shares_ratelimit = 250000;
+unsigned int normalized_sysctl_sched_shares_ratelimit = 250000;
 
 /*
  * Inject some fuzzyness into changing the per-cpu group shares
@@ -890,7 +893,7 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
         */
        spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
 
-       spin_unlock_irq(&rq->lock);
+       raw_spin_unlock_irq(&rq->lock);
 }
 
 #else /* __ARCH_WANT_UNLOCKED_CTXSW */
@@ -914,9 +917,9 @@ static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
        next->oncpu = 1;
 #endif
 #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
-       spin_unlock_irq(&rq->lock);
+       raw_spin_unlock_irq(&rq->lock);
 #else
-       spin_unlock(&rq->lock);
+       raw_spin_unlock(&rq->lock);
 #endif
 }
 
@@ -946,10 +949,10 @@ static inline struct rq *__task_rq_lock(struct task_struct *p)
 {
        for (;;) {
                struct rq *rq = task_rq(p);
-               spin_lock(&rq->lock);
+               raw_spin_lock(&rq->lock);
                if (likely(rq == task_rq(p)))
                        return rq;
-               spin_unlock(&rq->lock);
+               raw_spin_unlock(&rq->lock);
        }
 }
 
@@ -966,10 +969,10 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
        for (;;) {
                local_irq_save(*flags);
                rq = task_rq(p);
-               spin_lock(&rq->lock);
+               raw_spin_lock(&rq->lock);
                if (likely(rq == task_rq(p)))
                        return rq;
-               spin_unlock_irqrestore(&rq->lock, *flags);
+               raw_spin_unlock_irqrestore(&rq->lock, *flags);
        }
 }
 
@@ -978,19 +981,19 @@ void task_rq_unlock_wait(struct task_struct *p)
        struct rq *rq = task_rq(p);
 
        smp_mb(); /* spin-unlock-wait is not a full memory barrier */
-       spin_unlock_wait(&rq->lock);
+       raw_spin_unlock_wait(&rq->lock);
 }
 
 static void __task_rq_unlock(struct rq *rq)
        __releases(rq->lock)
 {
-       spin_unlock(&rq->lock);
+       raw_spin_unlock(&rq->lock);
 }
 
 static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
        __releases(rq->lock)
 {
-       spin_unlock_irqrestore(&rq->lock, *flags);
+       raw_spin_unlock_irqrestore(&rq->lock, *flags);
 }
 
 /*
@@ -1003,7 +1006,7 @@ static struct rq *this_rq_lock(void)
 
        local_irq_disable();
        rq = this_rq();
-       spin_lock(&rq->lock);
+       raw_spin_lock(&rq->lock);
 
        return rq;
 }
@@ -1050,10 +1053,10 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
 
        WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
 
-       spin_lock(&rq->lock);
+       raw_spin_lock(&rq->lock);
        update_rq_clock(rq);
        rq->curr->sched_class->task_tick(rq, rq->curr, 1);
-       spin_unlock(&rq->lock);
+       raw_spin_unlock(&rq->lock);
 
        return HRTIMER_NORESTART;
 }
@@ -1066,10 +1069,10 @@ static void __hrtick_start(void *arg)
 {
        struct rq *rq = arg;
 
-       spin_lock(&rq->lock);
+       raw_spin_lock(&rq->lock);
        hrtimer_restart(&rq->hrtick_timer);
        rq->hrtick_csd_pending = 0;
-       spin_unlock(&rq->lock);
+       raw_spin_unlock(&rq->lock);
 }
 
 /*
@@ -1176,7 +1179,7 @@ static void resched_task(struct task_struct *p)
 {
        int cpu;
 
-       assert_spin_locked(&task_rq(p)->lock);
+       assert_raw_spin_locked(&task_rq(p)->lock);
 
        if (test_tsk_need_resched(p))
                return;
@@ -1198,10 +1201,10 @@ static void resched_cpu(int cpu)
        struct rq *rq = cpu_rq(cpu);
        unsigned long flags;
 
-       if (!spin_trylock_irqsave(&rq->lock, flags))
+       if (!raw_spin_trylock_irqsave(&rq->lock, flags))
                return;
        resched_task(cpu_curr(cpu));
-       spin_unlock_irqrestore(&rq->lock, flags);
+       raw_spin_unlock_irqrestore(&rq->lock, flags);
 }
 
 #ifdef CONFIG_NO_HZ
@@ -1270,7 +1273,7 @@ static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
 #else /* !CONFIG_SMP */
 static void resched_task(struct task_struct *p)
 {
-       assert_spin_locked(&task_rq(p)->lock);
+       assert_raw_spin_locked(&task_rq(p)->lock);
        set_tsk_need_resched(p);
 }
 
@@ -1563,11 +1566,7 @@ static unsigned long cpu_avg_load_per_task(int cpu)
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 
-struct update_shares_data {
-       unsigned long rq_weight[NR_CPUS];
-};
-
-static DEFINE_PER_CPU(struct update_shares_data, update_shares_data);
+static __read_mostly unsigned long *update_shares_data;
 
 static void __set_se_shares(struct sched_entity *se, unsigned long shares);
 
@@ -1577,12 +1576,12 @@ static void __set_se_shares(struct sched_entity *se, unsigned long shares);
 static void update_group_shares_cpu(struct task_group *tg, int cpu,
                                    unsigned long sd_shares,
                                    unsigned long sd_rq_weight,
-                                   struct update_shares_data *usd)
+                                   unsigned long *usd_rq_weight)
 {
        unsigned long shares, rq_weight;
        int boost = 0;
 
-       rq_weight = usd->rq_weight[cpu];
+       rq_weight = usd_rq_weight[cpu];
        if (!rq_weight) {
                boost = 1;
                rq_weight = NICE_0_LOAD;
@@ -1601,11 +1600,11 @@ static void update_group_shares_cpu(struct task_group *tg, int cpu,
                struct rq *rq = cpu_rq(cpu);
                unsigned long flags;
 
-               spin_lock_irqsave(&rq->lock, flags);
+               raw_spin_lock_irqsave(&rq->lock, flags);
                tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight;
                tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
                __set_se_shares(tg->se[cpu], shares);
-               spin_unlock_irqrestore(&rq->lock, flags);
+               raw_spin_unlock_irqrestore(&rq->lock, flags);
        }
 }
 
@@ -1616,8 +1615,8 @@ static void update_group_shares_cpu(struct task_group *tg, int cpu,
  */
 static int tg_shares_up(struct task_group *tg, void *data)
 {
-       unsigned long weight, rq_weight = 0, shares = 0;
-       struct update_shares_data *usd;
+       unsigned long weight, rq_weight = 0, sum_weight = 0, shares = 0;
+       unsigned long *usd_rq_weight;
        struct sched_domain *sd = data;
        unsigned long flags;
        int i;
@@ -1626,12 +1625,13 @@ static int tg_shares_up(struct task_group *tg, void *data)
                return 0;
 
        local_irq_save(flags);
-       usd = &__get_cpu_var(update_shares_data);
+       usd_rq_weight = per_cpu_ptr(update_shares_data, smp_processor_id());
 
        for_each_cpu(i, sched_domain_span(sd)) {
                weight = tg->cfs_rq[i]->load.weight;
-               usd->rq_weight[i] = weight;
+               usd_rq_weight[i] = weight;
 
+               rq_weight += weight;
                /*
                 * If there are currently no tasks on the cpu pretend there
                 * is one of average load so that when a new task gets to
@@ -1640,10 +1640,13 @@ static int tg_shares_up(struct task_group *tg, void *data)
                if (!weight)
                        weight = NICE_0_LOAD;
 
-               rq_weight += weight;
+               sum_weight += weight;
                shares += tg->cfs_rq[i]->shares;
        }
 
+       if (!rq_weight)
+               rq_weight = sum_weight;
+
        if ((!shares && rq_weight) || shares > tg->shares)
                shares = tg->shares;
 
@@ -1651,7 +1654,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
                shares = tg->shares;
 
        for_each_cpu(i, sched_domain_span(sd))
-               update_group_shares_cpu(tg, i, shares, rq_weight, usd);
+               update_group_shares_cpu(tg, i, shares, rq_weight, usd_rq_weight);
 
        local_irq_restore(flags);
 
@@ -1703,9 +1706,9 @@ static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
        if (root_task_group_empty())
                return;
 
-       spin_unlock(&rq->lock);
+       raw_spin_unlock(&rq->lock);
        update_shares(sd);
-       spin_lock(&rq->lock);
+       raw_spin_lock(&rq->lock);
 }
 
 static void update_h_load(long cpu)
@@ -1745,7 +1748,7 @@ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
        __acquires(busiest->lock)
        __acquires(this_rq->lock)
 {
-       spin_unlock(&this_rq->lock);
+       raw_spin_unlock(&this_rq->lock);
        double_rq_lock(this_rq, busiest);
 
        return 1;
@@ -1766,14 +1769,16 @@ static int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
 {
        int ret = 0;
 
-       if (unlikely(!spin_trylock(&busiest->lock))) {
+       if (unlikely(!raw_spin_trylock(&busiest->lock))) {
                if (busiest < this_rq) {
-                       spin_unlock(&this_rq->lock);
-                       spin_lock(&busiest->lock);
-                       spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
+                       raw_spin_unlock(&this_rq->lock);
+                       raw_spin_lock(&busiest->lock);
+                       raw_spin_lock_nested(&this_rq->lock,
+                                             SINGLE_DEPTH_NESTING);
                        ret = 1;
                } else
-                       spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
+                       raw_spin_lock_nested(&busiest->lock,
+                                             SINGLE_DEPTH_NESTING);
        }
        return ret;
 }
@@ -1787,7 +1792,7 @@ static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
 {
        if (unlikely(!irqs_disabled())) {
                /* printk() doesn't work good under rq->lock */
-               spin_unlock(&this_rq->lock);
+               raw_spin_unlock(&this_rq->lock);
                BUG_ON(1);
        }
 
@@ -1797,7 +1802,7 @@ static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
 static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
        __releases(busiest->lock)
 {
-       spin_unlock(&busiest->lock);
+       raw_spin_unlock(&busiest->lock);
        lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
 }
 #endif
@@ -1812,6 +1817,22 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
 #endif
 
 static void calc_load_account_active(struct rq *this_rq);
+static void update_sysctl(void);
+static int get_update_sysctl_factor(void);
+
+static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
+{
+       set_task_rq(p, cpu);
+#ifdef CONFIG_SMP
+       /*
+        * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
+        * successfuly executed on another CPU. We must ensure that updates of
+        * per-task data have been completed by this moment.
+        */
+       smp_wmb();
+       task_thread_info(p)->cpu = cpu;
+#endif
+}
 
 #include "sched_stats.h"
 #include "sched_idletask.c"
@@ -1969,20 +1990,6 @@ inline int task_curr(const struct task_struct *p)
        return cpu_curr(task_cpu(p)) == p;
 }
 
-static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
-{
-       set_task_rq(p, cpu);
-#ifdef CONFIG_SMP
-       /*
-        * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
-        * successfuly executed on another CPU. We must ensure that updates of
-        * per-task data have been completed by this moment.
-        */
-       smp_wmb();
-       task_thread_info(p)->cpu = cpu;
-#endif
-}
-
 static inline void check_class_changed(struct rq *rq, struct task_struct *p,
                                       const struct sched_class *prev_class,
                                       int oldprio, int running)
@@ -2004,17 +2011,17 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
 {
        s64 delta;
 
+       if (p->sched_class != &fair_sched_class)
+               return 0;
+
        /*
         * Buddy candidates are cache hot:
         */
-       if (sched_feat(CACHE_HOT_BUDDY) &&
+       if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
                        (&p->se == cfs_rq_of(&p->se)->next ||
                         &p->se == cfs_rq_of(&p->se)->last))
                return 1;
 
-       if (p->sched_class != &fair_sched_class)
-               return 0;
-
        if (sysctl_sched_migration_cost == -1)
                return 1;
        if (sysctl_sched_migration_cost == 0)
@@ -2025,39 +2032,23 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
        return delta < (s64)sysctl_sched_migration_cost;
 }
 
-
 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
 {
-       int old_cpu = task_cpu(p);
-       struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
-       struct cfs_rq *old_cfsrq = task_cfs_rq(p),
-                     *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
-       u64 clock_offset;
-
-       clock_offset = old_rq->clock - new_rq->clock;
+#ifdef CONFIG_SCHED_DEBUG
+       /*
+        * We should never call set_task_cpu() on a blocked task,
+        * ttwu() will sort out the placement.
+        */
+       WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
+                       !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
+#endif
 
        trace_sched_migrate_task(p, new_cpu);
 
-#ifdef CONFIG_SCHEDSTATS
-       if (p->se.wait_start)
-               p->se.wait_start -= clock_offset;
-       if (p->se.sleep_start)
-               p->se.sleep_start -= clock_offset;
-       if (p->se.block_start)
-               p->se.block_start -= clock_offset;
-#endif
-       if (old_cpu != new_cpu) {
+       if (task_cpu(p) != new_cpu) {
                p->se.nr_migrations++;
-               new_rq->nr_migrations_in++;
-#ifdef CONFIG_SCHEDSTATS
-               if (task_hot(p, old_rq->clock, NULL))
-                       schedstat_inc(p, se.nr_forced2_migrations);
-#endif
-               perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS,
-                                    1, 1, NULL, 0);
+               perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);
        }
-       p->se.vruntime -= old_cfsrq->min_vruntime -
-                                        new_cfsrq->min_vruntime;
 
        __set_task_cpu(p, new_cpu);
 }
@@ -2082,12 +2073,10 @@ migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
 
        /*
         * If the task is not on a runqueue (and not running), then
-        * it is sufficient to simply update the task's cpu field.
+        * the next wake-up will properly place the task.
         */
-       if (!p->se.on_rq && !task_running(rq, p)) {
-               set_task_cpu(p, dest_cpu);
+       if (!p->se.on_rq && !task_running(rq, p))
                return 0;
-       }
 
        init_completion(&req->done);
        req->task = p;
@@ -2292,6 +2281,77 @@ void task_oncpu_function_call(struct task_struct *p,
        preempt_enable();
 }
 
+#ifdef CONFIG_SMP
+static int select_fallback_rq(int cpu, struct task_struct *p)
+{
+       int dest_cpu;
+       const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu));
+
+       /* Look for allowed, online CPU in same node. */
+       for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
+               if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
+                       return dest_cpu;
+
+       /* Any allowed, online CPU? */
+       dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
+       if (dest_cpu < nr_cpu_ids)
+               return dest_cpu;
+
+       /* No more Mr. Nice Guy. */
+       if (dest_cpu >= nr_cpu_ids) {
+               rcu_read_lock();
+               cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
+               rcu_read_unlock();
+               dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
+
+               /*
+                * Don't tell them about moving exiting tasks or
+                * kernel threads (both mm NULL), since they never
+                * leave kernel.
+                */
+               if (p->mm && printk_ratelimit()) {
+                       printk(KERN_INFO "process %d (%s) no "
+                              "longer affine to cpu%d\n",
+                              task_pid_nr(p), p->comm, cpu);
+               }
+       }
+
+       return dest_cpu;
+}
+
+/*
+ * Called from:
+ *
+ *  - fork, @p is stable because it isn't on the tasklist yet
+ *
+ *  - exec, @p is unstable, retry loop
+ *
+ *  - wake-up, we serialize ->cpus_allowed against TASK_WAKING so
+ *             we should be good.
+ */
+static inline
+int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
+{
+       int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
+
+       /*
+        * In order not to call set_task_cpu() on a blocking task we need
+        * to rely on ttwu() to place the task on a valid ->cpus_allowed
+        * cpu.
+        *
+        * Since this is common to all placement strategies, this lives here.
+        *
+        * [ this allows ->select_task() to simply return task_cpu(p) and
+        *   not worry about this generic constraint ]
+        */
+       if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
+                    !cpu_online(cpu)))
+               cpu = select_fallback_rq(task_cpu(p), p);
+
+       return cpu;
+}
+#endif
+
 /***
  * try_to_wake_up - wake up a thread
  * @p: the to-be-woken-up thread
@@ -2311,7 +2371,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state,
 {
        int cpu, orig_cpu, this_cpu, success = 0;
        unsigned long flags;
-       struct rq *rq;
+       struct rq *rq, *orig_rq;
 
        if (!sched_feat(SYNC_WAKEUPS))
                wake_flags &= ~WF_SYNC;
@@ -2319,7 +2379,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state,
        this_cpu = get_cpu();
 
        smp_wmb();
-       rq = task_rq_lock(p, &flags);
+       rq = orig_rq = task_rq_lock(p, &flags);
        update_rq_clock(rq);
        if (!(p->state & state))
                goto out;
@@ -2343,13 +2403,19 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state,
        if (task_contributes_to_load(p))
                rq->nr_uninterruptible--;
        p->state = TASK_WAKING;
-       task_rq_unlock(rq, &flags);
 
-       cpu = p->sched_class->select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
+       if (p->sched_class->task_waking)
+               p->sched_class->task_waking(rq, p);
+
+       __task_rq_unlock(rq);
+
+       cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
        if (cpu != orig_cpu)
                set_task_cpu(p, cpu);
 
-       rq = task_rq_lock(p, &flags);
+       rq = __task_rq_lock(p);
+       update_rq_clock(rq);
+
        WARN_ON(p->state != TASK_WAKING);
        cpu = task_cpu(p);
 
@@ -2404,8 +2470,19 @@ out_running:
 
        p->state = TASK_RUNNING;
 #ifdef CONFIG_SMP
-       if (p->sched_class->task_wake_up)
-               p->sched_class->task_wake_up(rq, p);
+       if (p->sched_class->task_woken)
+               p->sched_class->task_woken(rq, p);
+
+       if (unlikely(rq->idle_stamp)) {
+               u64 delta = rq->clock - rq->idle_stamp;
+               u64 max = 2*sysctl_sched_migration_cost;
+
+               if (delta > max)
+                       rq->avg_idle = max;
+               else
+                       update_avg(&rq->avg_idle, delta);
+               rq->idle_stamp = 0;
+       }
 #endif
 out:
        task_rq_unlock(rq, &flags);
@@ -2452,7 +2529,6 @@ static void __sched_fork(struct task_struct *p)
        p->se.avg_overlap               = 0;
        p->se.start_runtime             = 0;
        p->se.avg_wakeup                = sysctl_sched_wakeup_granularity;
-       p->se.avg_running               = 0;
 
 #ifdef CONFIG_SCHEDSTATS
        p->se.wait_start                        = 0;
@@ -2474,7 +2550,6 @@ static void __sched_fork(struct task_struct *p)
        p->se.nr_failed_migrations_running      = 0;
        p->se.nr_failed_migrations_hot          = 0;
        p->se.nr_forced_migrations              = 0;
-       p->se.nr_forced2_migrations             = 0;
 
        p->se.nr_wakeups                        = 0;
        p->se.nr_wakeups_sync                   = 0;
@@ -2495,14 +2570,6 @@ static void __sched_fork(struct task_struct *p)
 #ifdef CONFIG_PREEMPT_NOTIFIERS
        INIT_HLIST_HEAD(&p->preempt_notifiers);
 #endif
-
-       /*
-        * We mark the process as running here, but have not actually
-        * inserted it onto the runqueue yet. This guarantees that
-        * nobody will actually run it, and a signal or other external
-        * event cannot wake it up and insert it on the runqueue either.
-        */
-       p->state = TASK_RUNNING;
 }
 
 /*
@@ -2513,24 +2580,25 @@ void sched_fork(struct task_struct *p, int clone_flags)
        int cpu = get_cpu();
 
        __sched_fork(p);
-
        /*
-        * Make sure we do not leak PI boosting priority to the child.
+        * We mark the process as waking here. This guarantees that
+        * nobody will actually run it, and a signal or other external
+        * event cannot wake it up and insert it on the runqueue either.
         */
-       p->prio = current->normal_prio;
+       p->state = TASK_WAKING;
 
        /*
         * Revert to default priority/policy on fork if requested.
         */
        if (unlikely(p->sched_reset_on_fork)) {
-               if (p->policy == SCHED_FIFO || p->policy == SCHED_RR)
+               if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) {
                        p->policy = SCHED_NORMAL;
-
-               if (p->normal_prio < DEFAULT_PRIO)
-                       p->prio = DEFAULT_PRIO;
+                       p->normal_prio = p->static_prio;
+               }
 
                if (PRIO_TO_NICE(p->static_prio) < 0) {
                        p->static_prio = NICE_TO_PRIO(0);
+                       p->normal_prio = p->static_prio;
                        set_load_weight(p);
                }
 
@@ -2541,11 +2609,19 @@ void sched_fork(struct task_struct *p, int clone_flags)
                p->sched_reset_on_fork = 0;
        }
 
+       /*
+        * Make sure we do not leak PI boosting priority to the child.
+        */
+       p->prio = current->normal_prio;
+
        if (!rt_prio(p->prio))
                p->sched_class = &fair_sched_class;
 
+       if (p->sched_class->task_fork)
+               p->sched_class->task_fork(p);
+
 #ifdef CONFIG_SMP
-       cpu = p->sched_class->select_task_rq(p, SD_BALANCE_FORK, 0);
+       cpu = select_task_rq(p, SD_BALANCE_FORK, 0);
 #endif
        set_task_cpu(p, cpu);
 
@@ -2578,26 +2654,15 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
        struct rq *rq;
 
        rq = task_rq_lock(p, &flags);
-       BUG_ON(p->state != TASK_RUNNING);
+       BUG_ON(p->state != TASK_WAKING);
+       p->state = TASK_RUNNING;
        update_rq_clock(rq);
-
-       p->prio = effective_prio(p);
-
-       if (!p->sched_class->task_new || !current->se.on_rq) {
-               activate_task(rq, p, 0);
-       } else {
-               /*
-                * Let the scheduling class do new task startup
-                * management (if any):
-                */
-               p->sched_class->task_new(rq, p);
-               inc_nr_running(rq);
-       }
+       activate_task(rq, p, 0);
        trace_sched_wakeup_new(rq, p, 1);
        check_preempt_curr(rq, p, WF_FORK);
 #ifdef CONFIG_SMP
-       if (p->sched_class->task_wake_up)
-               p->sched_class->task_wake_up(rq, p);
+       if (p->sched_class->task_woken)
+               p->sched_class->task_woken(rq, p);
 #endif
        task_rq_unlock(rq, &flags);
 }
@@ -2749,10 +2814,10 @@ static inline void post_schedule(struct rq *rq)
        if (rq->post_schedule) {
                unsigned long flags;
 
-               spin_lock_irqsave(&rq->lock, flags);
+               raw_spin_lock_irqsave(&rq->lock, flags);
                if (rq->curr->sched_class->post_schedule)
                        rq->curr->sched_class->post_schedule(rq);
-               spin_unlock_irqrestore(&rq->lock, flags);
+               raw_spin_unlock_irqrestore(&rq->lock, flags);
 
                rq->post_schedule = 0;
        }
@@ -2816,14 +2881,14 @@ context_switch(struct rq *rq, struct task_struct *prev,
         */
        arch_start_context_switch(prev);
 
-       if (unlikely(!mm)) {
+       if (likely(!mm)) {
                next->active_mm = oldmm;
                atomic_inc(&oldmm->mm_count);
                enter_lazy_tlb(oldmm, next);
        } else
                switch_mm(oldmm, mm, next);
 
-       if (unlikely(!prev->mm)) {
+       if (likely(!prev->mm)) {
                prev->active_mm = NULL;
                rq->prev_mm = oldmm;
        }
@@ -2986,15 +3051,6 @@ static void calc_load_account_active(struct rq *this_rq)
 }
 
 /*
- * Externally visible per-cpu scheduler statistics:
- * cpu_nr_migrations(cpu) - number of migrations into that cpu
- */
-u64 cpu_nr_migrations(int cpu)
-{
-       return cpu_rq(cpu)->nr_migrations_in;
-}
-
-/*
  * Update rq->cpu_load[] statistics. This function is usually called every
  * scheduler tick (TICK_NSEC).
  */
@@ -3043,15 +3099,15 @@ static void double_rq_lock(struct rq *rq1, struct rq *rq2)
 {
        BUG_ON(!irqs_disabled());
        if (rq1 == rq2) {
-               spin_lock(&rq1->lock);
+               raw_spin_lock(&rq1->lock);
                __acquire(rq2->lock);   /* Fake it out ;) */
        } else {
                if (rq1 < rq2) {
-                       spin_lock(&rq1->lock);
-                       spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
+                       raw_spin_lock(&rq1->lock);
+                       raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
                } else {
-                       spin_lock(&rq2->lock);
-                       spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
+                       raw_spin_lock(&rq2->lock);
+                       raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
                }
        }
        update_rq_clock(rq1);
@@ -3068,29 +3124,44 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
        __releases(rq1->lock)
        __releases(rq2->lock)
 {
-       spin_unlock(&rq1->lock);
+       raw_spin_unlock(&rq1->lock);
        if (rq1 != rq2)
-               spin_unlock(&rq2->lock);
+               raw_spin_unlock(&rq2->lock);
        else
                __release(rq2->lock);
 }
 
 /*
- * If dest_cpu is allowed for this process, migrate the task to it.
- * This is accomplished by forcing the cpu_allowed mask to only
- * allow dest_cpu, which will force the cpu onto dest_cpu. Then
- * the cpu_allowed mask is restored.
+ * sched_exec - execve() is a valuable balancing opportunity, because at
+ * this point the task has the smallest effective memory and cache footprint.
  */
-static void sched_migrate_task(struct task_struct *p, int dest_cpu)
+void sched_exec(void)
 {
+       struct task_struct *p = current;
        struct migration_req req;
+       int dest_cpu, this_cpu;
        unsigned long flags;
        struct rq *rq;
 
+again:
+       this_cpu = get_cpu();
+       dest_cpu = select_task_rq(p, SD_BALANCE_EXEC, 0);
+       if (dest_cpu == this_cpu) {
+               put_cpu();
+               return;
+       }
+
        rq = task_rq_lock(p, &flags);
+       put_cpu();
+
+       /*
+        * select_task_rq() can race against ->cpus_allowed
+        */
        if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
-           || unlikely(!cpu_active(dest_cpu)))
-               goto out;
+           || unlikely(!cpu_active(dest_cpu))) {
+               task_rq_unlock(rq, &flags);
+               goto again;
+       }
 
        /* force the process onto the specified CPU */
        if (migrate_task(p, dest_cpu, &req)) {
@@ -3105,24 +3176,10 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
 
                return;
        }
-out:
        task_rq_unlock(rq, &flags);
 }
 
 /*
- * sched_exec - execve() is a valuable balancing opportunity, because at
- * this point the task has the smallest effective memory and cache footprint.
- */
-void sched_exec(void)
-{
-       int new_cpu, this_cpu = get_cpu();
-       new_cpu = current->sched_class->select_task_rq(current, SD_BALANCE_EXEC, 0);
-       put_cpu();
-       if (new_cpu != this_cpu)
-               sched_migrate_task(current, new_cpu);
-}
-
-/*
  * pull_task - move a task from a remote runqueue to the local runqueue.
  * Both runqueues must be locked.
  */
@@ -3132,10 +3189,6 @@ static void pull_task(struct rq *src_rq, struct task_struct *p,
        deactivate_task(src_rq, p, 0);
        set_task_cpu(p, this_cpu);
        activate_task(this_rq, p, 0);
-       /*
-        * Note that idle threads have a prio of MAX_PRIO, for this test
-        * to be always true for them.
-        */
        check_preempt_curr(this_rq, p, 0);
 }
 
@@ -3658,6 +3711,7 @@ static void update_group_power(struct sched_domain *sd, int cpu)
 
 /**
  * update_sg_lb_stats - Update sched_group's statistics for load balancing.
+ * @sd: The sched_domain whose statistics are to be updated.
  * @group: sched_group whose statistics are to be updated.
  * @this_cpu: Cpu for which load balance is currently performed.
  * @idle: Idle status of this_cpu
@@ -4093,7 +4147,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
        unsigned long flags;
        struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
 
-       cpumask_setall(cpus);
+       cpumask_copy(cpus, cpu_active_mask);
 
        /*
         * When power savings policy is enabled for the parent domain, idle
@@ -4166,14 +4220,15 @@ redo:
 
                if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {
 
-                       spin_lock_irqsave(&busiest->lock, flags);
+                       raw_spin_lock_irqsave(&busiest->lock, flags);
 
                        /* don't kick the migration_thread, if the curr
                         * task on busiest cpu can't be moved to this_cpu
                         */
                        if (!cpumask_test_cpu(this_cpu,
                                              &busiest->curr->cpus_allowed)) {
-                               spin_unlock_irqrestore(&busiest->lock, flags);
+                               raw_spin_unlock_irqrestore(&busiest->lock,
+                                                           flags);
                                all_pinned = 1;
                                goto out_one_pinned;
                        }
@@ -4183,7 +4238,7 @@ redo:
                                busiest->push_cpu = this_cpu;
                                active_balance = 1;
                        }
-                       spin_unlock_irqrestore(&busiest->lock, flags);
+                       raw_spin_unlock_irqrestore(&busiest->lock, flags);
                        if (active_balance)
                                wake_up_process(busiest->migration_thread);
 
@@ -4256,7 +4311,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
        int all_pinned = 0;
        struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
 
-       cpumask_setall(cpus);
+       cpumask_copy(cpus, cpu_active_mask);
 
        /*
         * When power savings policy is enabled for the parent domain, idle
@@ -4365,10 +4420,10 @@ redo:
                /*
                 * Should not call ttwu while holding a rq->lock
                 */
-               spin_unlock(&this_rq->lock);
+               raw_spin_unlock(&this_rq->lock);
                if (active_balance)
                        wake_up_process(busiest->migration_thread);
-               spin_lock(&this_rq->lock);
+               raw_spin_lock(&this_rq->lock);
 
        } else
                sd->nr_balance_failed = 0;
@@ -4396,6 +4451,11 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
        int pulled_task = 0;
        unsigned long next_balance = jiffies + HZ;
 
+       this_rq->idle_stamp = this_rq->clock;
+
+       if (this_rq->avg_idle < sysctl_sched_migration_cost)
+               return;
+
        for_each_domain(this_cpu, sd) {
                unsigned long interval;
 
@@ -4410,8 +4470,10 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
                interval = msecs_to_jiffies(sd->balance_interval);
                if (time_after(next_balance, sd->last_balance + interval))
                        next_balance = sd->last_balance + interval;
-               if (pulled_task)
+               if (pulled_task) {
+                       this_rq->idle_stamp = 0;
                        break;
+               }
        }
        if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
                /*
@@ -4646,7 +4708,7 @@ int select_nohz_load_balancer(int stop_tick)
                cpumask_set_cpu(cpu, nohz.cpu_mask);
 
                /* time for ilb owner also to sleep */
-               if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
+               if (cpumask_weight(nohz.cpu_mask) == num_active_cpus()) {
                        if (atomic_read(&nohz.load_balancer) == cpu)
                                atomic_set(&nohz.load_balancer, -1);
                        return 0;
@@ -5013,8 +5075,13 @@ static void account_guest_time(struct task_struct *p, cputime_t cputime,
        p->gtime = cputime_add(p->gtime, cputime);
 
        /* Add guest time to cpustat. */
-       cpustat->user = cputime64_add(cpustat->user, tmp);
-       cpustat->guest = cputime64_add(cpustat->guest, tmp);
+       if (TASK_NICE(p) > 0) {
+               cpustat->nice = cputime64_add(cpustat->nice, tmp);
+               cpustat->guest_nice = cputime64_add(cpustat->guest_nice, tmp);
+       } else {
+               cpustat->user = cputime64_add(cpustat->user, tmp);
+               cpustat->guest = cputime64_add(cpustat->guest, tmp);
+       }
 }
 
 /*
@@ -5129,60 +5196,86 @@ void account_idle_ticks(unsigned long ticks)
  * Use precise platform statistics if available:
  */
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
-cputime_t task_utime(struct task_struct *p)
+void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
-       return p->utime;
+       *ut = p->utime;
+       *st = p->stime;
 }
 
-cputime_t task_stime(struct task_struct *p)
+void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
-       return p->stime;
+       struct task_cputime cputime;
+
+       thread_group_cputime(p, &cputime);
+
+       *ut = cputime.utime;
+       *st = cputime.stime;
 }
 #else
-cputime_t task_utime(struct task_struct *p)
+
+#ifndef nsecs_to_cputime
+# define nsecs_to_cputime(__nsecs)     nsecs_to_jiffies(__nsecs)
+#endif
+
+void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
-       clock_t utime = cputime_to_clock_t(p->utime),
-               total = utime + cputime_to_clock_t(p->stime);
-       u64 temp;
+       cputime_t rtime, utime = p->utime, total = cputime_add(utime, p->stime);
 
        /*
         * Use CFS's precise accounting:
         */
-       temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime);
+       rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
 
        if (total) {
-               temp *= utime;
+               u64 temp;
+
+               temp = (u64)(rtime * utime);
                do_div(temp, total);
-       }
-       utime = (clock_t)temp;
+               utime = (cputime_t)temp;
+       } else
+               utime = rtime;
+
+       /*
+        * Compare with previous values, to keep monotonicity:
+        */
+       p->prev_utime = max(p->prev_utime, utime);
+       p->prev_stime = max(p->prev_stime, cputime_sub(rtime, p->prev_utime));
 
-       p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime));
-       return p->prev_utime;
+       *ut = p->prev_utime;
+       *st = p->prev_stime;
 }
 
-cputime_t task_stime(struct task_struct *p)
+/*
+ * Must be called with siglock held.
+ */
+void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
 {
-       clock_t stime;
+       struct signal_struct *sig = p->signal;
+       struct task_cputime cputime;
+       cputime_t rtime, utime, total;
 
-       /*
-        * Use CFS's precise accounting. (we subtract utime from
-        * the total, to make sure the total observed by userspace
-        * grows monotonically - apps rely on that):
-        */
-       stime = nsec_to_clock_t(p->se.sum_exec_runtime) -
-                       cputime_to_clock_t(task_utime(p));
+       thread_group_cputime(p, &cputime);
 
-       if (stime >= 0)
-               p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime));
+       total = cputime_add(cputime.utime, cputime.stime);
+       rtime = nsecs_to_cputime(cputime.sum_exec_runtime);
 
-       return p->prev_stime;
-}
-#endif
+       if (total) {
+               u64 temp;
 
-inline cputime_t task_gtime(struct task_struct *p)
-{
-       return p->gtime;
+               temp = (u64)(rtime * cputime.utime);
+               do_div(temp, total);
+               utime = (cputime_t)temp;
+       } else
+               utime = rtime;
+
+       sig->prev_utime = max(sig->prev_utime, utime);
+       sig->prev_stime = max(sig->prev_stime,
+                             cputime_sub(rtime, sig->prev_utime));
+
+       *ut = sig->prev_utime;
+       *st = sig->prev_stime;
 }
+#endif
 
 /*
  * This function gets called by the timer code, with HZ frequency.
@@ -5199,11 +5292,11 @@ void scheduler_tick(void)
 
        sched_clock_tick();
 
-       spin_lock(&rq->lock);
+       raw_spin_lock(&rq->lock);
        update_rq_clock(rq);
        update_cpu_load(rq);
        curr->sched_class->task_tick(rq, curr, 0);
-       spin_unlock(&rq->lock);
+       raw_spin_unlock(&rq->lock);
 
        perf_event_task_tick(curr, cpu);
 
@@ -5317,13 +5410,14 @@ static inline void schedule_debug(struct task_struct *prev)
 #endif
 }
 
-static void put_prev_task(struct rq *rq, struct task_struct *p)
+static void put_prev_task(struct rq *rq, struct task_struct *prev)
 {
-       u64 runtime = p->se.sum_exec_runtime - p->se.prev_sum_exec_runtime;
+       if (prev->state == TASK_RUNNING) {
+               u64 runtime = prev->se.sum_exec_runtime;
 
-       update_avg(&p->se.avg_running, runtime);
+               runtime -= prev->se.prev_sum_exec_runtime;
+               runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
 
-       if (p->state == TASK_RUNNING) {
                /*
                 * In order to avoid avg_overlap growing stale when we are
                 * indeed overlapping and hence not getting put to sleep, grow
@@ -5333,12 +5427,9 @@ static void put_prev_task(struct rq *rq, struct task_struct *p)
                 * correlates to the amount of cache footprint a task can
                 * build up.
                 */
-               runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
-               update_avg(&p->se.avg_overlap, runtime);
-       } else {
-               update_avg(&p->se.avg_running, 0);
+               update_avg(&prev->se.avg_overlap, runtime);
        }
-       p->sched_class->put_prev_task(rq, p);
+       prev->sched_class->put_prev_task(rq, prev);
 }
 
 /*
@@ -5399,7 +5490,7 @@ need_resched_nonpreemptible:
        if (sched_feat(HRTICK))
                hrtick_clear(rq);
 
-       spin_lock_irq(&rq->lock);
+       raw_spin_lock_irq(&rq->lock);
        update_rq_clock(rq);
        clear_tsk_need_resched(prev);
 
@@ -5435,7 +5526,7 @@ need_resched_nonpreemptible:
                cpu = smp_processor_id();
                rq = cpu_rq(cpu);
        } else
-               spin_unlock_irq(&rq->lock);
+               raw_spin_unlock_irq(&rq->lock);
 
        post_schedule(rq);
 
@@ -5448,7 +5539,7 @@ need_resched_nonpreemptible:
 }
 EXPORT_SYMBOL(schedule);
 
-#ifdef CONFIG_SMP
+#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
 /*
  * Look out! "owner" is an entirely speculative pointer
  * access and not reliable.
@@ -5852,14 +5943,15 @@ EXPORT_SYMBOL(wait_for_completion_killable);
  */
 bool try_wait_for_completion(struct completion *x)
 {
+       unsigned long flags;
        int ret = 1;
 
-       spin_lock_irq(&x->wait.lock);
+       spin_lock_irqsave(&x->wait.lock, flags);
        if (!x->done)
                ret = 0;
        else
                x->done--;
-       spin_unlock_irq(&x->wait.lock);
+       spin_unlock_irqrestore(&x->wait.lock, flags);
        return ret;
 }
 EXPORT_SYMBOL(try_wait_for_completion);
@@ -5874,12 +5966,13 @@ EXPORT_SYMBOL(try_wait_for_completion);
  */
 bool completion_done(struct completion *x)
 {
+       unsigned long flags;
        int ret = 1;
 
-       spin_lock_irq(&x->wait.lock);
+       spin_lock_irqsave(&x->wait.lock, flags);
        if (!x->done)
                ret = 0;
-       spin_unlock_irq(&x->wait.lock);
+       spin_unlock_irqrestore(&x->wait.lock, flags);
        return ret;
 }
 EXPORT_SYMBOL(completion_done);
@@ -6142,22 +6235,14 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
        BUG_ON(p->se.on_rq);
 
        p->policy = policy;
-       switch (p->policy) {
-       case SCHED_NORMAL:
-       case SCHED_BATCH:
-       case SCHED_IDLE:
-               p->sched_class = &fair_sched_class;
-               break;
-       case SCHED_FIFO:
-       case SCHED_RR:
-               p->sched_class = &rt_sched_class;
-               break;
-       }
-
        p->rt_priority = prio;
        p->normal_prio = normal_prio(p);
        /* we are holding p->pi_lock already */
        p->prio = rt_mutex_getprio(p);
+       if (rt_prio(p->prio))
+               p->sched_class = &rt_sched_class;
+       else
+               p->sched_class = &fair_sched_class;
        set_load_weight(p);
 }
 
@@ -6272,7 +6357,7 @@ recheck:
         * make sure no PI-waiters arrive (or leave) while we are
         * changing the priority of the task:
         */
-       spin_lock_irqsave(&p->pi_lock, flags);
+       raw_spin_lock_irqsave(&p->pi_lock, flags);
        /*
         * To be able to change p->policy safely, the apropriate
         * runqueue lock must be held.
@@ -6282,7 +6367,7 @@ recheck:
        if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
                policy = oldpolicy = -1;
                __task_rq_unlock(rq);
-               spin_unlock_irqrestore(&p->pi_lock, flags);
+               raw_spin_unlock_irqrestore(&p->pi_lock, flags);
                goto recheck;
        }
        update_rq_clock(rq);
@@ -6306,7 +6391,7 @@ recheck:
                check_class_changed(rq, p, prev_class, oldprio, running);
        }
        __task_rq_unlock(rq);
-       spin_unlock_irqrestore(&p->pi_lock, flags);
+       raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 
        rt_mutex_adjust_pi(p);
 
@@ -6406,7 +6491,7 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
                return -EINVAL;
 
        retval = -ESRCH;
-       read_lock(&tasklist_lock);
+       rcu_read_lock();
        p = find_process_by_pid(pid);
        if (p) {
                retval = security_task_getscheduler(p);
@@ -6414,7 +6499,7 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
                        retval = p->policy
                                | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
        }
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
        return retval;
 }
 
@@ -6432,7 +6517,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
        if (!param || pid < 0)
                return -EINVAL;
 
-       read_lock(&tasklist_lock);
+       rcu_read_lock();
        p = find_process_by_pid(pid);
        retval = -ESRCH;
        if (!p)
@@ -6443,7 +6528,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
                goto out_unlock;
 
        lp.sched_priority = p->rt_priority;
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
 
        /*
         * This one might sleep, we cannot do it with a spinlock held ...
@@ -6453,7 +6538,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
        return retval;
 
 out_unlock:
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
        return retval;
 }
 
@@ -6464,22 +6549,18 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
        int retval;
 
        get_online_cpus();
-       read_lock(&tasklist_lock);
+       rcu_read_lock();
 
        p = find_process_by_pid(pid);
        if (!p) {
-               read_unlock(&tasklist_lock);
+               rcu_read_unlock();
                put_online_cpus();
                return -ESRCH;
        }
 
-       /*
-        * It is not safe to call set_cpus_allowed with the
-        * tasklist_lock held. We will bump the task_struct's
-        * usage count and then drop tasklist_lock.
-        */
+       /* Prevent p going away */
        get_task_struct(p);
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
 
        if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
                retval = -ENOMEM;
@@ -6560,10 +6641,12 @@ SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
 long sched_getaffinity(pid_t pid, struct cpumask *mask)
 {
        struct task_struct *p;
+       unsigned long flags;
+       struct rq *rq;
        int retval;
 
        get_online_cpus();
-       read_lock(&tasklist_lock);
+       rcu_read_lock();
 
        retval = -ESRCH;
        p = find_process_by_pid(pid);
@@ -6574,10 +6657,12 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
        if (retval)
                goto out_unlock;
 
+       rq = task_rq_lock(p, &flags);
        cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
+       task_rq_unlock(rq, &flags);
 
 out_unlock:
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
        put_online_cpus();
 
        return retval;
@@ -6632,7 +6717,7 @@ SYSCALL_DEFINE0(sched_yield)
         */
        __release(rq->lock);
        spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
-       _raw_spin_unlock(&rq->lock);
+       do_raw_spin_unlock(&rq->lock);
        preempt_enable_no_resched();
 
        schedule();
@@ -6720,9 +6805,6 @@ EXPORT_SYMBOL(yield);
 /*
  * This task is about to go to sleep on IO. Increment rq->nr_iowait so
  * that process accounting knows that this is a task in IO wait state.
- *
- * But don't do that if it is a deliberate, throttling IO wait (this task
- * has set its backing_dev_info: the queue against which it should throttle)
  */
 void __sched io_schedule(void)
 {
@@ -6815,6 +6897,8 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
 {
        struct task_struct *p;
        unsigned int time_slice;
+       unsigned long flags;
+       struct rq *rq;
        int retval;
        struct timespec t;
 
@@ -6822,7 +6906,7 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
                return -EINVAL;
 
        retval = -ESRCH;
-       read_lock(&tasklist_lock);
+       rcu_read_lock();
        p = find_process_by_pid(pid);
        if (!p)
                goto out_unlock;
@@ -6831,15 +6915,17 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
        if (retval)
                goto out_unlock;
 
-       time_slice = p->sched_class->get_rr_interval(p);
+       rq = task_rq_lock(p, &flags);
+       time_slice = p->sched_class->get_rr_interval(rq, p);
+       task_rq_unlock(rq, &flags);
 
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
        jiffies_to_timespec(time_slice, &t);
        retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
        return retval;
 
 out_unlock:
-       read_unlock(&tasklist_lock);
+       rcu_read_unlock();
        return retval;
 }
 
@@ -6905,7 +6991,7 @@ void show_state_filter(unsigned long state_filter)
        /*
         * Only show locks if all tasks are dumped:
         */
-       if (state_filter == -1)
+       if (!state_filter)
                debug_show_all_locks();
 }
 
@@ -6927,12 +7013,12 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
        struct rq *rq = cpu_rq(cpu);
        unsigned long flags;
 
-       spin_lock_irqsave(&rq->lock, flags);
+       raw_spin_lock_irqsave(&rq->lock, flags);
 
        __sched_fork(idle);
+       idle->state = TASK_RUNNING;
        idle->se.exec_start = sched_clock();
 
-       idle->prio = idle->normal_prio = MAX_PRIO;
        cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
        __set_task_cpu(idle, cpu);
 
@@ -6940,7 +7026,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
 #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
        idle->oncpu = 1;
 #endif
-       spin_unlock_irqrestore(&rq->lock, flags);
+       raw_spin_unlock_irqrestore(&rq->lock, flags);
 
        /* Set the preempt count _outside_ the spinlocks! */
 #if defined(CONFIG_PREEMPT)
@@ -6973,22 +7059,43 @@ cpumask_var_t nohz_cpu_mask;
  *
  * This idea comes from the SD scheduler of Con Kolivas:
  */
-static inline void sched_init_granularity(void)
+static int get_update_sysctl_factor(void)
 {
-       unsigned int factor = 1 + ilog2(num_online_cpus());
-       const unsigned long limit = 200000000;
+       unsigned int cpus = min_t(int, num_online_cpus(), 8);
+       unsigned int factor;
 
-       sysctl_sched_min_granularity *= factor;
-       if (sysctl_sched_min_granularity > limit)
-               sysctl_sched_min_granularity = limit;
+       switch (sysctl_sched_tunable_scaling) {
+       case SCHED_TUNABLESCALING_NONE:
+               factor = 1;
+               break;
+       case SCHED_TUNABLESCALING_LINEAR:
+               factor = cpus;
+               break;
+       case SCHED_TUNABLESCALING_LOG:
+       default:
+               factor = 1 + ilog2(cpus);
+               break;
+       }
 
-       sysctl_sched_latency *= factor;
-       if (sysctl_sched_latency > limit)
-               sysctl_sched_latency = limit;
+       return factor;
+}
+
+static void update_sysctl(void)
+{
+       unsigned int factor = get_update_sysctl_factor();
 
-       sysctl_sched_wakeup_granularity *= factor;
+#define SET_SYSCTL(name) \
+       (sysctl_##name = (factor) * normalized_sysctl_##name)
+       SET_SYSCTL(sched_min_granularity);
+       SET_SYSCTL(sched_latency);
+       SET_SYSCTL(sched_wakeup_granularity);
+       SET_SYSCTL(sched_shares_ratelimit);
+#undef SET_SYSCTL
+}
 
-       sysctl_sched_shares_ratelimit *= factor;
+static inline void sched_init_granularity(void)
+{
+       update_sysctl();
 }
 
 #ifdef CONFIG_SMP
@@ -7024,8 +7131,24 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
        struct rq *rq;
        int ret = 0;
 
+       /*
+        * Since we rely on wake-ups to migrate sleeping tasks, don't change
+        * the ->cpus_allowed mask from under waking tasks, which would be
+        * possible when we change rq->lock in ttwu(), so synchronize against
+        * TASK_WAKING to avoid that.
+        */
+again:
+       while (p->state == TASK_WAKING)
+               cpu_relax();
+
        rq = task_rq_lock(p, &flags);
-       if (!cpumask_intersects(new_mask, cpu_online_mask)) {
+
+       if (p->state == TASK_WAKING) {
+               task_rq_unlock(rq, &flags);
+               goto again;
+       }
+
+       if (!cpumask_intersects(new_mask, cpu_active_mask)) {
                ret = -EINVAL;
                goto out;
        }
@@ -7047,7 +7170,7 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
        if (cpumask_test_cpu(task_cpu(p), new_mask))
                goto out;
 
-       if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
+       if (migrate_task(p, cpumask_any_and(cpu_active_mask, new_mask), &req)) {
                /* Need help from migration thread: drop lock and wait. */
                struct task_struct *mt = rq->migration_thread;
 
@@ -7080,7 +7203,7 @@ EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
 static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
 {
        struct rq *rq_dest, *rq_src;
-       int ret = 0, on_rq;
+       int ret = 0;
 
        if (unlikely(!cpu_active(dest_cpu)))
                return ret;
@@ -7096,12 +7219,13 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
        if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
                goto fail;
 
-       on_rq = p->se.on_rq;
-       if (on_rq)
+       /*
+        * If we're not on a rq, the next wake-up will ensure we're
+        * placed properly.
+        */
+       if (p->se.on_rq) {
                deactivate_task(rq_src, p, 0);
-
-       set_task_cpu(p, dest_cpu);
-       if (on_rq) {
+               set_task_cpu(p, dest_cpu);
                activate_task(rq_dest, p, 0);
                check_preempt_curr(rq_dest, p, 0);
        }
@@ -7136,10 +7260,10 @@ static int migration_thread(void *data)
                struct migration_req *req;
                struct list_head *head;
 
-               spin_lock_irq(&rq->lock);
+               raw_spin_lock_irq(&rq->lock);
 
                if (cpu_is_offline(cpu)) {
-                       spin_unlock_irq(&rq->lock);
+                       raw_spin_unlock_irq(&rq->lock);
                        break;
                }
 
@@ -7151,7 +7275,7 @@ static int migration_thread(void *data)
                head = &rq->migration_queue;
 
                if (list_empty(head)) {
-                       spin_unlock_irq(&rq->lock);
+                       raw_spin_unlock_irq(&rq->lock);
                        schedule();
                        set_current_state(TASK_INTERRUPTIBLE);
                        continue;
@@ -7160,14 +7284,14 @@ static int migration_thread(void *data)
                list_del_init(head->next);
 
                if (req->task != NULL) {
-                       spin_unlock(&rq->lock);
+                       raw_spin_unlock(&rq->lock);
                        __migrate_task(req->task, cpu, req->dest_cpu);
                } else if (likely(cpu == (badcpu = smp_processor_id()))) {
                        req->dest_cpu = RCU_MIGRATION_GOT_QS;
-                       spin_unlock(&rq->lock);
+                       raw_spin_unlock(&rq->lock);
                } else {
                        req->dest_cpu = RCU_MIGRATION_MUST_SYNC;
-                       spin_unlock(&rq->lock);
+                       raw_spin_unlock(&rq->lock);
                        WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu);
                }
                local_irq_enable();
@@ -7197,37 +7321,10 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
 static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
 {
        int dest_cpu;
-       const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
 
 again:
-       /* Look for allowed, online CPU in same node. */
-       for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
-               if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
-                       goto move;
-
-       /* Any allowed, online CPU? */
-       dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
-       if (dest_cpu < nr_cpu_ids)
-               goto move;
-
-       /* No more Mr. Nice Guy. */
-       if (dest_cpu >= nr_cpu_ids) {
-               cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
-               dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
+       dest_cpu = select_fallback_rq(dead_cpu, p);
 
-               /*
-                * Don't tell them about moving exiting tasks or
-                * kernel threads (both mm NULL), since they never
-                * leave kernel.
-                */
-               if (p->mm && printk_ratelimit()) {
-                       printk(KERN_INFO "process %d (%s) no "
-                              "longer affine to cpu%d\n",
-                              task_pid_nr(p), p->comm, dead_cpu);
-               }
-       }
-
-move:
        /* It can have affinity changed while we were choosing. */
        if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
                goto again;
@@ -7242,7 +7339,7 @@ move:
  */
 static void migrate_nr_uninterruptible(struct rq *rq_src)
 {
-       struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
+       struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
        unsigned long flags;
 
        local_irq_save(flags);
@@ -7290,14 +7387,14 @@ void sched_idle_next(void)
         * Strictly not necessary since rest of the CPUs are stopped by now
         * and interrupts disabled on the current cpu.
         */
-       spin_lock_irqsave(&rq->lock, flags);
+       raw_spin_lock_irqsave(&rq->lock, flags);
 
        __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
 
        update_rq_clock(rq);
        activate_task(rq, p, 0);
 
-       spin_unlock_irqrestore(&rq->lock, flags);
+       raw_spin_unlock_irqrestore(&rq->lock, flags);
 }
 
 /*
@@ -7333,9 +7430,9 @@ static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
         * that's OK. No task can be added to this CPU, so iteration is
         * fine.
         */
-       spin_unlock_irq(&rq->lock);
+       raw_spin_unlock_irq(&rq->lock);
        move_task_off_dead_cpu(dead_cpu, p);
-       spin_lock_irq(&rq->lock);
+       raw_spin_lock_irq(&rq->lock);
 
        put_task_struct(p);
 }
@@ -7376,17 +7473,16 @@ static struct ctl_table sd_ctl_dir[] = {
                .procname       = "sched_domain",
                .mode           = 0555,
        },
-       {0, },
+       {}
 };
 
 static struct ctl_table sd_ctl_root[] = {
        {
-               .ctl_name       = CTL_KERN,
                .procname       = "kernel",
                .mode           = 0555,
                .child          = sd_ctl_dir,
        },
-       {0, },
+       {}
 };
 
 static struct ctl_table *sd_alloc_ctl_entry(int n)
@@ -7496,7 +7592,7 @@ static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
 static struct ctl_table_header *sd_sysctl_header;
 static void register_sched_domain_sysctl(void)
 {
-       int i, cpu_num = num_online_cpus();
+       int i, cpu_num = num_possible_cpus();
        struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
        char buf[32];
 
@@ -7506,7 +7602,7 @@ static void register_sched_domain_sysctl(void)
        if (entry == NULL)
                return;
 
-       for_each_online_cpu(i) {
+       for_each_possible_cpu(i) {
                snprintf(buf, 32, "cpu%d", i);
                entry->procname = kstrdup(buf, GFP_KERNEL);
                entry->mode = 0555;
@@ -7602,13 +7698,13 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
 
                /* Update our root-domain */
                rq = cpu_rq(cpu);
-               spin_lock_irqsave(&rq->lock, flags);
+               raw_spin_lock_irqsave(&rq->lock, flags);
                if (rq->rd) {
                        BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
 
                        set_rq_online(rq);
                }
-               spin_unlock_irqrestore(&rq->lock, flags);
+               raw_spin_unlock_irqrestore(&rq->lock, flags);
                break;
 
 #ifdef CONFIG_HOTPLUG_CPU
@@ -7633,14 +7729,13 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                put_task_struct(rq->migration_thread);
                rq->migration_thread = NULL;
                /* Idle task back to normal (off runqueue, low prio) */
-               spin_lock_irq(&rq->lock);
+               raw_spin_lock_irq(&rq->lock);
                update_rq_clock(rq);
                deactivate_task(rq, rq->idle, 0);
-               rq->idle->static_prio = MAX_PRIO;
                __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
                rq->idle->sched_class = &idle_sched_class;
                migrate_dead_tasks(cpu);
-               spin_unlock_irq(&rq->lock);
+               raw_spin_unlock_irq(&rq->lock);
                cpuset_unlock();
                migrate_nr_uninterruptible(rq);
                BUG_ON(rq->nr_running != 0);
@@ -7650,30 +7745,30 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 * they didn't take sched_hotcpu_mutex. Just wake up
                 * the requestors.
                 */
-               spin_lock_irq(&rq->lock);
+               raw_spin_lock_irq(&rq->lock);
                while (!list_empty(&rq->migration_queue)) {
                        struct migration_req *req;
 
                        req = list_entry(rq->migration_queue.next,
                                         struct migration_req, list);
                        list_del_init(&req->list);
-                       spin_unlock_irq(&rq->lock);
+                       raw_spin_unlock_irq(&rq->lock);
                        complete(&req->done);
-                       spin_lock_irq(&rq->lock);
+                       raw_spin_lock_irq(&rq->lock);
                }
-               spin_unlock_irq(&rq->lock);
+               raw_spin_unlock_irq(&rq->lock);
                break;
 
        case CPU_DYING:
        case CPU_DYING_FROZEN:
                /* Update our root-domain */
                rq = cpu_rq(cpu);
-               spin_lock_irqsave(&rq->lock, flags);
+               raw_spin_lock_irqsave(&rq->lock, flags);
                if (rq->rd) {
                        BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
                        set_rq_offline(rq);
                }
-               spin_unlock_irqrestore(&rq->lock, flags);
+               raw_spin_unlock_irqrestore(&rq->lock, flags);
                break;
 #endif
        }
@@ -7710,6 +7805,16 @@ early_initcall(migration_init);
 
 #ifdef CONFIG_SCHED_DEBUG
 
+static __read_mostly int sched_domain_debug_enabled;
+
+static int __init sched_domain_debug_setup(char *str)
+{
+       sched_domain_debug_enabled = 1;
+
+       return 0;
+}
+early_param("sched_debug", sched_domain_debug_setup);
+
 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                                  struct cpumask *groupmask)
 {
@@ -7796,6 +7901,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
        cpumask_var_t groupmask;
        int level = 0;
 
+       if (!sched_domain_debug_enabled)
+               return;
+
        if (!sd) {
                printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
                return;
@@ -7875,6 +7983,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
 
 static void free_rootdomain(struct root_domain *rd)
 {
+       synchronize_sched();
+
        cpupri_cleanup(&rd->cpupri);
 
        free_cpumask_var(rd->rto_mask);
@@ -7888,7 +7998,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
        struct root_domain *old_rd = NULL;
        unsigned long flags;
 
-       spin_lock_irqsave(&rq->lock, flags);
+       raw_spin_lock_irqsave(&rq->lock, flags);
 
        if (rq->rd) {
                old_rd = rq->rd;
@@ -7914,7 +8024,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
        if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
                set_rq_online(rq);
 
-       spin_unlock_irqrestore(&rq->lock, flags);
+       raw_spin_unlock_irqrestore(&rq->lock, flags);
 
        if (old_rd)
                free_rootdomain(old_rd);
@@ -8015,6 +8125,7 @@ static cpumask_var_t cpu_isolated_map;
 /* Setup the mask of cpus configured for isolated domains */
 static int __init isolated_cpu_setup(char *str)
 {
+       alloc_bootmem_cpumask_var(&cpu_isolated_map);
        cpulist_parse(str, cpu_isolated_map);
        return 1;
 }
@@ -8851,7 +8962,7 @@ static int build_sched_domains(const struct cpumask *cpu_map)
        return __build_sched_domains(cpu_map, NULL);
 }
 
-static struct cpumask *doms_cur;       /* current sched domains */
+static cpumask_var_t *doms_cur;        /* current sched domains */
 static int ndoms_cur;          /* number of sched domains in 'doms_cur' */
 static struct sched_domain_attr *dattr_cur;
                                /* attribues of custom domains in 'doms_cur' */
@@ -8873,6 +8984,31 @@ int __attribute__((weak)) arch_update_cpu_topology(void)
        return 0;
 }
 
+cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
+{
+       int i;
+       cpumask_var_t *doms;
+
+       doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
+       if (!doms)
+               return NULL;
+       for (i = 0; i < ndoms; i++) {
+               if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
+                       free_sched_domains(doms, i);
+                       return NULL;
+               }
+       }
+       return doms;
+}
+
+void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
+{
+       unsigned int i;
+       for (i = 0; i < ndoms; i++)
+               free_cpumask_var(doms[i]);
+       kfree(doms);
+}
+
 /*
  * Set up scheduler domains and groups. Callers must hold the hotplug lock.
  * For now this just excludes isolated cpus, but could be used to
@@ -8884,12 +9020,12 @@ static int arch_init_sched_domains(const struct cpumask *cpu_map)
 
        arch_update_cpu_topology();
        ndoms_cur = 1;
-       doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
+       doms_cur = alloc_sched_domains(ndoms_cur);
        if (!doms_cur)
-               doms_cur = fallback_doms;
-       cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
+               doms_cur = &fallback_doms;
+       cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
        dattr_cur = NULL;
-       err = build_sched_domains(doms_cur);
+       err = build_sched_domains(doms_cur[0]);
        register_sched_domain_sysctl();
 
        return err;
@@ -8939,19 +9075,19 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
  * doms_new[] to the current sched domain partitioning, doms_cur[].
  * It destroys each deleted domain and builds each new domain.
  *
- * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
+ * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
  * The masks don't intersect (don't overlap.) We should setup one
  * sched domain for each mask. CPUs not in any of the cpumasks will
  * not be load balanced. If the same cpumask appears both in the
  * current 'doms_cur' domains and in the new 'doms_new', we can leave
  * it as it is.
  *
- * The passed in 'doms_new' should be kmalloc'd. This routine takes
- * ownership of it and will kfree it when done with it. If the caller
- * failed the kmalloc call, then it can pass in doms_new == NULL &&
- * ndoms_new == 1, and partition_sched_domains() will fallback to
- * the single partition 'fallback_doms', it also forces the domains
- * to be rebuilt.
+ * The passed in 'doms_new' should be allocated using
+ * alloc_sched_domains.  This routine takes ownership of it and will
+ * free_sched_domains it when done with it. If the caller failed the
+ * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
+ * and partition_sched_domains() will fallback to the single partition
+ * 'fallback_doms', it also forces the domains to be rebuilt.
  *
  * If doms_new == NULL it will be replaced with cpu_online_mask.
  * ndoms_new == 0 is a special case for destroying existing domains,
@@ -8959,8 +9095,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
  *
  * Call with hotplug lock held
  */
-/* FIXME: Change to struct cpumask *doms_new[] */
-void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
+void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
                             struct sched_domain_attr *dattr_new)
 {
        int i, j, n;
@@ -8979,40 +9114,40 @@ void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
        /* Destroy deleted domains */
        for (i = 0; i < ndoms_cur; i++) {
                for (j = 0; j < n && !new_topology; j++) {
-                       if (cpumask_equal(&doms_cur[i], &doms_new[j])
+                       if (cpumask_equal(doms_cur[i], doms_new[j])
                            && dattrs_equal(dattr_cur, i, dattr_new, j))
                                goto match1;
                }
                /* no match - a current sched domain not in new doms_new[] */
-               detach_destroy_domains(doms_cur + i);
+               detach_destroy_domains(doms_cur[i]);
 match1:
                ;
        }
 
        if (doms_new == NULL) {
                ndoms_cur = 0;
-               doms_new = fallback_doms;
-               cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
+               doms_new = &fallback_doms;
+               cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
                WARN_ON_ONCE(dattr_new);
        }
 
        /* Build new domains */
        for (i = 0; i < ndoms_new; i++) {
                for (j = 0; j < ndoms_cur && !new_topology; j++) {
-                       if (cpumask_equal(&doms_new[i], &doms_cur[j])
+                       if (cpumask_equal(doms_new[i], doms_cur[j])
                            && dattrs_equal(dattr_new, i, dattr_cur, j))
                                goto match2;
                }
                /* no match - add a new doms_new */
-               __build_sched_domains(doms_new + i,
+               __build_sched_domains(doms_new[i],
                                        dattr_new ? dattr_new + i : NULL);
 match2:
                ;
        }
 
        /* Remember the new sched domains */
-       if (doms_cur != fallback_doms)
-               kfree(doms_cur);
+       if (doms_cur != &fallback_doms)
+               free_sched_domains(doms_cur, ndoms_cur);
        kfree(dattr_cur);       /* kfree(NULL) is safe */
        doms_cur = doms_new;
        dattr_cur = dattr_new;
@@ -9123,8 +9258,10 @@ static int update_sched_domains(struct notifier_block *nfb,
        switch (action) {
        case CPU_ONLINE:
        case CPU_ONLINE_FROZEN:
-       case CPU_DEAD:
-       case CPU_DEAD_FROZEN:
+       case CPU_DOWN_PREPARE:
+       case CPU_DOWN_PREPARE_FROZEN:
+       case CPU_DOWN_FAILED:
+       case CPU_DOWN_FAILED_FROZEN:
                partition_sched_domains(1, NULL, NULL);
                return NOTIFY_OK;
 
@@ -9171,7 +9308,7 @@ void __init sched_init_smp(void)
 #endif
        get_online_cpus();
        mutex_lock(&sched_domains_mutex);
-       arch_init_sched_domains(cpu_online_mask);
+       arch_init_sched_domains(cpu_active_mask);
        cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
        if (cpumask_empty(non_isolated_cpus))
                cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
@@ -9244,13 +9381,13 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
 #ifdef CONFIG_SMP
        rt_rq->rt_nr_migratory = 0;
        rt_rq->overloaded = 0;
-       plist_head_init(&rt_rq->pushable_tasks, &rq->lock);
+       plist_head_init_raw(&rt_rq->pushable_tasks, &rq->lock);
 #endif
 
        rt_rq->rt_time = 0;
        rt_rq->rt_throttled = 0;
        rt_rq->rt_runtime = 0;
-       spin_lock_init(&rt_rq->rt_runtime_lock);
+       raw_spin_lock_init(&rt_rq->rt_runtime_lock);
 
 #ifdef CONFIG_RT_GROUP_SCHED
        rt_rq->rt_nr_boosted = 0;
@@ -9334,10 +9471,6 @@ void __init sched_init(void)
 #ifdef CONFIG_CPUMASK_OFFSTACK
        alloc_size += num_possible_cpus() * cpumask_size();
 #endif
-       /*
-        * As sched_init() is called before page_alloc is setup,
-        * we use alloc_bootmem().
-        */
        if (alloc_size) {
                ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
 
@@ -9406,11 +9539,15 @@ void __init sched_init(void)
 #endif /* CONFIG_USER_SCHED */
 #endif /* CONFIG_GROUP_SCHED */
 
+#if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP
+       update_shares_data = __alloc_percpu(nr_cpu_ids * sizeof(unsigned long),
+                                           __alignof__(unsigned long));
+#endif
        for_each_possible_cpu(i) {
                struct rq *rq;
 
                rq = cpu_rq(i);
-               spin_lock_init(&rq->lock);
+               raw_spin_lock_init(&rq->lock);
                rq->nr_running = 0;
                rq->calc_load_active = 0;
                rq->calc_load_update = jiffies + LOAD_FREQ;
@@ -9488,6 +9625,8 @@ void __init sched_init(void)
                rq->cpu = i;
                rq->online = 0;
                rq->migration_thread = NULL;
+               rq->idle_stamp = 0;
+               rq->avg_idle = 2*sysctl_sched_migration_cost;
                INIT_LIST_HEAD(&rq->migration_queue);
                rq_attach_root(rq, &def_root_domain);
 #endif
@@ -9506,7 +9645,7 @@ void __init sched_init(void)
 #endif
 
 #ifdef CONFIG_RT_MUTEXES
-       plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
+       plist_head_init_raw(&init_task.pi_waiters, &init_task.pi_lock);
 #endif
 
        /*
@@ -9531,13 +9670,15 @@ void __init sched_init(void)
        current->sched_class = &fair_sched_class;
 
        /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
-       alloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT);
+       zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT);
 #ifdef CONFIG_SMP
 #ifdef CONFIG_NO_HZ
-       alloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT);
+       zalloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT);
        alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT);
 #endif
-       alloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
+       /* May be allocated at isolcpus cmdline parse time */
+       if (cpu_isolated_map == NULL)
+               zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
 #endif /* SMP */
 
        perf_event_init();
@@ -9548,7 +9689,7 @@ void __init sched_init(void)
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
 static inline int preempt_count_equals(int preempt_offset)
 {
-       int nested = preempt_count() & ~PREEMPT_ACTIVE;
+       int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
 
        return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
 }
@@ -9629,13 +9770,13 @@ void normalize_rt_tasks(void)
                        continue;
                }
 
-               spin_lock(&p->pi_lock);
+               raw_spin_lock(&p->pi_lock);
                rq = __task_rq_lock(p);
 
                normalize_task(rq, p);
 
                __task_rq_unlock(rq);
-               spin_unlock(&p->pi_lock);
+               raw_spin_unlock(&p->pi_lock);
        } while_each_thread(g, p);
 
        read_unlock_irqrestore(&tasklist_lock, flags);
@@ -9731,13 +9872,15 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
                se = kzalloc_node(sizeof(struct sched_entity),
                                  GFP_KERNEL, cpu_to_node(i));
                if (!se)
-                       goto err;
+                       goto err_free_rq;
 
                init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
        }
 
        return 1;
 
+ err_free_rq:
+       kfree(cfs_rq);
  err:
        return 0;
 }
@@ -9819,13 +9962,15 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
                rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
                                     GFP_KERNEL, cpu_to_node(i));
                if (!rt_se)
-                       goto err;
+                       goto err_free_rq;
 
                init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
        }
 
        return 1;
 
+ err_free_rq:
+       kfree(rt_rq);
  err:
        return 0;
 }
@@ -9959,7 +10104,7 @@ void sched_move_task(struct task_struct *tsk)
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
        if (tsk->sched_class->moved_group)
-               tsk->sched_class->moved_group(tsk);
+               tsk->sched_class->moved_group(tsk, on_rq);
 #endif
 
        if (unlikely(running))
@@ -9994,9 +10139,9 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares)
        struct rq *rq = cfs_rq->rq;
        unsigned long flags;
 
-       spin_lock_irqsave(&rq->lock, flags);
+       raw_spin_lock_irqsave(&rq->lock, flags);
        __set_se_shares(se, shares);
-       spin_unlock_irqrestore(&rq->lock, flags);
+       raw_spin_unlock_irqrestore(&rq->lock, flags);
 }
 
 static DEFINE_MUTEX(shares_mutex);
@@ -10181,18 +10326,18 @@ static int tg_set_bandwidth(struct task_group *tg,
        if (err)
                goto unlock;
 
-       spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
+       raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
        tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
        tg->rt_bandwidth.rt_runtime = rt_runtime;
 
        for_each_possible_cpu(i) {
                struct rt_rq *rt_rq = tg->rt_rq[i];
 
-               spin_lock(&rt_rq->rt_runtime_lock);
+               raw_spin_lock(&rt_rq->rt_runtime_lock);
                rt_rq->rt_runtime = rt_runtime;
-               spin_unlock(&rt_rq->rt_runtime_lock);
+               raw_spin_unlock(&rt_rq->rt_runtime_lock);
        }
-       spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
+       raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  unlock:
        read_unlock(&tasklist_lock);
        mutex_unlock(&rt_constraints_mutex);
@@ -10297,15 +10442,15 @@ static int sched_rt_global_constraints(void)
        if (sysctl_sched_rt_runtime == 0)
                return -EBUSY;
 
-       spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
+       raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
        for_each_possible_cpu(i) {
                struct rt_rq *rt_rq = &cpu_rq(i)->rt;
 
-               spin_lock(&rt_rq->rt_runtime_lock);
+               raw_spin_lock(&rt_rq->rt_runtime_lock);
                rt_rq->rt_runtime = global_rt_runtime();
-               spin_unlock(&rt_rq->rt_runtime_lock);
+               raw_spin_unlock(&rt_rq->rt_runtime_lock);
        }
-       spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
+       raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
 
        return 0;
 }
@@ -10596,9 +10741,9 @@ static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
        /*
         * Take rq->lock to make 64-bit read safe on 32-bit platforms.
         */
-       spin_lock_irq(&cpu_rq(cpu)->lock);
+       raw_spin_lock_irq(&cpu_rq(cpu)->lock);
        data = *cpuusage;
-       spin_unlock_irq(&cpu_rq(cpu)->lock);
+       raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
 #else
        data = *cpuusage;
 #endif
@@ -10614,9 +10759,9 @@ static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
        /*
         * Take rq->lock to make 64-bit write safe on 32-bit platforms.
         */
-       spin_lock_irq(&cpu_rq(cpu)->lock);
+       raw_spin_lock_irq(&cpu_rq(cpu)->lock);
        *cpuusage = val;
-       spin_unlock_irq(&cpu_rq(cpu)->lock);
+       raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
 #else
        *cpuusage = val;
 #endif
@@ -10850,9 +10995,9 @@ void synchronize_sched_expedited(void)
                init_completion(&req->done);
                req->task = NULL;
                req->dest_cpu = RCU_MIGRATION_NEED_QS;
-               spin_lock_irqsave(&rq->lock, flags);
+               raw_spin_lock_irqsave(&rq->lock, flags);
                list_add(&req->list, &rq->migration_queue);
-               spin_unlock_irqrestore(&rq->lock, flags);
+               raw_spin_unlock_irqrestore(&rq->lock, flags);
                wake_up_process(rq->migration_thread);
        }
        for_each_online_cpu(cpu) {
@@ -10860,13 +11005,14 @@ void synchronize_sched_expedited(void)
                req = &per_cpu(rcu_migration_req, cpu);
                rq = cpu_rq(cpu);
                wait_for_completion(&req->done);
-               spin_lock_irqsave(&rq->lock, flags);
+               raw_spin_lock_irqsave(&rq->lock, flags);
                if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC))
                        need_full_sync = 1;
                req->dest_cpu = RCU_MIGRATION_IDLE;
-               spin_unlock_irqrestore(&rq->lock, flags);
+               raw_spin_unlock_irqrestore(&rq->lock, flags);
        }
        rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
+       synchronize_sched_expedited_count++;
        mutex_unlock(&rcu_sched_expedited_mutex);
        put_online_cpus();
        if (need_full_sync)