Fix cpusets update_cpumask
[linux-3.10.git] / kernel / sched.c
index 2c6295b..72a809a 100644 (file)
@@ -44,6 +44,7 @@
 #include <linux/vmalloc.h>
 #include <linux/blkdev.h>
 #include <linux/delay.h>
+#include <linux/pid_namespace.h>
 #include <linux/smp.h>
 #include <linux/threads.h>
 #include <linux/timer.h>
@@ -51,6 +52,7 @@
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/percpu.h>
+#include <linux/cpu_acct.h>
 #include <linux/kthread.h>
 #include <linux/seq_file.h>
 #include <linux/sysctl.h>
@@ -75,12 +77,6 @@ unsigned long long __attribute__((weak)) sched_clock(void)
        return (unsigned long long)jiffies * (1000000000 / HZ);
 }
 
-#ifdef CONFIG_SMP
-#define is_migration_thread(p, rq) ((p) == (rq)->migration_thread)
-#else
-#define is_migration_thread(p, rq) 0
-#endif
-
 /*
  * Convert user-nice values [ -20 ... 0 ... 19 ]
  * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
@@ -272,7 +268,8 @@ struct rt_rq {
  * acquire operations must be ordered by ascending &runqueue.
  */
 struct rq {
-       spinlock_t lock;        /* runqueue lock */
+       /* runqueue lock: */
+       spinlock_t lock;
 
        /*
         * nr_running and cpu_load should be in the same cacheline because
@@ -285,13 +282,15 @@ struct rq {
 #ifdef CONFIG_NO_HZ
        unsigned char in_nohz_recently;
 #endif
-       struct load_weight load;        /* capture load from *all* tasks on this cpu */
+       /* capture load from *all* tasks on this cpu: */
+       struct load_weight load;
        unsigned long nr_load_updates;
        u64 nr_switches;
 
        struct cfs_rq cfs;
 #ifdef CONFIG_FAIR_GROUP_SCHED
-       struct list_head leaf_cfs_rq_list; /* list of leaf cfs_rq on this cpu */
+       /* list of leaf cfs_rq on this cpu: */
+       struct list_head leaf_cfs_rq_list;
 #endif
        struct rt_rq  rt;
 
@@ -323,7 +322,8 @@ struct rq {
        /* For active balancing */
        int active_balance;
        int push_cpu;
-       int cpu;                /* cpu of this runqueue */
+       /* cpu of this runqueue: */
+       int cpu;
 
        struct task_struct *migration_thread;
        struct list_head migration_queue;
@@ -334,22 +334,22 @@ struct rq {
        struct sched_info rq_sched_info;
 
        /* sys_sched_yield() stats */
-       unsigned long yld_exp_empty;
-       unsigned long yld_act_empty;
-       unsigned long yld_both_empty;
-       unsigned long yld_count;
+       unsigned int yld_exp_empty;
+       unsigned int yld_act_empty;
+       unsigned int yld_both_empty;
+       unsigned int yld_count;
 
        /* schedule() stats */
-       unsigned long sched_switch;
-       unsigned long sched_count;
-       unsigned long sched_goidle;
+       unsigned int sched_switch;
+       unsigned int sched_count;
+       unsigned int sched_goidle;
 
        /* try_to_wake_up() stats */
-       unsigned long ttwu_count;
-       unsigned long ttwu_local;
+       unsigned int ttwu_count;
+       unsigned int ttwu_local;
 
        /* BKL stats */
-       unsigned long bkl_count;
+       unsigned int bkl_count;
 #endif
        struct lock_class_key rq_lock_key;
 };
@@ -455,12 +455,12 @@ enum {
 };
 
 const_debug unsigned int sysctl_sched_features =
-               SCHED_FEAT_NEW_FAIR_SLEEPERS    *1 |
-               SCHED_FEAT_START_DEBIT          *1 |
-               SCHED_FEAT_TREE_AVG             *0 |
-               SCHED_FEAT_APPROX_AVG           *0 |
-               SCHED_FEAT_WAKEUP_PREEMPT       *1 |
-               SCHED_FEAT_PREEMPT_RESTRICT     *1;
+               SCHED_FEAT_NEW_FAIR_SLEEPERS    * 1 |
+               SCHED_FEAT_START_DEBIT          * 1 |
+               SCHED_FEAT_TREE_AVG             * 0 |
+               SCHED_FEAT_APPROX_AVG           * 0 |
+               SCHED_FEAT_WAKEUP_PREEMPT       * 1 |
+               SCHED_FEAT_PREEMPT_RESTRICT     * 1;
 
 #define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x)
 
@@ -1011,6 +1011,28 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
 
 #ifdef CONFIG_SMP
 
+/*
+ * Is this task likely cache-hot:
+ */
+static inline int
+task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
+{
+       s64 delta;
+
+       if (p->sched_class != &fair_sched_class)
+               return 0;
+
+       if (sysctl_sched_migration_cost == -1)
+               return 1;
+       if (sysctl_sched_migration_cost == 0)
+               return 0;
+
+       delta = now - p->se.exec_start;
+
+       return delta < (s64)sysctl_sched_migration_cost;
+}
+
+
 void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
 {
        int old_cpu = task_cpu(p);
@@ -1028,6 +1050,11 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
                p->se.sleep_start -= clock_offset;
        if (p->se.block_start)
                p->se.block_start -= clock_offset;
+       if (old_cpu != new_cpu) {
+               schedstat_inc(p, se.nr_migrations);
+               if (task_hot(p, old_rq->clock, NULL))
+                       schedstat_inc(p, se.nr_forced2_migrations);
+       }
 #endif
        p->se.vruntime -= old_cfsrq->min_vruntime -
                                         new_cfsrq->min_vruntime;
@@ -1400,8 +1427,13 @@ static int wake_idle(int cpu, struct task_struct *p)
                if (sd->flags & SD_WAKE_IDLE) {
                        cpus_and(tmp, sd->span, p->cpus_allowed);
                        for_each_cpu_mask(i, tmp) {
-                               if (idle_cpu(i))
+                               if (idle_cpu(i)) {
+                                       if (i != task_cpu(p)) {
+                                               schedstat_inc(p,
+                                                       se.nr_wakeups_idle);
+                                       }
                                        return i;
+                               }
                        }
                } else {
                        break;
@@ -1432,7 +1464,7 @@ static inline int wake_idle(int cpu, struct task_struct *p)
  */
 static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
 {
-       int cpu, this_cpu, success = 0;
+       int cpu, orig_cpu, this_cpu, success = 0;
        unsigned long flags;
        long old_state;
        struct rq *rq;
@@ -1451,6 +1483,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
                goto out_running;
 
        cpu = task_cpu(p);
+       orig_cpu = cpu;
        this_cpu = smp_processor_id();
 
 #ifdef CONFIG_SMP
@@ -1494,6 +1527,13 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
                        unsigned long tl = this_load;
                        unsigned long tl_per_task;
 
+                       /*
+                        * Attract cache-cold tasks on sync wakeups:
+                        */
+                       if (sync && !task_hot(p, rq->clock, this_sd))
+                               goto out_set_cpu;
+
+                       schedstat_inc(p, se.nr_wakeups_affine_attempts);
                        tl_per_task = cpu_avg_load_per_task(this_cpu);
 
                        /*
@@ -1513,6 +1553,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
                                 * there is no bad imbalance.
                                 */
                                schedstat_inc(this_sd, ttwu_move_affine);
+                               schedstat_inc(p, se.nr_wakeups_affine);
                                goto out_set_cpu;
                        }
                }
@@ -1524,6 +1565,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
                if (this_sd->flags & SD_WAKE_BALANCE) {
                        if (imbalance*this_load <= 100*load) {
                                schedstat_inc(this_sd, ttwu_move_balance);
+                               schedstat_inc(p, se.nr_wakeups_passive);
                                goto out_set_cpu;
                        }
                }
@@ -1549,18 +1591,18 @@ out_set_cpu:
 
 out_activate:
 #endif /* CONFIG_SMP */
+       schedstat_inc(p, se.nr_wakeups);
+       if (sync)
+               schedstat_inc(p, se.nr_wakeups_sync);
+       if (orig_cpu != cpu)
+               schedstat_inc(p, se.nr_wakeups_migrate);
+       if (cpu == this_cpu)
+               schedstat_inc(p, se.nr_wakeups_local);
+       else
+               schedstat_inc(p, se.nr_wakeups_remote);
        update_rq_clock(rq);
        activate_task(rq, p, 1);
-       /*
-        * Sync wakeups (i.e. those types of wakeups where the waker
-        * has indicated that it will leave the CPU in short order)
-        * don't trigger a preemption, if the woken up task will run on
-        * this cpu. (in this case the 'I will reschedule' promise of
-        * the waker guarantees that the freshly woken up task is going
-        * to be considered on this CPU.)
-        */
-       if (!sync || cpu != this_cpu)
-               check_preempt_curr(rq, p);
+       check_preempt_curr(rq, p);
        success = 1;
 
 out_running:
@@ -1676,7 +1718,7 @@ void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
 
        p->prio = effective_prio(p);
 
-       if (!p->sched_class->task_new || !current->se.on_rq || !rq->cfs.curr) {
+       if (!p->sched_class->task_new || !current->se.on_rq) {
                activate_task(rq, p, 0);
        } else {
                /*
@@ -1835,7 +1877,7 @@ asmlinkage void schedule_tail(struct task_struct *prev)
        preempt_enable();
 #endif
        if (current->set_child_tid)
-               put_user(current->pid, current->set_child_tid);
+               put_user(task_pid_vnr(current), current->set_child_tid);
 }
 
 /*
@@ -2138,13 +2180,38 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
         * 2) cannot be migrated to this CPU due to cpus_allowed, or
         * 3) are cache-hot on their current CPU.
         */
-       if (!cpu_isset(this_cpu, p->cpus_allowed))
+       if (!cpu_isset(this_cpu, p->cpus_allowed)) {
+               schedstat_inc(p, se.nr_failed_migrations_affine);
                return 0;
+       }
        *all_pinned = 0;
 
-       if (task_running(rq, p))
+       if (task_running(rq, p)) {
+               schedstat_inc(p, se.nr_failed_migrations_running);
                return 0;
+       }
+
+       /*
+        * Aggressive migration if:
+        * 1) task is cache cold, or
+        * 2) too many balance attempts have failed.
+        */
+
+       if (!task_hot(p, rq->clock, sd) ||
+                       sd->nr_balance_failed > sd->cache_nice_tries) {
+#ifdef CONFIG_SCHEDSTATS
+               if (task_hot(p, rq->clock, sd)) {
+                       schedstat_inc(sd, lb_hot_gained[idle]);
+                       schedstat_inc(p, se.nr_forced_migrations);
+               }
+#endif
+               return 1;
+       }
 
+       if (task_hot(p, rq->clock, sd)) {
+               schedstat_inc(p, se.nr_failed_migrations_hot);
+               return 0;
+       }
        return 1;
 }
 
@@ -2275,7 +2342,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
        unsigned long max_pull;
        unsigned long busiest_load_per_task, busiest_nr_running;
        unsigned long this_load_per_task, this_nr_running;
-       int load_idx;
+       int load_idx, group_imb = 0;
 #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
        int power_savings_balance = 1;
        unsigned long leader_nr_running = 0, min_load_per_task = 0;
@@ -2294,9 +2361,10 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                load_idx = sd->idle_idx;
 
        do {
-               unsigned long load, group_capacity;
+               unsigned long load, group_capacity, max_cpu_load, min_cpu_load;
                int local_group;
                int i;
+               int __group_imb = 0;
                unsigned int balance_cpu = -1, first_idle_cpu = 0;
                unsigned long sum_nr_running, sum_weighted_load;
 
@@ -2307,6 +2375,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
 
                /* Tally up the load of all CPUs in the group */
                sum_weighted_load = sum_nr_running = avg_load = 0;
+               max_cpu_load = 0;
+               min_cpu_load = ~0UL;
 
                for_each_cpu_mask(i, group->cpumask) {
                        struct rq *rq;
@@ -2327,8 +2397,13 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                                }
 
                                load = target_load(i, load_idx);
-                       } else
+                       } else {
                                load = source_load(i, load_idx);
+                               if (load > max_cpu_load)
+                                       max_cpu_load = load;
+                               if (min_cpu_load > load)
+                                       min_cpu_load = load;
+                       }
 
                        avg_load += load;
                        sum_nr_running += rq->nr_running;
@@ -2354,6 +2429,9 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                avg_load = sg_div_cpu_power(group,
                                avg_load * SCHED_LOAD_SCALE);
 
+               if ((max_cpu_load - min_cpu_load) > SCHED_LOAD_SCALE)
+                       __group_imb = 1;
+
                group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
 
                if (local_group) {
@@ -2362,11 +2440,12 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                        this_nr_running = sum_nr_running;
                        this_load_per_task = sum_weighted_load;
                } else if (avg_load > max_load &&
-                          sum_nr_running > group_capacity) {
+                          (sum_nr_running > group_capacity || __group_imb)) {
                        max_load = avg_load;
                        busiest = group;
                        busiest_nr_running = sum_nr_running;
                        busiest_load_per_task = sum_weighted_load;
+                       group_imb = __group_imb;
                }
 
 #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
@@ -2438,6 +2517,9 @@ group_next:
                goto out_balanced;
 
        busiest_load_per_task /= busiest_nr_running;
+       if (group_imb)
+               busiest_load_per_task = min(busiest_load_per_task, avg_load);
+
        /*
         * We're trying to get all the cpus to the average_load, so we don't
         * want to push ourselves above the average load, nor do we wish to
@@ -3227,9 +3309,13 @@ void account_user_time(struct task_struct *p, cputime_t cputime)
 {
        struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
        cputime64_t tmp;
+       struct rq *rq = this_rq();
 
        p->utime = cputime_add(p->utime, cputime);
 
+       if (p != rq->idle)
+               cpuacct_charge(p, cputime);
+
        /* Add user time to cpustat. */
        tmp = cputime_to_cputime64(cputime);
        if (TASK_NICE(p) > 0)
@@ -3239,6 +3325,35 @@ void account_user_time(struct task_struct *p, cputime_t cputime)
 }
 
 /*
+ * Account guest cpu time to a process.
+ * @p: the process that the cpu time gets accounted to
+ * @cputime: the cpu time spent in virtual machine since the last update
+ */
+void account_guest_time(struct task_struct *p, cputime_t cputime)
+{
+       cputime64_t tmp;
+       struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
+
+       tmp = cputime_to_cputime64(cputime);
+
+       p->utime = cputime_add(p->utime, cputime);
+       p->gtime = cputime_add(p->gtime, cputime);
+
+       cpustat->user = cputime64_add(cpustat->user, tmp);
+       cpustat->guest = cputime64_add(cpustat->guest, tmp);
+}
+
+/*
+ * Account scaled user cpu time to a process.
+ * @p: the process that the cpu time gets accounted to
+ * @cputime: the cpu time spent in user space since the last update
+ */
+void account_user_time_scaled(struct task_struct *p, cputime_t cputime)
+{
+       p->utimescaled = cputime_add(p->utimescaled, cputime);
+}
+
+/*
  * Account system cpu time to a process.
  * @p: the process that the cpu time gets accounted to
  * @hardirq_offset: the offset to subtract from hardirq_count()
@@ -3251,6 +3366,12 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
        struct rq *rq = this_rq();
        cputime64_t tmp;
 
+       if (p->flags & PF_VCPU) {
+               account_guest_time(p, cputime);
+               p->flags &= ~PF_VCPU;
+               return;
+       }
+
        p->stime = cputime_add(p->stime, cputime);
 
        /* Add system time to cpustat. */
@@ -3259,9 +3380,10 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
                cpustat->irq = cputime64_add(cpustat->irq, tmp);
        else if (softirq_count())
                cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
-       else if (p != rq->idle)
+       else if (p != rq->idle) {
                cpustat->system = cputime64_add(cpustat->system, tmp);
-       else if (atomic_read(&rq->nr_iowait) > 0)
+               cpuacct_charge(p, cputime);
+       } else if (atomic_read(&rq->nr_iowait) > 0)
                cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
        else
                cpustat->idle = cputime64_add(cpustat->idle, tmp);
@@ -3270,6 +3392,17 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
 }
 
 /*
+ * Account scaled system cpu time to a process.
+ * @p: the process that the cpu time gets accounted to
+ * @hardirq_offset: the offset to subtract from hardirq_count()
+ * @cputime: the cpu time spent in kernel space since the last update
+ */
+void account_system_time_scaled(struct task_struct *p, cputime_t cputime)
+{
+       p->stimescaled = cputime_add(p->stimescaled, cputime);
+}
+
+/*
  * Account for involuntary wait time.
  * @p: the process from which the cpu time has been stolen
  * @steal: the cpu time spent in involuntary wait
@@ -3286,8 +3419,10 @@ void account_steal_time(struct task_struct *p, cputime_t steal)
                        cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
                else
                        cpustat->idle = cputime64_add(cpustat->idle, tmp);
-       } else
+       } else {
                cpustat->steal = cputime64_add(cpustat->steal, tmp);
+               cpuacct_charge(p, -tmp);
+       }
 }
 
 /*
@@ -3758,7 +3893,10 @@ EXPORT_SYMBOL(wait_for_completion_timeout);
 
 int __sched wait_for_completion_interruptible(struct completion *x)
 {
-       return wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
+       long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
+       if (t == -ERESTARTSYS)
+               return t;
+       return 0;
 }
 EXPORT_SYMBOL(wait_for_completion_interruptible);
 
@@ -4030,7 +4168,7 @@ struct task_struct *idle_task(int cpu)
  */
 static struct task_struct *find_process_by_pid(pid_t pid)
 {
-       return pid ? find_task_by_pid(pid) : current;
+       return pid ? find_task_by_vpid(pid) : current;
 }
 
 /* Actually do priority change: must hold rq lock. */
@@ -4333,8 +4471,21 @@ long sched_setaffinity(pid_t pid, cpumask_t new_mask)
 
        cpus_allowed = cpuset_cpus_allowed(p);
        cpus_and(new_mask, new_mask, cpus_allowed);
+ again:
        retval = set_cpus_allowed(p, new_mask);
 
+       if (!retval) {
+               cpus_allowed = cpuset_cpus_allowed(p);
+               if (!cpus_subset(new_mask, cpus_allowed)) {
+                       /*
+                        * We must have raced with a concurrent cpuset
+                        * update. Just reset the cpus_allowed to the
+                        * cpuset's cpus_allowed
+                        */
+                       new_mask = cpus_allowed;
+                       goto again;
+               }
+       }
 out_unlock:
        put_task_struct(p);
        mutex_unlock(&sched_hotcpu_mutex);
@@ -4693,18 +4844,18 @@ static void show_task(struct task_struct *p)
        unsigned state;
 
        state = p->state ? __ffs(p->state) + 1 : 0;
-       printk("%-13.13s %c", p->comm,
+       printk(KERN_INFO "%-13.13s %c", p->comm,
                state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
 #if BITS_PER_LONG == 32
        if (state == TASK_RUNNING)
-               printk(" running  ");
+               printk(KERN_CONT " running  ");
        else
-               printk(" %08lx ", thread_saved_pc(p));
+               printk(KERN_CONT " %08lx ", thread_saved_pc(p));
 #else
        if (state == TASK_RUNNING)
-               printk("  running task    ");
+               printk(KERN_CONT "  running task    ");
        else
-               printk(" %016lx ", thread_saved_pc(p));
+               printk(KERN_CONT " %016lx ", thread_saved_pc(p));
 #endif
 #ifdef CONFIG_DEBUG_STACK_USAGE
        {
@@ -4714,7 +4865,7 @@ static void show_task(struct task_struct *p)
                free = (unsigned long)n - (unsigned long)end_of_stack(p);
        }
 #endif
-       printk("%5lu %5d %6d\n", free, p->pid, p->parent->pid);
+       printk(KERN_CONT "%5lu %5d %6d\n", free, p->pid, p->parent->pid);
 
        if (state != TASK_RUNNING)
                show_stack(p, NULL);
@@ -4974,6 +5125,17 @@ wait_to_die:
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
+
+static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
+{
+       int ret;
+
+       local_irq_disable();
+       ret = __migrate_task(p, src_cpu, dest_cpu);
+       local_irq_enable();
+       return ret;
+}
+
 /*
  * Figure out where task on dead CPU should go, use force if neccessary.
  * NOTE: interrupts should be disabled by the caller
@@ -5012,7 +5174,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
                                       "longer affine to cpu%d\n",
                                       p->pid, p->comm, dead_cpu);
                }
-       } while (!__migrate_task(p, dead_cpu, dest_cpu));
+       } while (!__migrate_task_irq(p, dead_cpu, dest_cpu));
 }
 
 /*
@@ -5040,7 +5202,7 @@ static void migrate_live_tasks(int src_cpu)
 {
        struct task_struct *p, *t;
 
-       write_lock_irq(&tasklist_lock);
+       read_lock(&tasklist_lock);
 
        do_each_thread(t, p) {
                if (p == current)
@@ -5050,7 +5212,7 @@ static void migrate_live_tasks(int src_cpu)
                        move_task_off_dead_cpu(src_cpu, p);
        } while_each_thread(t, p);
 
-       write_unlock_irq(&tasklist_lock);
+       read_unlock(&tasklist_lock);
 }
 
 /*
@@ -5128,11 +5290,10 @@ static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
         * Drop lock around migration; if someone else moves it,
         * that's OK.  No task can be added to this CPU, so iteration is
         * fine.
-        * NOTE: interrupts should be left disabled  --dev@
         */
-       spin_unlock(&rq->lock);
+       spin_unlock_irq(&rq->lock);
        move_task_off_dead_cpu(dead_cpu, p);
-       spin_lock(&rq->lock);
+       spin_lock_irq(&rq->lock);
 
        put_task_struct(p);
 }
@@ -5179,14 +5340,32 @@ static struct ctl_table sd_ctl_root[] = {
 static struct ctl_table *sd_alloc_ctl_entry(int n)
 {
        struct ctl_table *entry =
-               kmalloc(n * sizeof(struct ctl_table), GFP_KERNEL);
-
-       BUG_ON(!entry);
-       memset(entry, 0, n * sizeof(struct ctl_table));
+               kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
 
        return entry;
 }
 
+static void sd_free_ctl_entry(struct ctl_table **tablep)
+{
+       struct ctl_table *entry;
+
+       /*
+        * In the intermediate directories, both the child directory and
+        * procname are dynamically allocated and could fail but the mode
+        * will always be set.  In the lowest directory the names are
+        * static strings and all have proc handlers.
+        */
+       for (entry = *tablep; entry->mode; entry++) {
+               if (entry->child)
+                       sd_free_ctl_entry(&entry->child);
+               if (entry->proc_handler == NULL)
+                       kfree(entry->procname);
+       }
+
+       kfree(*tablep);
+       *tablep = NULL;
+}
+
 static void
 set_table_entry(struct ctl_table *entry,
                const char *procname, void *data, int maxlen,
@@ -5204,6 +5383,9 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd)
 {
        struct ctl_table *table = sd_alloc_ctl_entry(12);
 
+       if (table == NULL)
+               return NULL;
+
        set_table_entry(&table[0], "min_interval", &sd->min_interval,
                sizeof(long), 0644, proc_doulongvec_minmax);
        set_table_entry(&table[1], "max_interval", &sd->max_interval,
@@ -5227,11 +5409,12 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd)
                sizeof(int), 0644, proc_dointvec_minmax);
        set_table_entry(&table[10], "flags", &sd->flags,
                sizeof(int), 0644, proc_dointvec_minmax);
+       /* &table[11] is terminator */
 
        return table;
 }
 
-static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
+static ctl_table * sd_alloc_ctl_cpu_table(int cpu)
 {
        struct ctl_table *entry, *table;
        struct sched_domain *sd;
@@ -5241,6 +5424,8 @@ static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
        for_each_domain(cpu, sd)
                domain_num++;
        entry = table = sd_alloc_ctl_entry(domain_num + 1);
+       if (table == NULL)
+               return NULL;
 
        i = 0;
        for_each_domain(cpu, sd) {
@@ -5255,24 +5440,38 @@ static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
 }
 
 static struct ctl_table_header *sd_sysctl_header;
-static void init_sched_domain_sysctl(void)
+static void register_sched_domain_sysctl(void)
 {
        int i, cpu_num = num_online_cpus();
        struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
        char buf[32];
 
+       if (entry == NULL)
+               return;
+
        sd_ctl_dir[0].child = entry;
 
-       for (i = 0; i < cpu_num; i++, entry++) {
+       for_each_online_cpu(i) {
                snprintf(buf, 32, "cpu%d", i);
                entry->procname = kstrdup(buf, GFP_KERNEL);
                entry->mode = 0555;
                entry->child = sd_alloc_ctl_cpu_table(i);
+               entry++;
        }
        sd_sysctl_header = register_sysctl_table(sd_ctl_root);
 }
+
+static void unregister_sched_domain_sysctl(void)
+{
+       unregister_sysctl_table(sd_sysctl_header);
+       sd_sysctl_header = NULL;
+       sd_free_ctl_entry(&sd_ctl_dir[0].child);
+}
 #else
-static void init_sched_domain_sysctl(void)
+static void register_sched_domain_sysctl(void)
+{
+}
+static void unregister_sched_domain_sysctl(void)
 {
 }
 #endif
@@ -5332,14 +5531,14 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                kthread_stop(rq->migration_thread);
                rq->migration_thread = NULL;
                /* Idle task back to normal (off runqueue, low prio) */
-               rq = task_rq_lock(rq->idle, &flags);
+               spin_lock_irq(&rq->lock);
                update_rq_clock(rq);
                deactivate_task(rq, rq->idle, 0);
                rq->idle->static_prio = MAX_PRIO;
                __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
                rq->idle->sched_class = &idle_sched_class;
                migrate_dead_tasks(cpu);
-               task_rq_unlock(rq, &flags);
+               spin_unlock_irq(&rq->lock);
                migrate_nr_uninterruptible(rq);
                BUG_ON(rq->nr_running != 0);
 
@@ -5449,20 +5648,20 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
                        }
 
                        if (!group->__cpu_power) {
-                               printk("\n");
+                               printk(KERN_CONT "\n");
                                printk(KERN_ERR "ERROR: domain->cpu_power not "
                                                "set\n");
                                break;
                        }
 
                        if (!cpus_weight(group->cpumask)) {
-                               printk("\n");
+                               printk(KERN_CONT "\n");
                                printk(KERN_ERR "ERROR: empty group\n");
                                break;
                        }
 
                        if (cpus_intersects(groupmask, group->cpumask)) {
-                               printk("\n");
+                               printk(KERN_CONT "\n");
                                printk(KERN_ERR "ERROR: repeated CPUs\n");
                                break;
                        }
@@ -5470,11 +5669,11 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
                        cpus_or(groupmask, groupmask, group->cpumask);
 
                        cpumask_scnprintf(str, NR_CPUS, group->cpumask);
-                       printk(" %s", str);
+                       printk(KERN_CONT " %s", str);
 
                        group = group->next;
                } while (group != sd->groups);
-               printk("\n");
+               printk(KERN_CONT "\n");
 
                if (!cpus_equal(sd->span, groupmask))
                        printk(KERN_ERR "ERROR: groups don't span "
@@ -5754,7 +5953,7 @@ static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
                             struct sched_group **sg)
 {
        int group;
-       cpumask_t mask = cpu_sibling_map[cpu];
+       cpumask_t mask = per_cpu(cpu_sibling_map, cpu);
        cpus_and(mask, mask, *cpu_map);
        group = first_cpu(mask);
        if (sg)
@@ -5783,7 +5982,7 @@ static int cpu_to_phys_group(int cpu, const cpumask_t *cpu_map,
        cpus_and(mask, mask, *cpu_map);
        group = first_cpu(mask);
 #elif defined(CONFIG_SCHED_SMT)
-       cpumask_t mask = cpu_sibling_map[cpu];
+       cpumask_t mask = per_cpu(cpu_sibling_map, cpu);
        cpus_and(mask, mask, *cpu_map);
        group = first_cpu(mask);
 #else
@@ -5954,7 +6153,7 @@ static int build_sched_domains(const cpumask_t *cpu_map)
        /*
         * Allocate the per-node list of sched groups
         */
-       sched_group_nodes = kzalloc(sizeof(struct sched_group *)*MAX_NUMNODES,
+       sched_group_nodes = kcalloc(MAX_NUMNODES, sizeof(struct sched_group *),
                                           GFP_KERNEL);
        if (!sched_group_nodes) {
                printk(KERN_WARNING "Can not alloc sched group node list\n");
@@ -6017,7 +6216,7 @@ static int build_sched_domains(const cpumask_t *cpu_map)
                p = sd;
                sd = &per_cpu(cpu_domains, i);
                *sd = SD_SIBLING_INIT;
-               sd->span = cpu_sibling_map[i];
+               sd->span = per_cpu(cpu_sibling_map, i);
                cpus_and(sd->span, sd->span, *cpu_map);
                sd->parent = p;
                p->child = sd;
@@ -6028,7 +6227,7 @@ static int build_sched_domains(const cpumask_t *cpu_map)
 #ifdef CONFIG_SCHED_SMT
        /* Set up CPU (sibling) groups */
        for_each_cpu_mask(i, *cpu_map) {
-               cpumask_t this_sibling_map = cpu_sibling_map[i];
+               cpumask_t this_sibling_map = per_cpu(cpu_sibling_map, i);
                cpus_and(this_sibling_map, this_sibling_map, *cpu_map);
                if (i != first_cpu(this_sibling_map))
                        continue;
@@ -6190,24 +6389,31 @@ error:
        return -ENOMEM;
 #endif
 }
+
+static cpumask_t *doms_cur;    /* current sched domains */
+static int ndoms_cur;          /* number of sched domains in 'doms_cur' */
+
+/*
+ * Special case: If a kmalloc of a doms_cur partition (array of
+ * cpumask_t) fails, then fallback to a single sched domain,
+ * as determined by the single cpumask_t fallback_doms.
+ */
+static cpumask_t fallback_doms;
+
 /*
  * Set up scheduler domains and groups.  Callers must hold the hotplug lock.
+ * For now this just excludes isolated cpus, but could be used to
+ * exclude other special cases in the future.
  */
 static int arch_init_sched_domains(const cpumask_t *cpu_map)
 {
-       cpumask_t cpu_default_map;
-       int err;
-
-       /*
-        * Setup mask for cpus without special case scheduling requirements.
-        * For now this just excludes isolated cpus, but could be used to
-        * exclude other special cases in the future.
-        */
-       cpus_andnot(cpu_default_map, *cpu_map, cpu_isolated_map);
-
-       err = build_sched_domains(&cpu_default_map);
-
-       return err;
+       ndoms_cur = 1;
+       doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
+       if (!doms_cur)
+               doms_cur = &fallback_doms;
+       cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map);
+       register_sched_domain_sysctl();
+       return build_sched_domains(doms_cur);
 }
 
 static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
@@ -6223,6 +6429,8 @@ static void detach_destroy_domains(const cpumask_t *cpu_map)
 {
        int i;
 
+       unregister_sched_domain_sysctl();
+
        for_each_cpu_mask(i, *cpu_map)
                cpu_attach_domain(NULL, i);
        synchronize_sched();
@@ -6230,30 +6438,65 @@ static void detach_destroy_domains(const cpumask_t *cpu_map)
 }
 
 /*
- * Partition sched domains as specified by the cpumasks below.
- * This attaches all cpus from the cpumasks to the NULL domain,
- * waits for a RCU quiescent period, recalculates sched
- * domain information and then attaches them back to the
- * correct sched domains
+ * Partition sched domains as specified by the 'ndoms_new'
+ * cpumasks in the array doms_new[] of cpumasks.  This compares
+ * doms_new[] to the current sched domain partitioning, doms_cur[].
+ * It destroys each deleted domain and builds each new domain.
+ *
+ * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'.
+ * The masks don't intersect (don't overlap.)  We should setup one
+ * sched domain for each mask.  CPUs not in any of the cpumasks will
+ * not be load balanced.  If the same cpumask appears both in the
+ * current 'doms_cur' domains and in the new 'doms_new', we can leave
+ * it as it is.
+ *
+ * The passed in 'doms_new' should be kmalloc'd.  This routine takes
+ * ownership of it and will kfree it when done with it.  If the caller
+ * failed the kmalloc call, then it can pass in doms_new == NULL,
+ * and partition_sched_domains() will fallback to the single partition
+ * 'fallback_doms'.
+ *
  * Call with hotplug lock held
  */
-int partition_sched_domains(cpumask_t *partition1, cpumask_t *partition2)
+void partition_sched_domains(int ndoms_new, cpumask_t *doms_new)
 {
-       cpumask_t change_map;
-       int err = 0;
+       int i, j;
 
-       cpus_and(*partition1, *partition1, cpu_online_map);
-       cpus_and(*partition2, *partition2, cpu_online_map);
-       cpus_or(change_map, *partition1, *partition2);
+       if (doms_new == NULL) {
+               ndoms_new = 1;
+               doms_new = &fallback_doms;
+               cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
+       }
+
+       /* Destroy deleted domains */
+       for (i = 0; i < ndoms_cur; i++) {
+               for (j = 0; j < ndoms_new; j++) {
+                       if (cpus_equal(doms_cur[i], doms_new[j]))
+                               goto match1;
+               }
+               /* no match - a current sched domain not in new doms_new[] */
+               detach_destroy_domains(doms_cur + i);
+match1:
+               ;
+       }
 
-       /* Detach sched domains from all of the affected cpus */
-       detach_destroy_domains(&change_map);
-       if (!cpus_empty(*partition1))
-               err = build_sched_domains(partition1);
-       if (!err && !cpus_empty(*partition2))
-               err = build_sched_domains(partition2);
+       /* Build new domains */
+       for (i = 0; i < ndoms_new; i++) {
+               for (j = 0; j < ndoms_cur; j++) {
+                       if (cpus_equal(doms_new[i], doms_cur[j]))
+                               goto match2;
+               }
+               /* no match - add a new doms_new */
+               build_sched_domains(doms_new + i);
+match2:
+               ;
+       }
 
-       return err;
+       /* Remember the new sched domains */
+       if (doms_cur != &fallback_doms)
+               kfree(doms_cur);
+       doms_cur = doms_new;
+       ndoms_cur = ndoms_new;
 }
 
 #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
@@ -6384,8 +6627,6 @@ void __init sched_init_smp(void)
        /* XXX: Theoretical race here - CPU may be hotplugged now */
        hotcpu_notifier(update_sched_domains, 0);
 
-       init_sched_domain_sysctl();
-
        /* Move init over to a non-isolated CPU */
        if (set_cpus_allowed(current, non_isolated_cpus) < 0)
                BUG();
@@ -6560,6 +6801,12 @@ void normalize_rt_tasks(void)
 
        read_lock_irq(&tasklist_lock);
        do_each_thread(g, p) {
+               /*
+                * Only normalize user tasks:
+                */
+               if (!p->mm)
+                       continue;
+
                p->se.exec_start                = 0;
 #ifdef CONFIG_SCHEDSTATS
                p->se.wait_start                = 0;
@@ -6581,8 +6828,7 @@ void normalize_rt_tasks(void)
                spin_lock_irqsave(&p->pi_lock, flags);
                rq = __task_rq_lock(p);
 
-               if (!is_migration_thread(p, rq))
-                       normalize_task(rq, p);
+               normalize_task(rq, p);
 
                __task_rq_unlock(rq);
                spin_unlock_irqrestore(&p->pi_lock, flags);
@@ -6819,8 +7065,6 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares)
        if (tg->shares == shares)
                goto done;
 
-       /* return -EINVAL if the new value is not sane */
-
        tg->shares = shares;
        for_each_possible_cpu(i)
                set_se_shares(tg->se[i], shares);