Merge branch 'linus' into tracing/mmiotrace-mergefixups
[linux-2.6.git] / kernel / sched.c
index 78482e5..2a7ad35 100644 (file)
 #include <linux/unistd.h>
 #include <linux/pagemap.h>
 #include <linux/hrtimer.h>
+#include <linux/tick.h>
+#include <linux/bootmem.h>
+#include <linux/debugfs.h>
+#include <linux/ctype.h>
+#include <linux/ftrace.h>
 
 #include <asm/tlb.h>
 #include <asm/irq_regs.h>
 
 /*
- * Scheduler clock - returns current time in nanosec units.
- * This is default implementation.
- * Architectures and sub-architectures can override this.
- */
-unsigned long long __attribute__((weak)) sched_clock(void)
-{
-       return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ);
-}
-
-/*
  * Convert user-nice values [ -20 ... 0 ... 19 ]
  * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
  * and back.
@@ -114,6 +109,11 @@ unsigned long long __attribute__((weak)) sched_clock(void)
  */
 #define DEF_TIMESLICE          (100 * HZ / 1000)
 
+/*
+ * single value that denotes runtime == period, ie unlimited time.
+ */
+#define RUNTIME_INF    ((u64)~0ULL)
+
 #ifdef CONFIG_SMP
 /*
  * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
@@ -137,7 +137,7 @@ static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
 
 static inline int rt_policy(int policy)
 {
-       if (unlikely(policy == SCHED_FIFO) || unlikely(policy == SCHED_RR))
+       if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
                return 1;
        return 0;
 }
@@ -155,6 +155,90 @@ struct rt_prio_array {
        struct list_head queue[MAX_RT_PRIO];
 };
 
+struct rt_bandwidth {
+       /* nests inside the rq lock: */
+       spinlock_t              rt_runtime_lock;
+       ktime_t                 rt_period;
+       u64                     rt_runtime;
+       struct hrtimer          rt_period_timer;
+};
+
+static struct rt_bandwidth def_rt_bandwidth;
+
+static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
+
+static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
+{
+       struct rt_bandwidth *rt_b =
+               container_of(timer, struct rt_bandwidth, rt_period_timer);
+       ktime_t now;
+       int overrun;
+       int idle = 0;
+
+       for (;;) {
+               now = hrtimer_cb_get_time(timer);
+               overrun = hrtimer_forward(timer, now, rt_b->rt_period);
+
+               if (!overrun)
+                       break;
+
+               idle = do_sched_rt_period_timer(rt_b, overrun);
+       }
+
+       return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
+}
+
+static
+void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
+{
+       rt_b->rt_period = ns_to_ktime(period);
+       rt_b->rt_runtime = runtime;
+
+       spin_lock_init(&rt_b->rt_runtime_lock);
+
+       hrtimer_init(&rt_b->rt_period_timer,
+                       CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+       rt_b->rt_period_timer.function = sched_rt_period_timer;
+       rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
+}
+
+static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
+{
+       ktime_t now;
+
+       if (rt_b->rt_runtime == RUNTIME_INF)
+               return;
+
+       if (hrtimer_active(&rt_b->rt_period_timer))
+               return;
+
+       spin_lock(&rt_b->rt_runtime_lock);
+       for (;;) {
+               if (hrtimer_active(&rt_b->rt_period_timer))
+                       break;
+
+               now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
+               hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
+               hrtimer_start(&rt_b->rt_period_timer,
+                             rt_b->rt_period_timer.expires,
+                             HRTIMER_MODE_ABS);
+       }
+       spin_unlock(&rt_b->rt_runtime_lock);
+}
+
+#ifdef CONFIG_RT_GROUP_SCHED
+static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
+{
+       hrtimer_cancel(&rt_b->rt_period_timer);
+}
+#endif
+
+/*
+ * sched_domains_mutex serializes calls to arch_init_sched_domains,
+ * detach_destroy_domains and partition_sched_domains.
+ */
+static DEFINE_MUTEX(sched_domains_mutex);
+
 #ifdef CONFIG_GROUP_SCHED
 
 #include <linux/cgroup.h>
@@ -181,29 +265,39 @@ struct task_group {
        struct sched_rt_entity **rt_se;
        struct rt_rq **rt_rq;
 
-       u64 rt_runtime;
+       struct rt_bandwidth rt_bandwidth;
 #endif
 
        struct rcu_head rcu;
        struct list_head list;
+
+       struct task_group *parent;
+       struct list_head siblings;
+       struct list_head children;
 };
 
+#ifdef CONFIG_USER_SCHED
+
+/*
+ * Root task group.
+ *     Every UID task group (including init_task_group aka UID-0) will
+ *     be a child to this group.
+ */
+struct task_group root_task_group;
+
 #ifdef CONFIG_FAIR_GROUP_SCHED
 /* Default task group's sched entity on each cpu */
 static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
 /* Default task group's cfs_rq on each cpu */
 static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
-
-static struct sched_entity *init_sched_entity_p[NR_CPUS];
-static struct cfs_rq *init_cfs_rq_p[NR_CPUS];
 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
 static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
-
-static struct sched_rt_entity *init_sched_rt_entity_p[NR_CPUS];
-static struct rt_rq *init_rt_rq_p[NR_CPUS];
+#endif
+#else
+#define root_task_group init_task_group
 #endif
 
 /* task_group_lock serializes add/remove of task groups and also changes to
@@ -211,9 +305,6 @@ static struct rt_rq *init_rt_rq_p[NR_CPUS];
  */
 static DEFINE_SPINLOCK(task_group_lock);
 
-/* doms_cur_mutex serializes access to doms_cur[] array */
-static DEFINE_MUTEX(doms_cur_mutex);
-
 #ifdef CONFIG_FAIR_GROUP_SCHED
 #ifdef CONFIG_USER_SCHED
 # define INIT_TASK_GROUP_LOAD  (2*NICE_0_LOAD)
@@ -221,23 +312,24 @@ static DEFINE_MUTEX(doms_cur_mutex);
 # define INIT_TASK_GROUP_LOAD  NICE_0_LOAD
 #endif
 
+/*
+ * A weight of 0 or 1 can cause arithmetics problems.
+ * A weight of a cfs_rq is the sum of weights of which entities
+ * are queued on this cfs_rq, so a weight of a entity should not be
+ * too large, so as the shares value of a task group.
+ * (The default weight is 1024 - so there's no practical
+ *  limitation from this.)
+ */
+#define MIN_SHARES     2
+#define MAX_SHARES     (1UL << 18)
+
 static int init_task_group_load = INIT_TASK_GROUP_LOAD;
 #endif
 
 /* Default task group.
  *     Every task in system belong to this group at bootup.
  */
-struct task_group init_task_group = {
-#ifdef CONFIG_FAIR_GROUP_SCHED
-       .se     = init_sched_entity_p,
-       .cfs_rq = init_cfs_rq_p,
-#endif
-
-#ifdef CONFIG_RT_GROUP_SCHED
-       .rt_se  = init_sched_rt_entity_p,
-       .rt_rq  = init_rt_rq_p,
-#endif
-};
+struct task_group init_task_group;
 
 /* return group to which a task belongs */
 static inline struct task_group *task_group(struct task_struct *p)
@@ -269,21 +361,9 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 #endif
 }
 
-static inline void lock_doms_cur(void)
-{
-       mutex_lock(&doms_cur_mutex);
-}
-
-static inline void unlock_doms_cur(void)
-{
-       mutex_unlock(&doms_cur_mutex);
-}
-
 #else
 
 static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
-static inline void lock_doms_cur(void) { }
-static inline void unlock_doms_cur(void) { }
 
 #endif /* CONFIG_GROUP_SCHED */
 
@@ -297,8 +377,12 @@ struct cfs_rq {
 
        struct rb_root tasks_timeline;
        struct rb_node *rb_leftmost;
-       struct rb_node *rb_load_balance_curr;
-       /* 'curr' points to currently running entity on this cfs_rq.
+
+       struct list_head tasks;
+       struct list_head *balance_iterator;
+
+       /*
+        * 'curr' points to currently running entity on this cfs_rq.
         * It is set to NULL otherwise (i.e when none are currently running).
         */
        struct sched_entity *curr, *next;
@@ -334,6 +418,9 @@ struct rt_rq {
 #endif
        int rt_throttled;
        u64 rt_time;
+       u64 rt_runtime;
+       /* Nests inside the rq lock: */
+       spinlock_t rt_runtime_lock;
 
 #ifdef CONFIG_RT_GROUP_SCHED
        unsigned long rt_nr_boosted;
@@ -396,6 +483,7 @@ struct rq {
        unsigned long cpu_load[CPU_LOAD_IDX_MAX];
        unsigned char idle_at_tick;
 #ifdef CONFIG_NO_HZ
+       unsigned long last_tick_seen;
        unsigned char in_nohz_recently;
 #endif
        /* capture load from *all* tasks on this cpu: */
@@ -405,8 +493,6 @@ struct rq {
 
        struct cfs_rq cfs;
        struct rt_rq rt;
-       u64 rt_period_expire;
-       int rt_throttled;
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
        /* list of leaf cfs_rq on this cpu: */
@@ -428,13 +514,7 @@ struct rq {
        unsigned long next_balance;
        struct mm_struct *prev_mm;
 
-       u64 clock, prev_clock_raw;
-       s64 clock_max_delta;
-
-       unsigned int clock_warps, clock_overflows, clock_underflows;
-       u64 idle_clock;
-       unsigned int clock_deep_idle_events;
-       u64 tick_timestamp;
+       u64 clock;
 
        atomic_t nr_iowait;
 
@@ -500,53 +580,6 @@ static inline int cpu_of(struct rq *rq)
 }
 
 /*
- * Update the per-runqueue clock, as finegrained as the platform can give
- * us, but without assuming monotonicity, etc.:
- */
-static void __update_rq_clock(struct rq *rq)
-{
-       u64 prev_raw = rq->prev_clock_raw;
-       u64 now = sched_clock();
-       s64 delta = now - prev_raw;
-       u64 clock = rq->clock;
-
-#ifdef CONFIG_SCHED_DEBUG
-       WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
-#endif
-       /*
-        * Protect against sched_clock() occasionally going backwards:
-        */
-       if (unlikely(delta < 0)) {
-               clock++;
-               rq->clock_warps++;
-       } else {
-               /*
-                * Catch too large forward jumps too:
-                */
-               if (unlikely(clock + delta > rq->tick_timestamp + TICK_NSEC)) {
-                       if (clock < rq->tick_timestamp + TICK_NSEC)
-                               clock = rq->tick_timestamp + TICK_NSEC;
-                       else
-                               clock++;
-                       rq->clock_overflows++;
-               } else {
-                       if (unlikely(delta > rq->clock_max_delta))
-                               rq->clock_max_delta = delta;
-                       clock += delta;
-               }
-       }
-
-       rq->prev_clock_raw = now;
-       rq->clock = clock;
-}
-
-static void update_rq_clock(struct rq *rq)
-{
-       if (likely(smp_processor_id() == cpu_of(rq)))
-               __update_rq_clock(rq);
-}
-
-/*
  * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
  * See detach_destroy_domains: synchronize_sched for details.
  *
@@ -561,21 +594,9 @@ static void update_rq_clock(struct rq *rq)
 #define task_rq(p)             cpu_rq(task_cpu(p))
 #define cpu_curr(cpu)          (cpu_rq(cpu)->curr)
 
-unsigned long rt_needs_cpu(int cpu)
+static inline void update_rq_clock(struct rq *rq)
 {
-       struct rq *rq = cpu_rq(cpu);
-       u64 delta;
-
-       if (!rq->rt_throttled)
-               return 0;
-
-       if (rq->clock > rq->rt_period_expire)
-               return 1;
-
-       delta = rq->rt_period_expire - rq->clock;
-       do_div(delta, NSEC_PER_SEC / HZ);
-
-       return (unsigned long)delta;
+       rq->clock = sched_clock_cpu(cpu_of(rq));
 }
 
 /*
@@ -587,25 +608,158 @@ unsigned long rt_needs_cpu(int cpu)
 # define const_debug static const
 #endif
 
+/**
+ * runqueue_is_locked
+ *
+ * Returns true if the current cpu runqueue is locked.
+ * This interface allows printk to be called with the runqueue lock
+ * held and know whether or not it is OK to wake up the klogd.
+ */
+int runqueue_is_locked(void)
+{
+       int cpu = get_cpu();
+       struct rq *rq = cpu_rq(cpu);
+       int ret;
+
+       ret = spin_is_locked(&rq->lock);
+       put_cpu();
+       return ret;
+}
+
 /*
  * Debugging: various feature bits
  */
+
+#define SCHED_FEAT(name, enabled)      \
+       __SCHED_FEAT_##name ,
+
 enum {
-       SCHED_FEAT_NEW_FAIR_SLEEPERS    = 1,
-       SCHED_FEAT_WAKEUP_PREEMPT       = 2,
-       SCHED_FEAT_START_DEBIT          = 4,
-       SCHED_FEAT_HRTICK               = 8,
-       SCHED_FEAT_DOUBLE_TICK          = 16,
+#include "sched_features.h"
 };
 
+#undef SCHED_FEAT
+
+#define SCHED_FEAT(name, enabled)      \
+       (1UL << __SCHED_FEAT_##name) * enabled |
+
 const_debug unsigned int sysctl_sched_features =
-               SCHED_FEAT_NEW_FAIR_SLEEPERS    * 1 |
-               SCHED_FEAT_WAKEUP_PREEMPT       * 1 |
-               SCHED_FEAT_START_DEBIT          * 1 |
-               SCHED_FEAT_HRTICK               * 1 |
-               SCHED_FEAT_DOUBLE_TICK          * 0;
+#include "sched_features.h"
+       0;
+
+#undef SCHED_FEAT
+
+#ifdef CONFIG_SCHED_DEBUG
+#define SCHED_FEAT(name, enabled)      \
+       #name ,
+
+static __read_mostly char *sched_feat_names[] = {
+#include "sched_features.h"
+       NULL
+};
+
+#undef SCHED_FEAT
+
+static int sched_feat_open(struct inode *inode, struct file *filp)
+{
+       filp->private_data = inode->i_private;
+       return 0;
+}
+
+static ssize_t
+sched_feat_read(struct file *filp, char __user *ubuf,
+               size_t cnt, loff_t *ppos)
+{
+       char *buf;
+       int r = 0;
+       int len = 0;
+       int i;
+
+       for (i = 0; sched_feat_names[i]; i++) {
+               len += strlen(sched_feat_names[i]);
+               len += 4;
+       }
+
+       buf = kmalloc(len + 2, GFP_KERNEL);
+       if (!buf)
+               return -ENOMEM;
+
+       for (i = 0; sched_feat_names[i]; i++) {
+               if (sysctl_sched_features & (1UL << i))
+                       r += sprintf(buf + r, "%s ", sched_feat_names[i]);
+               else
+                       r += sprintf(buf + r, "NO_%s ", sched_feat_names[i]);
+       }
+
+       r += sprintf(buf + r, "\n");
+       WARN_ON(r >= len + 2);
+
+       r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
+
+       kfree(buf);
+
+       return r;
+}
+
+static ssize_t
+sched_feat_write(struct file *filp, const char __user *ubuf,
+               size_t cnt, loff_t *ppos)
+{
+       char buf[64];
+       char *cmp = buf;
+       int neg = 0;
+       int i;
+
+       if (cnt > 63)
+               cnt = 63;
+
+       if (copy_from_user(&buf, ubuf, cnt))
+               return -EFAULT;
+
+       buf[cnt] = 0;
+
+       if (strncmp(buf, "NO_", 3) == 0) {
+               neg = 1;
+               cmp += 3;
+       }
+
+       for (i = 0; sched_feat_names[i]; i++) {
+               int len = strlen(sched_feat_names[i]);
+
+               if (strncmp(cmp, sched_feat_names[i], len) == 0) {
+                       if (neg)
+                               sysctl_sched_features &= ~(1UL << i);
+                       else
+                               sysctl_sched_features |= (1UL << i);
+                       break;
+               }
+       }
+
+       if (!sched_feat_names[i])
+               return -EINVAL;
 
-#define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x)
+       filp->f_pos += cnt;
+
+       return cnt;
+}
+
+static struct file_operations sched_feat_fops = {
+       .open   = sched_feat_open,
+       .read   = sched_feat_read,
+       .write  = sched_feat_write,
+};
+
+static __init int sched_init_debug(void)
+{
+       debugfs_create_file("sched_features", 0644, NULL, NULL,
+                       &sched_feat_fops);
+
+       return 0;
+}
+late_initcall(sched_init_debug);
+
+#endif
+
+#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
 
 /*
  * Number of tasks to iterate in a single balance run.
@@ -627,20 +781,58 @@ static __read_mostly int scheduler_running;
  */
 int sysctl_sched_rt_runtime = 950000;
 
-/*
- * single value that denotes runtime == period, ie unlimited time.
- */
-#define RUNTIME_INF    ((u64)~0ULL)
+static inline u64 global_rt_period(void)
+{
+       return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
+}
+
+static inline u64 global_rt_runtime(void)
+{
+       if (sysctl_sched_rt_period < 0)
+               return RUNTIME_INF;
+
+       return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
+}
+
+unsigned long long time_sync_thresh = 100000;
+
+static DEFINE_PER_CPU(unsigned long long, time_offset);
+static DEFINE_PER_CPU(unsigned long long, prev_cpu_time);
 
 /*
- * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
- * clock constructed from sched_clock():
+ * Global lock which we take every now and then to synchronize
+ * the CPUs time. This method is not warp-safe, but it's good
+ * enough to synchronize slowly diverging time sources and thus
+ * it's good enough for tracing:
  */
-unsigned long long cpu_clock(int cpu)
+static DEFINE_SPINLOCK(time_sync_lock);
+static unsigned long long prev_global_time;
+
+static unsigned long long __sync_cpu_clock(unsigned long long time, int cpu)
+{
+       /*
+        * We want this inlined, to not get tracer function calls
+        * in this critical section:
+        */
+       spin_acquire(&time_sync_lock.dep_map, 0, 0, _THIS_IP_);
+       __raw_spin_lock(&time_sync_lock.raw_lock);
+
+       if (time < prev_global_time) {
+               per_cpu(time_offset, cpu) += prev_global_time - time;
+               time = prev_global_time;
+       } else {
+               prev_global_time = time;
+       }
+
+       __raw_spin_unlock(&time_sync_lock.raw_lock);
+       spin_release(&time_sync_lock.dep_map, 1, _THIS_IP_);
+
+       return time;
+}
+
+static unsigned long long __cpu_clock(int cpu)
 {
        unsigned long long now;
-       unsigned long flags;
-       struct rq *rq;
 
        /*
         * Only call sched_clock() if the scheduler has already been
@@ -649,13 +841,32 @@ unsigned long long cpu_clock(int cpu)
        if (unlikely(!scheduler_running))
                return 0;
 
+       now = sched_clock_cpu(cpu);
+
+       return now;
+}
+
+/*
+ * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
+ * clock constructed from sched_clock():
+ */
+unsigned long long cpu_clock(int cpu)
+{
+       unsigned long long prev_cpu_time, time, delta_time;
+       unsigned long flags;
+
        local_irq_save(flags);
-       rq = cpu_rq(cpu);
-       update_rq_clock(rq);
-       now = rq->clock;
+       prev_cpu_time = per_cpu(prev_cpu_time, cpu);
+       time = __cpu_clock(cpu) + per_cpu(time_offset, cpu);
+       delta_time = time-prev_cpu_time;
+
+       if (unlikely(delta_time > time_sync_thresh)) {
+               time = __sync_cpu_clock(time, cpu);
+               per_cpu(prev_cpu_time, cpu) = time;
+       }
        local_irq_restore(flags);
 
-       return now;
+       return time;
 }
 EXPORT_SYMBOL_GPL(cpu_clock);
 
@@ -804,43 +1015,6 @@ static struct rq *this_rq_lock(void)
        return rq;
 }
 
-/*
- * We are going deep-idle (irqs are disabled):
- */
-void sched_clock_idle_sleep_event(void)
-{
-       struct rq *rq = cpu_rq(smp_processor_id());
-
-       spin_lock(&rq->lock);
-       __update_rq_clock(rq);
-       spin_unlock(&rq->lock);
-       rq->clock_deep_idle_events++;
-}
-EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event);
-
-/*
- * We just idled delta nanoseconds (called with irqs disabled):
- */
-void sched_clock_idle_wakeup_event(u64 delta_ns)
-{
-       struct rq *rq = cpu_rq(smp_processor_id());
-       u64 now = sched_clock();
-
-       rq->idle_clock += delta_ns;
-       /*
-        * Override the previous timestamp and ignore all
-        * sched_clock() deltas that occured while we idled,
-        * and use the PM-provided delta_ns to advance the
-        * rq clock:
-        */
-       spin_lock(&rq->lock);
-       rq->prev_clock_raw = now;
-       rq->clock += delta_ns;
-       spin_unlock(&rq->lock);
-       touch_softlockup_watchdog();
-}
-EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
-
 static void __resched_task(struct task_struct *p, int tif_bit);
 
 static inline void resched_task(struct task_struct *p)
@@ -876,6 +1050,7 @@ static inline void resched_rq(struct rq *rq)
 enum {
        HRTICK_SET,             /* re-programm hrtick_timer */
        HRTICK_RESET,           /* not a new slice */
+       HRTICK_BLOCK,           /* stop hrtick operations */
 };
 
 /*
@@ -887,6 +1062,8 @@ static inline int hrtick_enabled(struct rq *rq)
 {
        if (!sched_feat(HRTICK))
                return 0;
+       if (unlikely(test_bit(HRTICK_BLOCK, &rq->hrtick_flags)))
+               return 0;
        return hrtimer_is_hres_active(&rq->hrtick_timer);
 }
 
@@ -962,14 +1139,72 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
        WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
 
        spin_lock(&rq->lock);
-       __update_rq_clock(rq);
+       update_rq_clock(rq);
        rq->curr->sched_class->task_tick(rq, rq->curr, 1);
        spin_unlock(&rq->lock);
 
        return HRTIMER_NORESTART;
 }
 
-static inline void init_rq_hrtick(struct rq *rq)
+#ifdef CONFIG_SMP
+static void hotplug_hrtick_disable(int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+       unsigned long flags;
+
+       spin_lock_irqsave(&rq->lock, flags);
+       rq->hrtick_flags = 0;
+       __set_bit(HRTICK_BLOCK, &rq->hrtick_flags);
+       spin_unlock_irqrestore(&rq->lock, flags);
+
+       hrtick_clear(rq);
+}
+
+static void hotplug_hrtick_enable(int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+       unsigned long flags;
+
+       spin_lock_irqsave(&rq->lock, flags);
+       __clear_bit(HRTICK_BLOCK, &rq->hrtick_flags);
+       spin_unlock_irqrestore(&rq->lock, flags);
+}
+
+static int
+hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
+{
+       int cpu = (int)(long)hcpu;
+
+       switch (action) {
+       case CPU_UP_CANCELED:
+       case CPU_UP_CANCELED_FROZEN:
+       case CPU_DOWN_PREPARE:
+       case CPU_DOWN_PREPARE_FROZEN:
+       case CPU_DEAD:
+       case CPU_DEAD_FROZEN:
+               hotplug_hrtick_disable(cpu);
+               return NOTIFY_OK;
+
+       case CPU_UP_PREPARE:
+       case CPU_UP_PREPARE_FROZEN:
+       case CPU_DOWN_FAILED:
+       case CPU_DOWN_FAILED_FROZEN:
+       case CPU_ONLINE:
+       case CPU_ONLINE_FROZEN:
+               hotplug_hrtick_enable(cpu);
+               return NOTIFY_OK;
+       }
+
+       return NOTIFY_DONE;
+}
+
+static void init_hrtick(void)
+{
+       hotcpu_notifier(hotplug_hrtick, 0);
+}
+#endif /* CONFIG_SMP */
+
+static void init_rq_hrtick(struct rq *rq)
 {
        rq->hrtick_flags = 0;
        hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
@@ -1006,6 +1241,10 @@ static inline void init_rq_hrtick(struct rq *rq)
 void hrtick_resched(void)
 {
 }
+
+static inline void init_hrtick(void)
+{
+}
 #endif
 
 /*
@@ -1052,6 +1291,49 @@ static void resched_cpu(int cpu)
        resched_task(cpu_curr(cpu));
        spin_unlock_irqrestore(&rq->lock, flags);
 }
+
+#ifdef CONFIG_NO_HZ
+/*
+ * When add_timer_on() enqueues a timer into the timer wheel of an
+ * idle CPU then this timer might expire before the next timer event
+ * which is scheduled to wake up that CPU. In case of a completely
+ * idle system the next event might even be infinite time into the
+ * future. wake_up_idle_cpu() ensures that the CPU is woken up and
+ * leaves the inner idle loop so the newly added timer is taken into
+ * account when the CPU goes back to idle and evaluates the timer
+ * wheel for the next timer event.
+ */
+void wake_up_idle_cpu(int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+
+       if (cpu == smp_processor_id())
+               return;
+
+       /*
+        * This is safe, as this function is called with the timer
+        * wheel base lock of (cpu) held. When the CPU is on the way
+        * to idle and has not yet set rq->curr to idle then it will
+        * be serialized on the timer wheel base lock and take the new
+        * timer into account automatically.
+        */
+       if (rq->curr != rq->idle)
+               return;
+
+       /*
+        * We can set TIF_RESCHED on the idle task of the other CPU
+        * lockless. The worst case is that the other CPU runs the
+        * idle task through an additional NOOP schedule()
+        */
+       set_tsk_thread_flag(rq->idle, TIF_NEED_RESCHED);
+
+       /* NEED_RESCHED must be visible before we test polling */
+       smp_mb();
+       if (!tsk_is_polling(rq->idle))
+               smp_send_reschedule(cpu);
+}
+#endif
+
 #else
 static void __resched_task(struct task_struct *p, int tif_bit)
 {
@@ -1079,8 +1361,13 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight,
 {
        u64 tmp;
 
-       if (unlikely(!lw->inv_weight))
-               lw->inv_weight = (WMULT_CONST-lw->weight/2) / (lw->weight+1);
+       if (!lw->inv_weight) {
+               if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
+                       lw->inv_weight = 1;
+               else
+                       lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
+                               / (lw->weight+1);
+       }
 
        tmp = (u64)delta_exec * weight;
        /*
@@ -1198,11 +1485,29 @@ static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
 static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
 #endif
 
+static inline void inc_cpu_load(struct rq *rq, unsigned long load)
+{
+       update_load_add(&rq->load, load);
+}
+
+static inline void dec_cpu_load(struct rq *rq, unsigned long load)
+{
+       update_load_sub(&rq->load, load);
+}
+
 #ifdef CONFIG_SMP
 static unsigned long source_load(int cpu, int type);
 static unsigned long target_load(int cpu, int type);
 static unsigned long cpu_avg_load_per_task(int cpu);
 static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
+#else /* CONFIG_SMP */
+
+#ifdef CONFIG_FAIR_GROUP_SCHED
+static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
+{
+}
+#endif
+
 #endif /* CONFIG_SMP */
 
 #include "sched_stats.h"
@@ -1395,7 +1700,7 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
        /*
         * Buddy candidates are cache hot:
         */
-       if (&p->se == cfs_rq_of(&p->se)->next)
+       if (sched_feat(CACHE_HOT_BUDDY) && (&p->se == cfs_rq_of(&p->se)->next))
                return 1;
 
        if (p->sched_class != &fair_sched_class)
@@ -1685,17 +1990,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
  * find_idlest_cpu - find the idlest cpu among the cpus in group.
  */
 static int
-find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
+find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu,
+               cpumask_t *tmp)
 {
-       cpumask_t tmp;
        unsigned long load, min_load = ULONG_MAX;
        int idlest = -1;
        int i;
 
        /* Traverse only the allowed CPUs */
-       cpus_and(tmp, group->cpumask, p->cpus_allowed);
+       cpus_and(*tmp, group->cpumask, p->cpus_allowed);
 
-       for_each_cpu_mask(i, tmp) {
+       for_each_cpu_mask(i, *tmp) {
                load = weighted_cpuload(i);
 
                if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -1734,7 +2039,7 @@ static int sched_balance_self(int cpu, int flag)
        }
 
        while (sd) {
-               cpumask_t span;
+               cpumask_t span, tmpmask;
                struct sched_group *group;
                int new_cpu, weight;
 
@@ -1750,7 +2055,7 @@ static int sched_balance_self(int cpu, int flag)
                        continue;
                }
 
-               new_cpu = find_idlest_cpu(group, t, cpu);
+               new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask);
                if (new_cpu == -1 || new_cpu == cpu) {
                        /* Now try balancing at a lower domain level of cpu */
                        sd = sd->child;
@@ -1796,6 +2101,9 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
        long old_state;
        struct rq *rq;
 
+       if (!sched_feat(SYNC_WAKEUPS))
+               sync = 0;
+
        smp_wmb();
        rq = task_rq_lock(p, &flags);
        old_state = p->state;
@@ -1860,6 +2168,9 @@ out_activate:
        success = 1;
 
 out_running:
+       trace_mark(kernel_sched_wakeup,
+               "pid %d state %ld ## rq %p task %p rq->curr %p",
+               p->pid, p->state, rq, p, rq->curr);
        check_preempt_curr(rq, p);
 
        p->state = TASK_RUNNING;
@@ -1912,6 +2223,7 @@ static void __sched_fork(struct task_struct *p)
 
        INIT_LIST_HEAD(&p->rt.run_list);
        p->se.on_rq = 0;
+       INIT_LIST_HEAD(&p->se.group_node);
 
 #ifdef CONFIG_PREEMPT_NOTIFIERS
        INIT_HLIST_HEAD(&p->preempt_notifiers);
@@ -1989,6 +2301,9 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
                p->sched_class->task_new(rq, p);
                inc_nr_running(p, rq);
        }
+       trace_mark(kernel_sched_wakeup_new,
+               "pid %d state %ld ## rq %p task %p rq->curr %p",
+               p->pid, p->state, rq, p, rq->curr);
        check_preempt_curr(rq, p);
 #ifdef CONFIG_SMP
        if (p->sched_class->task_wake_up)
@@ -2161,6 +2476,11 @@ context_switch(struct rq *rq, struct task_struct *prev,
        struct mm_struct *mm, *oldmm;
 
        prepare_task_switch(rq, prev, next);
+       trace_mark(kernel_sched_schedule,
+               "prev_pid %d next_pid %d prev_state %ld "
+               "## rq %p prev %p next %p",
+               prev->pid, next->pid, prev->state,
+               rq, prev, next);
        mm = next->mm;
        oldmm = prev->active_mm;
        /*
@@ -2631,7 +2951,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
 static struct sched_group *
 find_busiest_group(struct sched_domain *sd, int this_cpu,
                   unsigned long *imbalance, enum cpu_idle_type idle,
-                  int *sd_idle, cpumask_t *cpus, int *balance)
+                  int *sd_idle, const cpumask_t *cpus, int *balance)
 {
        struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
        unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -2932,7 +3252,7 @@ ret:
  */
 static struct rq *
 find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
-                  unsigned long imbalance, cpumask_t *cpus)
+                  unsigned long imbalance, const cpumask_t *cpus)
 {
        struct rq *busiest = NULL, *rq;
        unsigned long max_load = 0;
@@ -2971,15 +3291,16 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
  */
 static int load_balance(int this_cpu, struct rq *this_rq,
                        struct sched_domain *sd, enum cpu_idle_type idle,
-                       int *balance)
+                       int *balance, cpumask_t *cpus)
 {
        int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
        struct sched_group *group;
        unsigned long imbalance;
        struct rq *busiest;
-       cpumask_t cpus = CPU_MASK_ALL;
        unsigned long flags;
 
+       cpus_setall(*cpus);
+
        /*
         * When power savings policy is enabled for the parent domain, idle
         * sibling can pick up load irrespective of busy siblings. In this case,
@@ -2994,7 +3315,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
 
 redo:
        group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
-                                  &cpus, balance);
+                                  cpus, balance);
 
        if (*balance == 0)
                goto out_balanced;
@@ -3004,7 +3325,7 @@ redo:
                goto out_balanced;
        }
 
-       busiest = find_busiest_queue(group, idle, imbalance, &cpus);
+       busiest = find_busiest_queue(group, idle, imbalance, cpus);
        if (!busiest) {
                schedstat_inc(sd, lb_nobusyq[idle]);
                goto out_balanced;
@@ -3037,8 +3358,8 @@ redo:
 
                /* All tasks on this runqueue were pinned by CPU affinity */
                if (unlikely(all_pinned)) {
-                       cpu_clear(cpu_of(busiest), cpus);
-                       if (!cpus_empty(cpus))
+                       cpu_clear(cpu_of(busiest), *cpus);
+                       if (!cpus_empty(*cpus))
                                goto redo;
                        goto out_balanced;
                }
@@ -3123,7 +3444,8 @@ out_one_pinned:
  * this_rq is locked.
  */
 static int
-load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
+load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
+                       cpumask_t *cpus)
 {
        struct sched_group *group;
        struct rq *busiest = NULL;
@@ -3131,7 +3453,8 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
        int ld_moved = 0;
        int sd_idle = 0;
        int all_pinned = 0;
-       cpumask_t cpus = CPU_MASK_ALL;
+
+       cpus_setall(*cpus);
 
        /*
         * When power savings policy is enabled for the parent domain, idle
@@ -3146,14 +3469,13 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
        schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
 redo:
        group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
-                                  &sd_idle, &cpus, NULL);
+                                  &sd_idle, cpus, NULL);
        if (!group) {
                schedstat_inc(sd, lb_nobusyg[CPU_NEWLY_IDLE]);
                goto out_balanced;
        }
 
-       busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance,
-                               &cpus);
+       busiest = find_busiest_queue(group, CPU_NEWLY_IDLE, imbalance, cpus);
        if (!busiest) {
                schedstat_inc(sd, lb_nobusyq[CPU_NEWLY_IDLE]);
                goto out_balanced;
@@ -3175,8 +3497,8 @@ redo:
                spin_unlock(&busiest->lock);
 
                if (unlikely(all_pinned)) {
-                       cpu_clear(cpu_of(busiest), cpus);
-                       if (!cpus_empty(cpus))
+                       cpu_clear(cpu_of(busiest), *cpus);
+                       if (!cpus_empty(*cpus))
                                goto redo;
                }
        }
@@ -3210,6 +3532,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
        struct sched_domain *sd;
        int pulled_task = -1;
        unsigned long next_balance = jiffies + HZ;
+       cpumask_t tmpmask;
 
        for_each_domain(this_cpu, sd) {
                unsigned long interval;
@@ -3219,8 +3542,8 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
 
                if (sd->flags & SD_BALANCE_NEWIDLE)
                        /* If we've pulled tasks over stop searching: */
-                       pulled_task = load_balance_newidle(this_cpu,
-                                                               this_rq, sd);
+                       pulled_task = load_balance_newidle(this_cpu, this_rq,
+                                                          sd, &tmpmask);
 
                interval = msecs_to_jiffies(sd->balance_interval);
                if (time_after(next_balance, sd->last_balance + interval))
@@ -3379,6 +3702,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
        /* Earliest time when we have to do rebalance again */
        unsigned long next_balance = jiffies + 60*HZ;
        int update_next_balance = 0;
+       cpumask_t tmp;
 
        for_each_domain(cpu, sd) {
                if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3402,7 +3726,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                }
 
                if (time_after_eq(jiffies, sd->last_balance + interval)) {
-                       if (load_balance(cpu, rq, sd, idle, &balance)) {
+                       if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) {
                                /*
                                 * We've pulled tasks over so either we're no
                                 * longer idle, or one of our SMT siblings is
@@ -3518,7 +3842,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
                         */
                        int ilb = first_cpu(nohz.cpu_mask);
 
-                       if (ilb != NR_CPUS)
+                       if (ilb < nr_cpu_ids)
                                resched_cpu(ilb);
                }
        }
@@ -3645,8 +3969,10 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
        struct rq *rq = this_rq();
        cputime64_t tmp;
 
-       if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0))
-               return account_guest_time(p, cputime);
+       if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
+               account_guest_time(p, cputime);
+               return;
+       }
 
        p->stime = cputime_add(p->stime, cputime);
 
@@ -3710,21 +4036,13 @@ void scheduler_tick(void)
        int cpu = smp_processor_id();
        struct rq *rq = cpu_rq(cpu);
        struct task_struct *curr = rq->curr;
-       u64 next_tick = rq->tick_timestamp + TICK_NSEC;
+
+       sched_clock_tick();
 
        spin_lock(&rq->lock);
-       __update_rq_clock(rq);
-       /*
-        * Let rq->clock advance by at least TICK_NSEC:
-        */
-       if (unlikely(rq->clock < next_tick)) {
-               rq->clock = next_tick;
-               rq->clock_underflows++;
-       }
-       rq->tick_timestamp = rq->clock;
+       update_rq_clock(rq);
        update_cpu_load(rq);
        curr->sched_class->task_tick(rq, curr, 0);
-       update_sched_rt_period(rq);
        spin_unlock(&rq->lock);
 
 #ifdef CONFIG_SMP
@@ -3733,26 +4051,44 @@ void scheduler_tick(void)
 #endif
 }
 
-#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)
+#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
+                               defined(CONFIG_PREEMPT_TRACER))
+
+static inline unsigned long get_parent_ip(unsigned long addr)
+{
+       if (in_lock_functions(addr)) {
+               addr = CALLER_ADDR2;
+               if (in_lock_functions(addr))
+                       addr = CALLER_ADDR3;
+       }
+       return addr;
+}
 
 void __kprobes add_preempt_count(int val)
 {
+#ifdef CONFIG_DEBUG_PREEMPT
        /*
         * Underflow?
         */
        if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
                return;
+#endif
        preempt_count() += val;
+#ifdef CONFIG_DEBUG_PREEMPT
        /*
         * Spinlock count overflowing soon?
         */
        DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
                                PREEMPT_MASK - 10);
+#endif
+       if (preempt_count() == val)
+               trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 }
 EXPORT_SYMBOL(add_preempt_count);
 
 void __kprobes sub_preempt_count(int val)
 {
+#ifdef CONFIG_DEBUG_PREEMPT
        /*
         * Underflow?
         */
@@ -3764,7 +4100,10 @@ void __kprobes sub_preempt_count(int val)
        if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
                        !(preempt_count() & PREEMPT_MASK)))
                return;
+#endif
 
+       if (preempt_count() == val)
+               trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
        preempt_count() -= val;
 }
 EXPORT_SYMBOL(sub_preempt_count);
@@ -3801,7 +4140,7 @@ static inline void schedule_debug(struct task_struct *prev)
         * schedule() atomically, we ignore that path for now.
         * Otherwise, whine if we are scheduling when we should not be.
         */
-       if (unlikely(in_atomic_preempt_off()) && unlikely(!prev->exit_state))
+       if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
                __schedule_bug(prev);
 
        profile_hit(SCHED_PROFILING, __builtin_return_address(0));
@@ -3876,17 +4215,15 @@ need_resched_nonpreemptible:
         * Do the rq-clock update outside the rq lock:
         */
        local_irq_disable();
-       __update_rq_clock(rq);
+       update_rq_clock(rq);
        spin_lock(&rq->lock);
        clear_tsk_need_resched(prev);
 
        if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
-               if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
-                               signal_pending(prev))) {
+               if (unlikely(signal_pending_state(prev->state, prev)))
                        prev->state = TASK_RUNNING;
-               } else {
+               else
                        deactivate_task(rq, prev, 1);
-               }
                switch_count = &prev->nvcsw;
        }
 
@@ -3901,9 +4238,9 @@ need_resched_nonpreemptible:
        prev->sched_class->put_prev_task(rq, prev);
        next = pick_next_task(rq, prev);
 
-       sched_info_switch(prev, next);
-
        if (likely(prev != next)) {
+               sched_info_switch(prev, next);
+
                rq->nr_switches++;
                rq->curr = next;
                ++*switch_count;
@@ -3938,8 +4275,6 @@ EXPORT_SYMBOL(schedule);
 asmlinkage void __sched preempt_schedule(void)
 {
        struct thread_info *ti = current_thread_info();
-       struct task_struct *task = current;
-       int saved_lock_depth;
 
        /*
         * If there is a non-zero preempt_count or interrupts are disabled,
@@ -3950,16 +4285,7 @@ asmlinkage void __sched preempt_schedule(void)
 
        do {
                add_preempt_count(PREEMPT_ACTIVE);
-
-               /*
-                * We keep the big kernel semaphore locked, but we
-                * clear ->lock_depth so that schedule() doesnt
-                * auto-release the semaphore:
-                */
-               saved_lock_depth = task->lock_depth;
-               task->lock_depth = -1;
                schedule();
-               task->lock_depth = saved_lock_depth;
                sub_preempt_count(PREEMPT_ACTIVE);
 
                /*
@@ -3980,26 +4306,15 @@ EXPORT_SYMBOL(preempt_schedule);
 asmlinkage void __sched preempt_schedule_irq(void)
 {
        struct thread_info *ti = current_thread_info();
-       struct task_struct *task = current;
-       int saved_lock_depth;
 
        /* Catch callers which need to be fixed */
        BUG_ON(ti->preempt_count || !irqs_disabled());
 
        do {
                add_preempt_count(PREEMPT_ACTIVE);
-
-               /*
-                * We keep the big kernel semaphore locked, but we
-                * clear ->lock_depth so that schedule() doesnt
-                * auto-release the semaphore:
-                */
-               saved_lock_depth = task->lock_depth;
-               task->lock_depth = -1;
                local_irq_enable();
                schedule();
                local_irq_disable();
-               task->lock_depth = saved_lock_depth;
                sub_preempt_count(PREEMPT_ACTIVE);
 
                /*
@@ -4134,22 +4449,20 @@ do_wait_for_common(struct completion *x, long timeout, int state)
                             signal_pending(current)) ||
                            (state == TASK_KILLABLE &&
                             fatal_signal_pending(current))) {
-                               __remove_wait_queue(&x->wait, &wait);
-                               return -ERESTARTSYS;
+                               timeout = -ERESTARTSYS;
+                               break;
                        }
                        __set_current_state(state);
                        spin_unlock_irq(&x->wait.lock);
                        timeout = schedule_timeout(timeout);
                        spin_lock_irq(&x->wait.lock);
-                       if (!timeout) {
-                               __remove_wait_queue(&x->wait, &wait);
-                               return timeout;
-                       }
-               } while (!x->done);
+               } while (!x->done && timeout);
                __remove_wait_queue(&x->wait, &wait);
+               if (!x->done)
+                       return timeout;
        }
        x->done--;
-       return timeout;
+       return timeout ?: 1;
 }
 
 static long __sched
@@ -4559,7 +4872,7 @@ recheck:
         * Do not allow realtime tasks into groups that have no runtime
         * assigned.
         */
-       if (rt_policy(policy) && task_group(p)->rt_runtime == 0)
+       if (rt_policy(policy) && task_group(p)->rt_bandwidth.rt_runtime == 0)
                return -EPERM;
 #endif
 
@@ -4721,9 +5034,10 @@ out_unlock:
        return retval;
 }
 
-long sched_setaffinity(pid_t pid, cpumask_t new_mask)
+long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
 {
        cpumask_t cpus_allowed;
+       cpumask_t new_mask = *in_mask;
        struct task_struct *p;
        int retval;
 
@@ -4754,13 +5068,13 @@ long sched_setaffinity(pid_t pid, cpumask_t new_mask)
        if (retval)
                goto out_unlock;
 
-       cpus_allowed = cpuset_cpus_allowed(p);
+       cpuset_cpus_allowed(p, &cpus_allowed);
        cpus_and(new_mask, new_mask, cpus_allowed);
  again:
-       retval = set_cpus_allowed(p, new_mask);
+       retval = set_cpus_allowed_ptr(p, &new_mask);
 
        if (!retval) {
-               cpus_allowed = cpuset_cpus_allowed(p);
+               cpuset_cpus_allowed(p, &cpus_allowed);
                if (!cpus_subset(new_mask, cpus_allowed)) {
                        /*
                         * We must have raced with a concurrent cpuset
@@ -4804,7 +5118,7 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
        if (retval)
                return retval;
 
-       return sched_setaffinity(pid, new_mask);
+       return sched_setaffinity(pid, &new_mask);
 }
 
 /*
@@ -4920,7 +5234,6 @@ static void __cond_resched(void)
        } while (need_resched());
 }
 
-#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PREEMPT_VOLUNTARY)
 int __sched _cond_resched(void)
 {
        if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
@@ -4931,7 +5244,6 @@ int __sched _cond_resched(void)
        return 0;
 }
 EXPORT_SYMBOL(_cond_resched);
-#endif
 
 /*
  * cond_resched_lock() - if a reschedule is pending, drop the given lock,
@@ -5123,7 +5435,7 @@ out_unlock:
        return retval;
 }
 
-static const char stat_nam[] = "RSDTtZX";
+static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
 
 void sched_show_task(struct task_struct *p)
 {
@@ -5226,8 +5538,11 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
        spin_unlock_irqrestore(&rq->lock, flags);
 
        /* Set the preempt count _outside_ the spinlocks! */
+#if defined(CONFIG_PREEMPT)
+       task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
+#else
        task_thread_info(idle)->preempt_count = 0;
-
+#endif
        /*
         * The idle tasks have their own, simple scheduling class:
         */
@@ -5266,7 +5581,6 @@ static inline void sched_init_granularity(void)
                sysctl_sched_latency = limit;
 
        sysctl_sched_wakeup_granularity *= factor;
-       sysctl_sched_batch_wakeup_granularity *= factor;
 }
 
 #ifdef CONFIG_SMP
@@ -5295,7 +5609,7 @@ static inline void sched_init_granularity(void)
  * task must not exit() & deallocate itself prematurely. The
  * call is not atomic; no spinlocks may be held.
  */
-int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask)
+int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
 {
        struct migration_req req;
        unsigned long flags;
@@ -5303,23 +5617,23 @@ int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask)
        int ret = 0;
 
        rq = task_rq_lock(p, &flags);
-       if (!cpus_intersects(new_mask, cpu_online_map)) {
+       if (!cpus_intersects(*new_mask, cpu_online_map)) {
                ret = -EINVAL;
                goto out;
        }
 
        if (p->sched_class->set_cpus_allowed)
-               p->sched_class->set_cpus_allowed(p, &new_mask);
+               p->sched_class->set_cpus_allowed(p, new_mask);
        else {
-               p->cpus_allowed = new_mask;
-               p->rt.nr_cpus_allowed = cpus_weight(new_mask);
+               p->cpus_allowed = *new_mask;
+               p->rt.nr_cpus_allowed = cpus_weight(*new_mask);
        }
 
        /* Can the task run on the task's current CPU? If so, we're done */
-       if (cpu_isset(task_cpu(p), new_mask))
+       if (cpu_isset(task_cpu(p), *new_mask))
                goto out;
 
-       if (migrate_task(p, any_online_cpu(new_mask), &req)) {
+       if (migrate_task(p, any_online_cpu(*new_mask), &req)) {
                /* Need help from migration thread: drop lock and wait. */
                task_rq_unlock(rq, &flags);
                wake_up_process(rq->migration_thread);
@@ -5332,7 +5646,7 @@ out:
 
        return ret;
 }
-EXPORT_SYMBOL_GPL(set_cpus_allowed);
+EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
 
 /*
  * Move (not current) task off this cpu, onto dest cpu. We're doing
@@ -5470,12 +5784,14 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
                dest_cpu = any_online_cpu(mask);
 
                /* On any allowed CPU? */
-               if (dest_cpu == NR_CPUS)
+               if (dest_cpu >= nr_cpu_ids)
                        dest_cpu = any_online_cpu(p->cpus_allowed);
 
                /* No more Mr. Nice Guy. */
-               if (dest_cpu == NR_CPUS) {
-                       cpumask_t cpus_allowed = cpuset_cpus_allowed_locked(p);
+               if (dest_cpu >= nr_cpu_ids) {
+                       cpumask_t cpus_allowed;
+
+                       cpuset_cpus_allowed_locked(p, &cpus_allowed);
                        /*
                         * Try to stay on the same cpuset, where the
                         * current cpuset may be a subset of all cpus.
@@ -5511,7 +5827,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
  */
 static void migrate_nr_uninterruptible(struct rq *rq_src)
 {
-       struct rq *rq_dest = cpu_rq(any_online_cpu(CPU_MASK_ALL));
+       struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR));
        unsigned long flags;
 
        local_irq_save(flags);
@@ -5923,20 +6239,16 @@ void __init migration_init(void)
 
 #ifdef CONFIG_SMP
 
-/* Number of possible processor ids */
-int nr_cpu_ids __read_mostly = NR_CPUS;
-EXPORT_SYMBOL(nr_cpu_ids);
-
 #ifdef CONFIG_SCHED_DEBUG
 
-static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level)
+static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
+                                 cpumask_t *groupmask)
 {
        struct sched_group *group = sd->groups;
-       cpumask_t groupmask;
-       char str[NR_CPUS];
+       char str[256];
 
-       cpumask_scnprintf(str, NR_CPUS, sd->span);
-       cpus_clear(groupmask);
+       cpulist_scnprintf(str, sizeof(str), sd->span);
+       cpus_clear(*groupmask);
 
        printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
 
@@ -5980,25 +6292,25 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level)
                        break;
                }
 
-               if (cpus_intersects(groupmask, group->cpumask)) {
+               if (cpus_intersects(*groupmask, group->cpumask)) {
                        printk(KERN_CONT "\n");
                        printk(KERN_ERR "ERROR: repeated CPUs\n");
                        break;
                }
 
-               cpus_or(groupmask, groupmask, group->cpumask);
+               cpus_or(*groupmask, *groupmask, group->cpumask);
 
-               cpumask_scnprintf(str, NR_CPUS, group->cpumask);
+               cpulist_scnprintf(str, sizeof(str), group->cpumask);
                printk(KERN_CONT " %s", str);
 
                group = group->next;
        } while (group != sd->groups);
        printk(KERN_CONT "\n");
 
-       if (!cpus_equal(sd->span, groupmask))
+       if (!cpus_equal(sd->span, *groupmask))
                printk(KERN_ERR "ERROR: groups don't span domain->span\n");
 
-       if (sd->parent && !cpus_subset(groupmask, sd->parent->span))
+       if (sd->parent && !cpus_subset(*groupmask, sd->parent->span))
                printk(KERN_ERR "ERROR: parent span is not a superset "
                        "of domain->span\n");
        return 0;
@@ -6006,6 +6318,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level)
 
 static void sched_domain_debug(struct sched_domain *sd, int cpu)
 {
+       cpumask_t *groupmask;
        int level = 0;
 
        if (!sd) {
@@ -6015,14 +6328,21 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
 
        printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
 
+       groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
+       if (!groupmask) {
+               printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
+               return;
+       }
+
        for (;;) {
-               if (sched_domain_debug_one(sd, cpu, level))
+               if (sched_domain_debug_one(sd, cpu, level, groupmask))
                        break;
                level++;
                sd = sd->parent;
                if (!sd)
                        break;
        }
+       kfree(groupmask);
 }
 #else
 # define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6210,30 +6530,33 @@ __setup("isolcpus=", isolated_cpu_setup);
  * and ->cpu_power to 0.
  */
 static void
-init_sched_build_groups(cpumask_t span, const cpumask_t *cpu_map,
+init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map,
                        int (*group_fn)(int cpu, const cpumask_t *cpu_map,
-                                       struct sched_group **sg))
+                                       struct sched_group **sg,
+                                       cpumask_t *tmpmask),
+                       cpumask_t *covered, cpumask_t *tmpmask)
 {
        struct sched_group *first = NULL, *last = NULL;
-       cpumask_t covered = CPU_MASK_NONE;
        int i;
 
-       for_each_cpu_mask(i, span) {
+       cpus_clear(*covered);
+
+       for_each_cpu_mask(i, *span) {
                struct sched_group *sg;
-               int group = group_fn(i, cpu_map, &sg);
+               int group = group_fn(i, cpu_map, &sg, tmpmask);
                int j;
 
-               if (cpu_isset(i, covered))
+               if (cpu_isset(i, *covered))
                        continue;
 
-               sg->cpumask = CPU_MASK_NONE;
+               cpus_clear(sg->cpumask);
                sg->__cpu_power = 0;
 
-               for_each_cpu_mask(j, span) {
-                       if (group_fn(j, cpu_map, NULL) != group)
+               for_each_cpu_mask(j, *span) {
+                       if (group_fn(j, cpu_map, NULL, tmpmask) != group)
                                continue;
 
-                       cpu_set(j, covered);
+                       cpu_set(j, *covered);
                        cpu_set(j, sg->cpumask);
                }
                if (!first)
@@ -6259,7 +6582,7 @@ init_sched_build_groups(cpumask_t span, const cpumask_t *cpu_map,
  *
  * Should use nodemask_t.
  */
-static int find_next_best_node(int node, unsigned long *used_nodes)
+static int find_next_best_node(int node, nodemask_t *used_nodes)
 {
        int i, n, val, min_val, best_node = 0;
 
@@ -6273,7 +6596,7 @@ static int find_next_best_node(int node, unsigned long *used_nodes)
                        continue;
 
                /* Skip already used nodes */
-               if (test_bit(n, used_nodes))
+               if (node_isset(n, *used_nodes))
                        continue;
 
                /* Simple min distance search */
@@ -6285,40 +6608,37 @@ static int find_next_best_node(int node, unsigned long *used_nodes)
                }
        }
 
-       set_bit(best_node, used_nodes);
+       node_set(best_node, *used_nodes);
        return best_node;
 }
 
 /**
  * sched_domain_node_span - get a cpumask for a node's sched_domain
  * @node: node whose cpumask we're constructing
- * @size: number of nodes to include in this span
+ * @span: resulting cpumask
  *
  * Given a node, construct a good cpumask for its sched_domain to span. It
  * should be one that prevents unnecessary balancing, but also spreads tasks
  * out optimally.
  */
-static cpumask_t sched_domain_node_span(int node)
+static void sched_domain_node_span(int node, cpumask_t *span)
 {
-       DECLARE_BITMAP(used_nodes, MAX_NUMNODES);
-       cpumask_t span, nodemask;
+       nodemask_t used_nodes;
+       node_to_cpumask_ptr(nodemask, node);
        int i;
 
-       cpus_clear(span);
-       bitmap_zero(used_nodes, MAX_NUMNODES);
+       cpus_clear(*span);
+       nodes_clear(used_nodes);
 
-       nodemask = node_to_cpumask(node);
-       cpus_or(span, span, nodemask);
-       set_bit(node, used_nodes);
+       cpus_or(*span, *span, *nodemask);
+       node_set(node, used_nodes);
 
        for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
-               int next_node = find_next_best_node(node, used_nodes);
+               int next_node = find_next_best_node(node, &used_nodes);
 
-               nodemask = node_to_cpumask(next_node);
-               cpus_or(span, span, nodemask);
+               node_to_cpumask_ptr_next(nodemask, next_node);
+               cpus_or(*span, *span, *nodemask);
        }
-
-       return span;
 }
 #endif
 
@@ -6332,7 +6652,8 @@ static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
 static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);
 
 static int
-cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg)
+cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
+                cpumask_t *unused)
 {
        if (sg)
                *sg = &per_cpu(sched_group_cpus, cpu);
@@ -6350,19 +6671,22 @@ static DEFINE_PER_CPU(struct sched_group, sched_group_core);
 
 #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
 static int
-cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg)
+cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
+                 cpumask_t *mask)
 {
        int group;
-       cpumask_t mask = per_cpu(cpu_sibling_map, cpu);
-       cpus_and(mask, mask, *cpu_map);
-       group = first_cpu(mask);
+
+       *mask = per_cpu(cpu_sibling_map, cpu);
+       cpus_and(*mask, *mask, *cpu_map);
+       group = first_cpu(*mask);
        if (sg)
                *sg = &per_cpu(sched_group_core, group);
        return group;
 }
 #elif defined(CONFIG_SCHED_MC)
 static int
-cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg)
+cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
+                 cpumask_t *unused)
 {
        if (sg)
                *sg = &per_cpu(sched_group_core, cpu);
@@ -6374,17 +6698,18 @@ static DEFINE_PER_CPU(struct sched_domain, phys_domains);
 static DEFINE_PER_CPU(struct sched_group, sched_group_phys);
 
 static int
-cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg)
+cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
+                 cpumask_t *mask)
 {
        int group;
 #ifdef CONFIG_SCHED_MC
-       cpumask_t mask = cpu_coregroup_map(cpu);
-       cpus_and(mask, mask, *cpu_map);
-       group = first_cpu(mask);
+       *mask = cpu_coregroup_map(cpu);
+       cpus_and(*mask, *mask, *cpu_map);
+       group = first_cpu(*mask);
 #elif defined(CONFIG_SCHED_SMT)
-       cpumask_t mask = per_cpu(cpu_sibling_map, cpu);
-       cpus_and(mask, mask, *cpu_map);
-       group = first_cpu(mask);
+       *mask = per_cpu(cpu_sibling_map, cpu);
+       cpus_and(*mask, *mask, *cpu_map);
+       group = first_cpu(*mask);
 #else
        group = cpu;
 #endif
@@ -6400,19 +6725,19 @@ cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg)
  * gets dynamically allocated.
  */
 static DEFINE_PER_CPU(struct sched_domain, node_domains);
-static struct sched_group **sched_group_nodes_bycpu[NR_CPUS];
+static struct sched_group ***sched_group_nodes_bycpu;
 
 static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
 static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);
 
 static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,
-                                struct sched_group **sg)
+                                struct sched_group **sg, cpumask_t *nodemask)
 {
-       cpumask_t nodemask = node_to_cpumask(cpu_to_node(cpu));
        int group;
 
-       cpus_and(nodemask, nodemask, *cpu_map);
-       group = first_cpu(nodemask);
+       *nodemask = node_to_cpumask(cpu_to_node(cpu));
+       cpus_and(*nodemask, *nodemask, *cpu_map);
+       group = first_cpu(*nodemask);
 
        if (sg)
                *sg = &per_cpu(sched_group_allnodes, group);
@@ -6448,7 +6773,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
 
 #ifdef CONFIG_NUMA
 /* Free memory allocated for various sched_group structures */
-static void free_sched_groups(const cpumask_t *cpu_map)
+static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
 {
        int cpu, i;
 
@@ -6460,11 +6785,11 @@ static void free_sched_groups(const cpumask_t *cpu_map)
                        continue;
 
                for (i = 0; i < MAX_NUMNODES; i++) {
-                       cpumask_t nodemask = node_to_cpumask(i);
                        struct sched_group *oldsg, *sg = sched_group_nodes[i];
 
-                       cpus_and(nodemask, nodemask, *cpu_map);
-                       if (cpus_empty(nodemask))
+                       *nodemask = node_to_cpumask(i);
+                       cpus_and(*nodemask, *nodemask, *cpu_map);
+                       if (cpus_empty(*nodemask))
                                continue;
 
                        if (sg == NULL)
@@ -6482,7 +6807,7 @@ next_sg:
        }
 }
 #else
-static void free_sched_groups(const cpumask_t *cpu_map)
+static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
 {
 }
 #endif
@@ -6540,13 +6865,111 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
 }
 
 /*
+ * Initializers for schedule domains
+ * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
+ */
+
+#define        SD_INIT(sd, type)       sd_init_##type(sd)
+#define SD_INIT_FUNC(type)     \
+static noinline void sd_init_##type(struct sched_domain *sd)   \
+{                                                              \
+       memset(sd, 0, sizeof(*sd));                             \
+       *sd = SD_##type##_INIT;                                 \
+       sd->level = SD_LV_##type;                               \
+}
+
+SD_INIT_FUNC(CPU)
+#ifdef CONFIG_NUMA
+ SD_INIT_FUNC(ALLNODES)
+ SD_INIT_FUNC(NODE)
+#endif
+#ifdef CONFIG_SCHED_SMT
+ SD_INIT_FUNC(SIBLING)
+#endif
+#ifdef CONFIG_SCHED_MC
+ SD_INIT_FUNC(MC)
+#endif
+
+/*
+ * To minimize stack usage kmalloc room for cpumasks and share the
+ * space as the usage in build_sched_domains() dictates.  Used only
+ * if the amount of space is significant.
+ */
+struct allmasks {
+       cpumask_t tmpmask;                      /* make this one first */
+       union {
+               cpumask_t nodemask;
+               cpumask_t this_sibling_map;
+               cpumask_t this_core_map;
+       };
+       cpumask_t send_covered;
+
+#ifdef CONFIG_NUMA
+       cpumask_t domainspan;
+       cpumask_t covered;
+       cpumask_t notcovered;
+#endif
+};
+
+#if    NR_CPUS > 128
+#define        SCHED_CPUMASK_ALLOC             1
+#define        SCHED_CPUMASK_FREE(v)           kfree(v)
+#define        SCHED_CPUMASK_DECLARE(v)        struct allmasks *v
+#else
+#define        SCHED_CPUMASK_ALLOC             0
+#define        SCHED_CPUMASK_FREE(v)
+#define        SCHED_CPUMASK_DECLARE(v)        struct allmasks _v, *v = &_v
+#endif
+
+#define        SCHED_CPUMASK_VAR(v, a)         cpumask_t *v = (cpumask_t *) \
+                       ((unsigned long)(a) + offsetof(struct allmasks, v))
+
+static int default_relax_domain_level = -1;
+
+static int __init setup_relax_domain_level(char *str)
+{
+       unsigned long val;
+
+       val = simple_strtoul(str, NULL, 0);
+       if (val < SD_LV_MAX)
+               default_relax_domain_level = val;
+
+       return 1;
+}
+__setup("relax_domain_level=", setup_relax_domain_level);
+
+static void set_domain_attribute(struct sched_domain *sd,
+                                struct sched_domain_attr *attr)
+{
+       int request;
+
+       if (!attr || attr->relax_domain_level < 0) {
+               if (default_relax_domain_level < 0)
+                       return;
+               else
+                       request = default_relax_domain_level;
+       } else
+               request = attr->relax_domain_level;
+       if (request < sd->level) {
+               /* turn off idle balance on this domain */
+               sd->flags &= ~(SD_WAKE_IDLE|SD_BALANCE_NEWIDLE);
+       } else {
+               /* turn on idle balance on this domain */
+               sd->flags |= (SD_WAKE_IDLE_FAR|SD_BALANCE_NEWIDLE);
+       }
+}
+
+/*
  * Build sched domains for a given set of cpus and attach the sched domains
  * to the individual cpus
  */
-static int build_sched_domains(const cpumask_t *cpu_map)
+static int __build_sched_domains(const cpumask_t *cpu_map,
+                                struct sched_domain_attr *attr)
 {
        int i;
        struct root_domain *rd;
+       SCHED_CPUMASK_DECLARE(allmasks);
+       cpumask_t *tmpmask;
 #ifdef CONFIG_NUMA
        struct sched_group **sched_group_nodes = NULL;
        int sd_allnodes = 0;
@@ -6560,39 +6983,63 @@ static int build_sched_domains(const cpumask_t *cpu_map)
                printk(KERN_WARNING "Can not alloc sched group node list\n");
                return -ENOMEM;
        }
-       sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
 #endif
 
        rd = alloc_rootdomain();
        if (!rd) {
                printk(KERN_WARNING "Cannot alloc root domain\n");
+#ifdef CONFIG_NUMA
+               kfree(sched_group_nodes);
+#endif
                return -ENOMEM;
        }
 
+#if SCHED_CPUMASK_ALLOC
+       /* get space for all scratch cpumask variables */
+       allmasks = kmalloc(sizeof(*allmasks), GFP_KERNEL);
+       if (!allmasks) {
+               printk(KERN_WARNING "Cannot alloc cpumask array\n");
+               kfree(rd);
+#ifdef CONFIG_NUMA
+               kfree(sched_group_nodes);
+#endif
+               return -ENOMEM;
+       }
+#endif
+       tmpmask = (cpumask_t *)allmasks;
+
+
+#ifdef CONFIG_NUMA
+       sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
+#endif
+
        /*
         * Set up domains for cpus specified by the cpu_map.
         */
        for_each_cpu_mask(i, *cpu_map) {
                struct sched_domain *sd = NULL, *p;
-               cpumask_t nodemask = node_to_cpumask(cpu_to_node(i));
+               SCHED_CPUMASK_VAR(nodemask, allmasks);
 
-               cpus_and(nodemask, nodemask, *cpu_map);
+               *nodemask = node_to_cpumask(cpu_to_node(i));
+               cpus_and(*nodemask, *nodemask, *cpu_map);
 
 #ifdef CONFIG_NUMA
                if (cpus_weight(*cpu_map) >
-                               SD_NODES_PER_DOMAIN*cpus_weight(nodemask)) {
+                               SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) {
                        sd = &per_cpu(allnodes_domains, i);
-                       *sd = SD_ALLNODES_INIT;
+                       SD_INIT(sd, ALLNODES);
+                       set_domain_attribute(sd, attr);
                        sd->span = *cpu_map;
-                       cpu_to_allnodes_group(i, cpu_map, &sd->groups);
+                       cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
                        p = sd;
                        sd_allnodes = 1;
                } else
                        p = NULL;
 
                sd = &per_cpu(node_domains, i);
-               *sd = SD_NODE_INIT;
-               sd->span = sched_domain_node_span(cpu_to_node(i));
+               SD_INIT(sd, NODE);
+               set_domain_attribute(sd, attr);
+               sched_domain_node_span(cpu_to_node(i), &sd->span);
                sd->parent = p;
                if (p)
                        p->child = sd;
@@ -6601,94 +7048,117 @@ static int build_sched_domains(const cpumask_t *cpu_map)
 
                p = sd;
                sd = &per_cpu(phys_domains, i);
-               *sd = SD_CPU_INIT;
-               sd->span = nodemask;
+               SD_INIT(sd, CPU);
+               set_domain_attribute(sd, attr);
+               sd->span = *nodemask;
                sd->parent = p;
                if (p)
                        p->child = sd;
-               cpu_to_phys_group(i, cpu_map, &sd->groups);
+               cpu_to_phys_group(i, cpu_map, &sd->groups, tmpmask);
 
 #ifdef CONFIG_SCHED_MC
                p = sd;
                sd = &per_cpu(core_domains, i);
-               *sd = SD_MC_INIT;
+               SD_INIT(sd, MC);
+               set_domain_attribute(sd, attr);
                sd->span = cpu_coregroup_map(i);
                cpus_and(sd->span, sd->span, *cpu_map);
                sd->parent = p;
                p->child = sd;
-               cpu_to_core_group(i, cpu_map, &sd->groups);
+               cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
 #endif
 
 #ifdef CONFIG_SCHED_SMT
                p = sd;
                sd = &per_cpu(cpu_domains, i);
-               *sd = SD_SIBLING_INIT;
+               SD_INIT(sd, SIBLING);
+               set_domain_attribute(sd, attr);
                sd->span = per_cpu(cpu_sibling_map, i);
                cpus_and(sd->span, sd->span, *cpu_map);
                sd->parent = p;
                p->child = sd;
-               cpu_to_cpu_group(i, cpu_map, &sd->groups);
+               cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
 #endif
        }
 
 #ifdef CONFIG_SCHED_SMT
        /* Set up CPU (sibling) groups */
        for_each_cpu_mask(i, *cpu_map) {
-               cpumask_t this_sibling_map = per_cpu(cpu_sibling_map, i);
-               cpus_and(this_sibling_map, this_sibling_map, *cpu_map);
-               if (i != first_cpu(this_sibling_map))
+               SCHED_CPUMASK_VAR(this_sibling_map, allmasks);
+               SCHED_CPUMASK_VAR(send_covered, allmasks);
+
+               *this_sibling_map = per_cpu(cpu_sibling_map, i);
+               cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
+               if (i != first_cpu(*this_sibling_map))
                        continue;
 
                init_sched_build_groups(this_sibling_map, cpu_map,
-                                       &cpu_to_cpu_group);
+                                       &cpu_to_cpu_group,
+                                       send_covered, tmpmask);
        }
 #endif
 
 #ifdef CONFIG_SCHED_MC
        /* Set up multi-core groups */
        for_each_cpu_mask(i, *cpu_map) {
-               cpumask_t this_core_map = cpu_coregroup_map(i);
-               cpus_and(this_core_map, this_core_map, *cpu_map);
-               if (i != first_cpu(this_core_map))
+               SCHED_CPUMASK_VAR(this_core_map, allmasks);
+               SCHED_CPUMASK_VAR(send_covered, allmasks);
+
+               *this_core_map = cpu_coregroup_map(i);
+               cpus_and(*this_core_map, *this_core_map, *cpu_map);
+               if (i != first_cpu(*this_core_map))
                        continue;
+
                init_sched_build_groups(this_core_map, cpu_map,
-                                       &cpu_to_core_group);
+                                       &cpu_to_core_group,
+                                       send_covered, tmpmask);
        }
 #endif
 
        /* Set up physical groups */
        for (i = 0; i < MAX_NUMNODES; i++) {
-               cpumask_t nodemask = node_to_cpumask(i);
+               SCHED_CPUMASK_VAR(nodemask, allmasks);
+               SCHED_CPUMASK_VAR(send_covered, allmasks);
 
-               cpus_and(nodemask, nodemask, *cpu_map);
-               if (cpus_empty(nodemask))
+               *nodemask = node_to_cpumask(i);
+               cpus_and(*nodemask, *nodemask, *cpu_map);
+               if (cpus_empty(*nodemask))
                        continue;
 
-               init_sched_build_groups(nodemask, cpu_map, &cpu_to_phys_group);
+               init_sched_build_groups(nodemask, cpu_map,
+                                       &cpu_to_phys_group,
+                                       send_covered, tmpmask);
        }
 
 #ifdef CONFIG_NUMA
        /* Set up node groups */
-       if (sd_allnodes)
-               init_sched_build_groups(*cpu_map, cpu_map,
-                                       &cpu_to_allnodes_group);
+       if (sd_allnodes) {
+               SCHED_CPUMASK_VAR(send_covered, allmasks);
+
+               init_sched_build_groups(cpu_map, cpu_map,
+                                       &cpu_to_allnodes_group,
+                                       send_covered, tmpmask);
+       }
 
        for (i = 0; i < MAX_NUMNODES; i++) {
                /* Set up node groups */
                struct sched_group *sg, *prev;
-               cpumask_t nodemask = node_to_cpumask(i);
-               cpumask_t domainspan;
-               cpumask_t covered = CPU_MASK_NONE;
+               SCHED_CPUMASK_VAR(nodemask, allmasks);
+               SCHED_CPUMASK_VAR(domainspan, allmasks);
+               SCHED_CPUMASK_VAR(covered, allmasks);
                int j;
 
-               cpus_and(nodemask, nodemask, *cpu_map);
-               if (cpus_empty(nodemask)) {
+               *nodemask = node_to_cpumask(i);
+               cpus_clear(*covered);
+
+               cpus_and(*nodemask, *nodemask, *cpu_map);
+               if (cpus_empty(*nodemask)) {
                        sched_group_nodes[i] = NULL;
                        continue;
                }
 
-               domainspan = sched_domain_node_span(i);
-               cpus_and(domainspan, domainspan, *cpu_map);
+               sched_domain_node_span(i, domainspan);
+               cpus_and(*domainspan, *domainspan, *cpu_map);
 
                sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i);
                if (!sg) {
@@ -6697,31 +7167,31 @@ static int build_sched_domains(const cpumask_t *cpu_map)
                        goto error;
                }
                sched_group_nodes[i] = sg;
-               for_each_cpu_mask(j, nodemask) {
+               for_each_cpu_mask(j, *nodemask) {
                        struct sched_domain *sd;
 
                        sd = &per_cpu(node_domains, j);
                        sd->groups = sg;
                }
                sg->__cpu_power = 0;
-               sg->cpumask = nodemask;
+               sg->cpumask = *nodemask;
                sg->next = sg;
-               cpus_or(covered, covered, nodemask);
+               cpus_or(*covered, *covered, *nodemask);
                prev = sg;
 
                for (j = 0; j < MAX_NUMNODES; j++) {
-                       cpumask_t tmp, notcovered;
+                       SCHED_CPUMASK_VAR(notcovered, allmasks);
                        int n = (i + j) % MAX_NUMNODES;
+                       node_to_cpumask_ptr(pnodemask, n);
 
-                       cpus_complement(notcovered, covered);
-                       cpus_and(tmp, notcovered, *cpu_map);
-                       cpus_and(tmp, tmp, domainspan);
-                       if (cpus_empty(tmp))
+                       cpus_complement(*notcovered, *covered);
+                       cpus_and(*tmpmask, *notcovered, *cpu_map);
+                       cpus_and(*tmpmask, *tmpmask, *domainspan);
+                       if (cpus_empty(*tmpmask))
                                break;
 
-                       nodemask = node_to_cpumask(n);
-                       cpus_and(tmp, tmp, nodemask);
-                       if (cpus_empty(tmp))
+                       cpus_and(*tmpmask, *tmpmask, *pnodemask);
+                       if (cpus_empty(*tmpmask))
                                continue;
 
                        sg = kmalloc_node(sizeof(struct sched_group),
@@ -6732,9 +7202,9 @@ static int build_sched_domains(const cpumask_t *cpu_map)
                                goto error;
                        }
                        sg->__cpu_power = 0;
-                       sg->cpumask = tmp;
+                       sg->cpumask = *tmpmask;
                        sg->next = prev->next;
-                       cpus_or(covered, covered, tmp);
+                       cpus_or(*covered, *covered, *tmpmask);
                        prev->next = sg;
                        prev = sg;
                }
@@ -6770,7 +7240,8 @@ static int build_sched_domains(const cpumask_t *cpu_map)
        if (sd_allnodes) {
                struct sched_group *sg;
 
-               cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg);
+               cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg,
+                                                               tmpmask);
                init_numa_sched_groups_power(sg);
        }
 #endif
@@ -6788,17 +7259,26 @@ static int build_sched_domains(const cpumask_t *cpu_map)
                cpu_attach_domain(sd, rd, i);
        }
 
+       SCHED_CPUMASK_FREE((void *)allmasks);
        return 0;
 
 #ifdef CONFIG_NUMA
 error:
-       free_sched_groups(cpu_map);
+       free_sched_groups(cpu_map, tmpmask);
+       SCHED_CPUMASK_FREE((void *)allmasks);
        return -ENOMEM;
 #endif
 }
 
+static int build_sched_domains(const cpumask_t *cpu_map)
+{
+       return __build_sched_domains(cpu_map, NULL);
+}
+
 static cpumask_t *doms_cur;    /* current sched domains */
 static int ndoms_cur;          /* number of sched domains in 'doms_cur' */
+static struct sched_domain_attr *dattr_cur;
+                               /* attribues of custom domains in 'doms_cur' */
 
 /*
  * Special case: If a kmalloc of a doms_cur partition (array of
@@ -6807,6 +7287,22 @@ static int ndoms_cur;            /* number of sched domains in 'doms_cur' */
  */
 static cpumask_t fallback_doms;
 
+void __attribute__((weak)) arch_update_cpu_topology(void)
+{
+}
+
+/*
+ * Free current domain masks.
+ * Called after all cpus are attached to NULL domain.
+ */
+static void free_sched_domains(void)
+{
+       ndoms_cur = 0;
+       if (doms_cur != &fallback_doms)
+               kfree(doms_cur);
+       doms_cur = &fallback_doms;
+}
+
 /*
  * Set up scheduler domains and groups. Callers must hold the hotplug lock.
  * For now this just excludes isolated cpus, but could be used to
@@ -6816,20 +7312,23 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
 {
        int err;
 
+       arch_update_cpu_topology();
        ndoms_cur = 1;
        doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
        if (!doms_cur)
                doms_cur = &fallback_doms;
        cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map);
+       dattr_cur = NULL;
        err = build_sched_domains(doms_cur);
        register_sched_domain_sysctl();
 
        return err;
 }
 
-static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
+static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
+                                      cpumask_t *tmpmask)
 {
-       free_sched_groups(cpu_map);
+       free_sched_groups(cpu_map, tmpmask);
 }
 
 /*
@@ -6838,6 +7337,7 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
  */
 static void detach_destroy_domains(const cpumask_t *cpu_map)
 {
+       cpumask_t tmpmask;
        int i;
 
        unregister_sched_domain_sysctl();
@@ -6845,7 +7345,23 @@ static void detach_destroy_domains(const cpumask_t *cpu_map)
        for_each_cpu_mask(i, *cpu_map)
                cpu_attach_domain(NULL, &def_root_domain, i);
        synchronize_sched();
-       arch_destroy_sched_domains(cpu_map);
+       arch_destroy_sched_domains(cpu_map, &tmpmask);
+}
+
+/* handle null as "default" */
+static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
+                       struct sched_domain_attr *new, int idx_new)
+{
+       struct sched_domain_attr tmp;
+
+       /* fast path */
+       if (!new && !cur)
+               return 1;
+
+       tmp = SD_ATTR_INIT;
+       return !memcmp(cur ? (cur + idx_cur) : &tmp,
+                       new ? (new + idx_new) : &tmp,
+                       sizeof(struct sched_domain_attr));
 }
 
 /*
@@ -6869,11 +7385,12 @@ static void detach_destroy_domains(const cpumask_t *cpu_map)
  *
  * Call with hotplug lock held
  */
-void partition_sched_domains(int ndoms_new, cpumask_t *doms_new)
+void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
+                            struct sched_domain_attr *dattr_new)
 {
        int i, j;
 
-       lock_doms_cur();
+       mutex_lock(&sched_domains_mutex);
 
        /* always unregister in case we don't destroy any domains */
        unregister_sched_domain_sysctl();
@@ -6882,12 +7399,14 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new)
                ndoms_new = 1;
                doms_new = &fallback_doms;
                cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
+               dattr_new = NULL;
        }
 
        /* Destroy deleted domains */
        for (i = 0; i < ndoms_cur; i++) {
                for (j = 0; j < ndoms_new; j++) {
-                       if (cpus_equal(doms_cur[i], doms_new[j]))
+                       if (cpus_equal(doms_cur[i], doms_new[j])
+                           && dattrs_equal(dattr_cur, i, dattr_new, j))
                                goto match1;
                }
                /* no match - a current sched domain not in new doms_new[] */
@@ -6899,11 +7418,13 @@ match1:
        /* Build new domains */
        for (i = 0; i < ndoms_new; i++) {
                for (j = 0; j < ndoms_cur; j++) {
-                       if (cpus_equal(doms_new[i], doms_cur[j]))
+                       if (cpus_equal(doms_new[i], doms_cur[j])
+                           && dattrs_equal(dattr_new, i, dattr_cur, j))
                                goto match2;
                }
                /* no match - add a new doms_new */
-               build_sched_domains(doms_new + i);
+               __build_sched_domains(doms_new + i,
+                                       dattr_new ? dattr_new + i : NULL);
 match2:
                ;
        }
@@ -6911,12 +7432,14 @@ match2:
        /* Remember the new sched domains */
        if (doms_cur != &fallback_doms)
                kfree(doms_cur);
+       kfree(dattr_cur);       /* kfree(NULL) is safe */
        doms_cur = doms_new;
+       dattr_cur = dattr_new;
        ndoms_cur = ndoms_new;
 
        register_sched_domain_sysctl();
 
-       unlock_doms_cur();
+       mutex_unlock(&sched_domains_mutex);
 }
 
 #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
@@ -6925,8 +7448,11 @@ int arch_reinit_sched_domains(void)
        int err;
 
        get_online_cpus();
+       mutex_lock(&sched_domains_mutex);
        detach_destroy_domains(&cpu_online_map);
+       free_sched_domains();
        err = arch_init_sched_domains(&cpu_online_map);
+       mutex_unlock(&sched_domains_mutex);
        put_online_cpus();
 
        return err;
@@ -7010,6 +7536,7 @@ static int update_sched_domains(struct notifier_block *nfb,
        case CPU_DOWN_PREPARE:
        case CPU_DOWN_PREPARE_FROZEN:
                detach_destroy_domains(&cpu_online_map);
+               free_sched_domains();
                return NOTIFY_OK;
 
        case CPU_UP_CANCELED:
@@ -7028,8 +7555,16 @@ static int update_sched_domains(struct notifier_block *nfb,
                return NOTIFY_DONE;
        }
 
+#ifndef CONFIG_CPUSETS
+       /*
+        * Create default domain partitioning if cpusets are disabled.
+        * Otherwise we let cpusets rebuild the domains based on the
+        * current setup.
+        */
+
        /* The hotplug lock is already held by cpu_up/cpu_down */
        arch_init_sched_domains(&cpu_online_map);
+#endif
 
        return NOTIFY_OK;
 }
@@ -7038,17 +7573,25 @@ void __init sched_init_smp(void)
 {
        cpumask_t non_isolated_cpus;
 
+#if defined(CONFIG_NUMA)
+       sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
+                                                               GFP_KERNEL);
+       BUG_ON(sched_group_nodes_bycpu == NULL);
+#endif
        get_online_cpus();
+       mutex_lock(&sched_domains_mutex);
        arch_init_sched_domains(&cpu_online_map);
        cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map);
        if (cpus_empty(non_isolated_cpus))
                cpu_set(smp_processor_id(), non_isolated_cpus);
+       mutex_unlock(&sched_domains_mutex);
        put_online_cpus();
        /* XXX: Theoretical race here - CPU may be hotplugged now */
        hotcpu_notifier(update_sched_domains, 0);
+       init_hrtick();
 
        /* Move init over to a non-isolated CPU */
-       if (set_cpus_allowed(current, non_isolated_cpus) < 0)
+       if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0)
                BUG();
        sched_init_granularity();
 }
@@ -7069,6 +7612,7 @@ int in_sched_functions(unsigned long addr)
 static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
 {
        cfs_rq->tasks_timeline = RB_ROOT;
+       INIT_LIST_HEAD(&cfs_rq->tasks);
 #ifdef CONFIG_FAIR_GROUP_SCHED
        cfs_rq->rq = rq;
 #endif
@@ -7098,6 +7642,8 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
 
        rt_rq->rt_time = 0;
        rt_rq->rt_throttled = 0;
+       rt_rq->rt_runtime = 0;
+       spin_lock_init(&rt_rq->rt_runtime_lock);
 
 #ifdef CONFIG_RT_GROUP_SCHED
        rt_rq->rt_nr_boosted = 0;
@@ -7106,10 +7652,11 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
 }
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static void init_tg_cfs_entry(struct rq *rq, struct task_group *tg,
-               struct cfs_rq *cfs_rq, struct sched_entity *se,
-               int cpu, int add)
+static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
+                               struct sched_entity *se, int cpu, int add,
+                               struct sched_entity *parent)
 {
+       struct rq *rq = cpu_rq(cpu);
        tg->cfs_rq[cpu] = cfs_rq;
        init_cfs_rq(cfs_rq, rq);
        cfs_rq->tg = tg;
@@ -7117,45 +7664,130 @@ static void init_tg_cfs_entry(struct rq *rq, struct task_group *tg,
                list_add(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
 
        tg->se[cpu] = se;
-       se->cfs_rq = &rq->cfs;
+       /* se could be NULL for init_task_group */
+       if (!se)
+               return;
+
+       if (!parent)
+               se->cfs_rq = &rq->cfs;
+       else
+               se->cfs_rq = parent->my_q;
+
        se->my_q = cfs_rq;
        se->load.weight = tg->shares;
-       se->load.inv_weight = div64_64(1ULL<<32, se->load.weight);
-       se->parent = NULL;
+       se->load.inv_weight = 0;
+       se->parent = parent;
 }
 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED
-static void init_tg_rt_entry(struct rq *rq, struct task_group *tg,
-               struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
-               int cpu, int add)
+static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
+               struct sched_rt_entity *rt_se, int cpu, int add,
+               struct sched_rt_entity *parent)
 {
+       struct rq *rq = cpu_rq(cpu);
+
        tg->rt_rq[cpu] = rt_rq;
        init_rt_rq(rt_rq, rq);
        rt_rq->tg = tg;
        rt_rq->rt_se = rt_se;
+       rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
        if (add)
                list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
 
        tg->rt_se[cpu] = rt_se;
-       rt_se->rt_rq = &rq->rt;
+       if (!rt_se)
+               return;
+
+       if (!parent)
+               rt_se->rt_rq = &rq->rt;
+       else
+               rt_se->rt_rq = parent->my_q;
+
        rt_se->my_q = rt_rq;
-       rt_se->parent = NULL;
+       rt_se->parent = parent;
        INIT_LIST_HEAD(&rt_se->run_list);
 }
 #endif
 
 void __init sched_init(void)
 {
-       int highest_cpu = 0;
        int i, j;
+       unsigned long alloc_size = 0, ptr;
+
+#ifdef CONFIG_FAIR_GROUP_SCHED
+       alloc_size += 2 * nr_cpu_ids * sizeof(void **);
+#endif
+#ifdef CONFIG_RT_GROUP_SCHED
+       alloc_size += 2 * nr_cpu_ids * sizeof(void **);
+#endif
+#ifdef CONFIG_USER_SCHED
+       alloc_size *= 2;
+#endif
+       /*
+        * As sched_init() is called before page_alloc is setup,
+        * we use alloc_bootmem().
+        */
+       if (alloc_size) {
+               ptr = (unsigned long)alloc_bootmem(alloc_size);
+
+#ifdef CONFIG_FAIR_GROUP_SCHED
+               init_task_group.se = (struct sched_entity **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+               init_task_group.cfs_rq = (struct cfs_rq **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+#ifdef CONFIG_USER_SCHED
+               root_task_group.se = (struct sched_entity **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+               root_task_group.cfs_rq = (struct cfs_rq **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+#endif
+#endif
+#ifdef CONFIG_RT_GROUP_SCHED
+               init_task_group.rt_se = (struct sched_rt_entity **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+               init_task_group.rt_rq = (struct rt_rq **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+#ifdef CONFIG_USER_SCHED
+               root_task_group.rt_se = (struct sched_rt_entity **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+               root_task_group.rt_rq = (struct rt_rq **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+#endif
+#endif
+       }
 
 #ifdef CONFIG_SMP
        init_defrootdomain();
 #endif
 
+       init_rt_bandwidth(&def_rt_bandwidth,
+                       global_rt_period(), global_rt_runtime());
+
+#ifdef CONFIG_RT_GROUP_SCHED
+       init_rt_bandwidth(&init_task_group.rt_bandwidth,
+                       global_rt_period(), global_rt_runtime());
+#ifdef CONFIG_USER_SCHED
+       init_rt_bandwidth(&root_task_group.rt_bandwidth,
+                       global_rt_period(), RUNTIME_INF);
+#endif
+#endif
+
 #ifdef CONFIG_GROUP_SCHED
        list_add(&init_task_group.list, &task_groups);
+       INIT_LIST_HEAD(&init_task_group.children);
+
+#ifdef CONFIG_USER_SCHED
+       INIT_LIST_HEAD(&root_task_group.children);
+       init_task_group.parent = &root_task_group;
+       list_add(&init_task_group.siblings, &root_task_group.children);
+#endif
 #endif
 
        for_each_possible_cpu(i) {
@@ -7165,27 +7797,67 @@ void __init sched_init(void)
                spin_lock_init(&rq->lock);
                lockdep_set_class(&rq->lock, &rq->rq_lock_key);
                rq->nr_running = 0;
-               rq->clock = 1;
                init_cfs_rq(&rq->cfs, rq);
                init_rt_rq(&rq->rt, rq);
 #ifdef CONFIG_FAIR_GROUP_SCHED
                init_task_group.shares = init_task_group_load;
                INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
-               init_tg_cfs_entry(rq, &init_task_group,
+#ifdef CONFIG_CGROUP_SCHED
+               /*
+                * How much cpu bandwidth does init_task_group get?
+                *
+                * In case of task-groups formed thr' the cgroup filesystem, it
+                * gets 100% of the cpu resources in the system. This overall
+                * system cpu resource is divided among the tasks of
+                * init_task_group and its child task-groups in a fair manner,
+                * based on each entity's (task or task-group's) weight
+                * (se->load.weight).
+                *
+                * In other words, if init_task_group has 10 tasks of weight
+                * 1024) and two child groups A0 and A1 (of weight 1024 each),
+                * then A0's share of the cpu resource is:
+                *
+                *      A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
+                *
+                * We achieve this by letting init_task_group's tasks sit
+                * directly in rq->cfs (i.e init_task_group->se[] = NULL).
+                */
+               init_tg_cfs_entry(&init_task_group, &rq->cfs, NULL, i, 1, NULL);
+#elif defined CONFIG_USER_SCHED
+               root_task_group.shares = NICE_0_LOAD;
+               init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, 0, NULL);
+               /*
+                * In case of task-groups formed thr' the user id of tasks,
+                * init_task_group represents tasks belonging to root user.
+                * Hence it forms a sibling of all subsequent groups formed.
+                * In this case, init_task_group gets only a fraction of overall
+                * system cpu resource, based on the weight assigned to root
+                * user's cpu share (INIT_TASK_GROUP_LOAD). This is accomplished
+                * by letting tasks of init_task_group sit in a separate cfs_rq
+                * (init_cfs_rq) and having one entity represent this group of
+                * tasks in rq->cfs (i.e init_task_group->se[] != NULL).
+                */
+               init_tg_cfs_entry(&init_task_group,
                                &per_cpu(init_cfs_rq, i),
-                               &per_cpu(init_sched_entity, i), i, 1);
+                               &per_cpu(init_sched_entity, i), i, 1,
+                               root_task_group.se[i]);
 
 #endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
+
+               rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
 #ifdef CONFIG_RT_GROUP_SCHED
-               init_task_group.rt_runtime =
-                       sysctl_sched_rt_runtime * NSEC_PER_USEC;
                INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
-               init_tg_rt_entry(rq, &init_task_group,
+#ifdef CONFIG_CGROUP_SCHED
+               init_tg_rt_entry(&init_task_group, &rq->rt, NULL, i, 1, NULL);
+#elif defined CONFIG_USER_SCHED
+               init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL);
+               init_tg_rt_entry(&init_task_group,
                                &per_cpu(init_rt_rq, i),
-                               &per_cpu(init_sched_rt_entity, i), i, 1);
+                               &per_cpu(init_sched_rt_entity, i), i, 1,
+                               root_task_group.rt_se[i]);
+#endif
 #endif
-               rq->rt_period_expire = 0;
-               rq->rt_throttled = 0;
 
                for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
                        rq->cpu_load[j] = 0;
@@ -7202,7 +7874,6 @@ void __init sched_init(void)
 #endif
                init_rq_hrtick(rq);
                atomic_set(&rq->nr_iowait, 0);
-               highest_cpu = i;
        }
 
        set_load_weight(&init_task);
@@ -7212,7 +7883,6 @@ void __init sched_init(void)
 #endif
 
 #ifdef CONFIG_SMP
-       nr_cpu_ids = highest_cpu + 1;
        open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL);
 #endif
 
@@ -7270,6 +7940,7 @@ EXPORT_SYMBOL(__might_sleep);
 static void normalize_task(struct rq *rq, struct task_struct *p)
 {
        int on_rq;
+
        update_rq_clock(rq);
        on_rq = p->se.on_rq;
        if (on_rq)
@@ -7301,7 +7972,6 @@ void normalize_rt_tasks(void)
                p->se.sleep_start               = 0;
                p->se.block_start               = 0;
 #endif
-               task_rq(p)->clock               = 0;
 
                if (!rt_task(p)) {
                        /*
@@ -7371,8 +8041,6 @@ void set_curr_task(int cpu, struct task_struct *p)
 
 #endif
 
-#ifdef CONFIG_GROUP_SCHED
-
 #ifdef CONFIG_FAIR_GROUP_SCHED
 static void free_fair_sched_group(struct task_group *tg)
 {
@@ -7389,17 +8057,18 @@ static void free_fair_sched_group(struct task_group *tg)
        kfree(tg->se);
 }
 
-static int alloc_fair_sched_group(struct task_group *tg)
+static
+int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
 {
        struct cfs_rq *cfs_rq;
-       struct sched_entity *se;
+       struct sched_entity *se, *parent_se;
        struct rq *rq;
        int i;
 
-       tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL);
+       tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
        if (!tg->cfs_rq)
                goto err;
-       tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL);
+       tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
        if (!tg->se)
                goto err;
 
@@ -7418,7 +8087,8 @@ static int alloc_fair_sched_group(struct task_group *tg)
                if (!se)
                        goto err;
 
-               init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0);
+               parent_se = parent ? parent->se[i] : NULL;
+               init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent_se);
        }
 
        return 1;
@@ -7442,7 +8112,8 @@ static inline void free_fair_sched_group(struct task_group *tg)
 {
 }
 
-static inline int alloc_fair_sched_group(struct task_group *tg)
+static inline
+int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
 {
        return 1;
 }
@@ -7461,6 +8132,8 @@ static void free_rt_sched_group(struct task_group *tg)
 {
        int i;
 
+       destroy_rt_bandwidth(&tg->rt_bandwidth);
+
        for_each_possible_cpu(i) {
                if (tg->rt_rq)
                        kfree(tg->rt_rq[i]);
@@ -7472,21 +8145,23 @@ static void free_rt_sched_group(struct task_group *tg)
        kfree(tg->rt_se);
 }
 
-static int alloc_rt_sched_group(struct task_group *tg)
+static
+int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
 {
        struct rt_rq *rt_rq;
-       struct sched_rt_entity *rt_se;
+       struct sched_rt_entity *rt_se, *parent_se;
        struct rq *rq;
        int i;
 
-       tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL);
+       tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
        if (!tg->rt_rq)
                goto err;
-       tg->rt_se = kzalloc(sizeof(rt_se) * NR_CPUS, GFP_KERNEL);
+       tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
        if (!tg->rt_se)
                goto err;
 
-       tg->rt_runtime = 0;
+       init_rt_bandwidth(&tg->rt_bandwidth,
+                       ktime_to_ns(def_rt_bandwidth.rt_period), 0);
 
        for_each_possible_cpu(i) {
                rq = cpu_rq(i);
@@ -7501,7 +8176,8 @@ static int alloc_rt_sched_group(struct task_group *tg)
                if (!rt_se)
                        goto err;
 
-               init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0);
+               parent_se = parent ? parent->rt_se[i] : NULL;
+               init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent_se);
        }
 
        return 1;
@@ -7525,7 +8201,8 @@ static inline void free_rt_sched_group(struct task_group *tg)
 {
 }
 
-static inline int alloc_rt_sched_group(struct task_group *tg)
+static inline
+int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
 {
        return 1;
 }
@@ -7539,6 +8216,7 @@ static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
 }
 #endif
 
+#ifdef CONFIG_GROUP_SCHED
 static void free_sched_group(struct task_group *tg)
 {
        free_fair_sched_group(tg);
@@ -7547,7 +8225,7 @@ static void free_sched_group(struct task_group *tg)
 }
 
 /* allocate runqueue etc for a new task group */
-struct task_group *sched_create_group(void)
+struct task_group *sched_create_group(struct task_group *parent)
 {
        struct task_group *tg;
        unsigned long flags;
@@ -7557,10 +8235,10 @@ struct task_group *sched_create_group(void)
        if (!tg)
                return ERR_PTR(-ENOMEM);
 
-       if (!alloc_fair_sched_group(tg))
+       if (!alloc_fair_sched_group(tg, parent))
                goto err;
 
-       if (!alloc_rt_sched_group(tg))
+       if (!alloc_rt_sched_group(tg, parent))
                goto err;
 
        spin_lock_irqsave(&task_group_lock, flags);
@@ -7569,6 +8247,12 @@ struct task_group *sched_create_group(void)
                register_rt_sched_group(tg, i);
        }
        list_add_rcu(&tg->list, &task_groups);
+
+       WARN_ON(!parent); /* root should already exist */
+
+       tg->parent = parent;
+       list_add_rcu(&tg->siblings, &parent->children);
+       INIT_LIST_HEAD(&tg->children);
        spin_unlock_irqrestore(&task_group_lock, flags);
 
        return tg;
@@ -7597,6 +8281,7 @@ void sched_destroy_group(struct task_group *tg)
                unregister_rt_sched_group(tg, i);
        }
        list_del_rcu(&tg->list);
+       list_del_rcu(&tg->siblings);
        spin_unlock_irqrestore(&task_group_lock, flags);
 
        /* wait for possible concurrent references to cfs_rqs complete */
@@ -7640,6 +8325,7 @@ void sched_move_task(struct task_struct *tsk)
 
        task_rq_unlock(rq, &flags);
 }
+#endif
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 static void set_se_shares(struct sched_entity *se, unsigned long shares)
@@ -7655,7 +8341,7 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares)
                dequeue_entity(cfs_rq, se, 0);
 
        se->load.weight = shares;
-       se->load.inv_weight = div64_64((1ULL<<32), shares);
+       se->load.inv_weight = 0;
 
        if (on_rq)
                enqueue_entity(cfs_rq, se, 0);
@@ -7671,12 +8357,15 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares)
        unsigned long flags;
 
        /*
-        * A weight of 0 or 1 can cause arithmetics problems.
-        * (The default weight is 1024 - so there's no practical
-        *  limitation from this.)
+        * We can't change the weight of the root cgroup.
         */
-       if (shares < 2)
-               shares = 2;
+       if (!tg->se[0])
+               return -EINVAL;
+
+       if (shares < MIN_SHARES)
+               shares = MIN_SHARES;
+       else if (shares > MAX_SHARES)
+               shares = MAX_SHARES;
 
        mutex_lock(&shares_mutex);
        if (tg->shares == shares)
@@ -7685,6 +8374,7 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares)
        spin_lock_irqsave(&task_group_lock, flags);
        for_each_possible_cpu(i)
                unregister_fair_sched_group(tg, i);
+       list_del_rcu(&tg->siblings);
        spin_unlock_irqrestore(&task_group_lock, flags);
 
        /* wait for any ongoing reference to this group to finish */
@@ -7705,6 +8395,7 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares)
        spin_lock_irqsave(&task_group_lock, flags);
        for_each_possible_cpu(i)
                register_fair_sched_group(tg, i);
+       list_add_rcu(&tg->siblings, &tg->parent->children);
        spin_unlock_irqrestore(&task_group_lock, flags);
 done:
        mutex_unlock(&shares_mutex);
@@ -7728,29 +8419,61 @@ static unsigned long to_ratio(u64 period, u64 runtime)
        if (runtime == RUNTIME_INF)
                return 1ULL << 16;
 
-       return div64_64(runtime << 16, period);
+       return div64_u64(runtime << 16, period);
 }
 
+#ifdef CONFIG_CGROUP_SCHED
+static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
+{
+       struct task_group *tgi, *parent = tg ? tg->parent : NULL;
+       unsigned long total = 0;
+
+       if (!parent) {
+               if (global_rt_period() < period)
+                       return 0;
+
+               return to_ratio(period, runtime) <
+                       to_ratio(global_rt_period(), global_rt_runtime());
+       }
+
+       if (ktime_to_ns(parent->rt_bandwidth.rt_period) < period)
+               return 0;
+
+       rcu_read_lock();
+       list_for_each_entry_rcu(tgi, &parent->children, siblings) {
+               if (tgi == tg)
+                       continue;
+
+               total += to_ratio(ktime_to_ns(tgi->rt_bandwidth.rt_period),
+                               tgi->rt_bandwidth.rt_runtime);
+       }
+       rcu_read_unlock();
+
+       return total + to_ratio(period, runtime) <
+               to_ratio(ktime_to_ns(parent->rt_bandwidth.rt_period),
+                               parent->rt_bandwidth.rt_runtime);
+}
+#elif defined CONFIG_USER_SCHED
 static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
 {
        struct task_group *tgi;
        unsigned long total = 0;
        unsigned long global_ratio =
-               to_ratio(sysctl_sched_rt_period,
-                        sysctl_sched_rt_runtime < 0 ?
-                               RUNTIME_INF : sysctl_sched_rt_runtime);
+               to_ratio(global_rt_period(), global_rt_runtime());
 
        rcu_read_lock();
        list_for_each_entry_rcu(tgi, &task_groups, list) {
                if (tgi == tg)
                        continue;
 
-               total += to_ratio(period, tgi->rt_runtime);
+               total += to_ratio(ktime_to_ns(tgi->rt_bandwidth.rt_period),
+                               tgi->rt_bandwidth.rt_runtime);
        }
        rcu_read_unlock();
 
        return total + to_ratio(period, runtime) < global_ratio;
 }
+#endif
 
 /* Must be called with tasklist_lock held */
 static inline int tg_has_rt_tasks(struct task_group *tg)
@@ -7763,19 +8486,14 @@ static inline int tg_has_rt_tasks(struct task_group *tg)
        return 0;
 }
 
-int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
+static int tg_set_bandwidth(struct task_group *tg,
+               u64 rt_period, u64 rt_runtime)
 {
-       u64 rt_runtime, rt_period;
-       int err = 0;
-
-       rt_period = (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
-       rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
-       if (rt_runtime_us == -1)
-               rt_runtime = RUNTIME_INF;
+       int i, err = 0;
 
        mutex_lock(&rt_constraints_mutex);
        read_lock(&tasklist_lock);
-       if (rt_runtime_us == 0 && tg_has_rt_tasks(tg)) {
+       if (rt_runtime == 0 && tg_has_rt_tasks(tg)) {
                err = -EBUSY;
                goto unlock;
        }
@@ -7783,7 +8501,19 @@ int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
                err = -EINVAL;
                goto unlock;
        }
-       tg->rt_runtime = rt_runtime;
+
+       spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
+       tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
+       tg->rt_bandwidth.rt_runtime = rt_runtime;
+
+       for_each_possible_cpu(i) {
+               struct rt_rq *rt_rq = tg->rt_rq[i];
+
+               spin_lock(&rt_rq->rt_runtime_lock);
+               rt_rq->rt_runtime = rt_runtime;
+               spin_unlock(&rt_rq->rt_runtime_lock);
+       }
+       spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
  unlock:
        read_unlock(&tasklist_lock);
        mutex_unlock(&rt_constraints_mutex);
@@ -7791,19 +8521,109 @@ int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
        return err;
 }
 
+int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
+{
+       u64 rt_runtime, rt_period;
+
+       rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
+       rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
+       if (rt_runtime_us < 0)
+               rt_runtime = RUNTIME_INF;
+
+       return tg_set_bandwidth(tg, rt_period, rt_runtime);
+}
+
 long sched_group_rt_runtime(struct task_group *tg)
 {
        u64 rt_runtime_us;
 
-       if (tg->rt_runtime == RUNTIME_INF)
+       if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
                return -1;
 
-       rt_runtime_us = tg->rt_runtime;
+       rt_runtime_us = tg->rt_bandwidth.rt_runtime;
        do_div(rt_runtime_us, NSEC_PER_USEC);
        return rt_runtime_us;
 }
+
+int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
+{
+       u64 rt_runtime, rt_period;
+
+       rt_period = (u64)rt_period_us * NSEC_PER_USEC;
+       rt_runtime = tg->rt_bandwidth.rt_runtime;
+
+       return tg_set_bandwidth(tg, rt_period, rt_runtime);
+}
+
+long sched_group_rt_period(struct task_group *tg)
+{
+       u64 rt_period_us;
+
+       rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
+       do_div(rt_period_us, NSEC_PER_USEC);
+       return rt_period_us;
+}
+
+static int sched_rt_global_constraints(void)
+{
+       int ret = 0;
+
+       mutex_lock(&rt_constraints_mutex);
+       if (!__rt_schedulable(NULL, 1, 0))
+               ret = -EINVAL;
+       mutex_unlock(&rt_constraints_mutex);
+
+       return ret;
+}
+#else
+static int sched_rt_global_constraints(void)
+{
+       unsigned long flags;
+       int i;
+
+       spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
+       for_each_possible_cpu(i) {
+               struct rt_rq *rt_rq = &cpu_rq(i)->rt;
+
+               spin_lock(&rt_rq->rt_runtime_lock);
+               rt_rq->rt_runtime = global_rt_runtime();
+               spin_unlock(&rt_rq->rt_runtime_lock);
+       }
+       spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
+
+       return 0;
+}
 #endif
-#endif /* CONFIG_GROUP_SCHED */
+
+int sched_rt_handler(struct ctl_table *table, int write,
+               struct file *filp, void __user *buffer, size_t *lenp,
+               loff_t *ppos)
+{
+       int ret;
+       int old_period, old_runtime;
+       static DEFINE_MUTEX(mutex);
+
+       mutex_lock(&mutex);
+       old_period = sysctl_sched_rt_period;
+       old_runtime = sysctl_sched_rt_runtime;
+
+       ret = proc_dointvec(table, write, filp, buffer, lenp, ppos);
+
+       if (!ret && write) {
+               ret = sched_rt_global_constraints();
+               if (ret) {
+                       sysctl_sched_rt_period = old_period;
+                       sysctl_sched_rt_runtime = old_runtime;
+               } else {
+                       def_rt_bandwidth.rt_runtime = global_rt_runtime();
+                       def_rt_bandwidth.rt_period =
+                               ns_to_ktime(global_rt_period());
+               }
+       }
+       mutex_unlock(&mutex);
+
+       return ret;
+}
 
 #ifdef CONFIG_CGROUP_SCHED
 
@@ -7817,7 +8637,7 @@ static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
 static struct cgroup_subsys_state *
 cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
 {
-       struct task_group *tg;
+       struct task_group *tg, *parent;
 
        if (!cgrp->parent) {
                /* This is early initialization for the top cgroup */
@@ -7825,11 +8645,8 @@ cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
                return &init_task_group.css;
        }
 
-       /* we support only 1-level deep hierarchical scheduler atm */
-       if (cgrp->parent->parent)
-               return ERR_PTR(-EINVAL);
-
-       tg = sched_create_group();
+       parent = cgroup_tg(cgrp->parent);
+       tg = sched_create_group(parent);
        if (IS_ERR(tg))
                return ERR_PTR(-ENOMEM);
 
@@ -7853,7 +8670,7 @@ cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
 {
 #ifdef CONFIG_RT_GROUP_SCHED
        /* Don't accept realtime tasks when there is no way for them to run */
-       if (rt_task(tsk) && cgroup_tg(cgrp)->rt_runtime == 0)
+       if (rt_task(tsk) && cgroup_tg(cgrp)->rt_bandwidth.rt_runtime == 0)
                return -EINVAL;
 #else
        /* We don't support RT-tasks being in separate groups */
@@ -7872,13 +8689,13 @@ cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
 }
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype,
+static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
                                u64 shareval)
 {
        return sched_group_set_shares(cgroup_tg(cgrp), shareval);
 }
 
-static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft)
+static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
 {
        struct task_group *tg = cgroup_tg(cgrp);
 
@@ -7888,48 +8705,25 @@ static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft)
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
-                               struct file *file,
-                               const char __user *userbuf,
-                               size_t nbytes, loff_t *unused_ppos)
+                               s64 val)
 {
-       char buffer[64];
-       int retval = 0;
-       s64 val;
-       char *end;
-
-       if (!nbytes)
-               return -EINVAL;
-       if (nbytes >= sizeof(buffer))
-               return -E2BIG;
-       if (copy_from_user(buffer, userbuf, nbytes))
-               return -EFAULT;
-
-       buffer[nbytes] = 0;     /* nul-terminate */
-
-       /* strip newline if necessary */
-       if (nbytes && (buffer[nbytes-1] == '\n'))
-               buffer[nbytes-1] = 0;
-       val = simple_strtoll(buffer, &end, 0);
-       if (*end)
-               return -EINVAL;
+       return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
+}
 
-       /* Pass to subsystem */
-       retval = sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
-       if (!retval)
-               retval = nbytes;
-       return retval;
+static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
+{
+       return sched_group_rt_runtime(cgroup_tg(cgrp));
 }
 
-static ssize_t cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft,
-                                  struct file *file,
-                                  char __user *buf, size_t nbytes,
-                                  loff_t *ppos)
+static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
+               u64 rt_period_us)
 {
-       char tmp[64];
-       long val = sched_group_rt_runtime(cgroup_tg(cgrp));
-       int len = sprintf(tmp, "%ld\n", val);
+       return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
+}
 
-       return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
+static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
+{
+       return sched_group_rt_period(cgroup_tg(cgrp));
 }
 #endif
 
@@ -7937,15 +8731,20 @@ static struct cftype cpu_files[] = {
 #ifdef CONFIG_FAIR_GROUP_SCHED
        {
                .name = "shares",
-               .read_uint = cpu_shares_read_uint,
-               .write_uint = cpu_shares_write_uint,
+               .read_u64 = cpu_shares_read_u64,
+               .write_u64 = cpu_shares_write_u64,
        },
 #endif
 #ifdef CONFIG_RT_GROUP_SCHED
        {
                .name = "rt_runtime_us",
-               .read = cpu_rt_runtime_read,
-               .write = cpu_rt_runtime_write,
+               .read_s64 = cpu_rt_runtime_read,
+               .write_s64 = cpu_rt_runtime_write,
+       },
+       {
+               .name = "rt_period_us",
+               .read_u64 = cpu_rt_period_read_uint,
+               .write_u64 = cpu_rt_period_write_uint,
        },
 #endif
 };
@@ -7987,9 +8786,9 @@ struct cpuacct {
 struct cgroup_subsys cpuacct_subsys;
 
 /* return cpu accounting group corresponding to this container */
-static inline struct cpuacct *cgroup_ca(struct cgroup *cont)
+static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
 {
-       return container_of(cgroup_subsys_state(cont, cpuacct_subsys_id),
+       return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
                            struct cpuacct, css);
 }
 
@@ -8002,7 +8801,7 @@ static inline struct cpuacct *task_ca(struct task_struct *tsk)
 
 /* create a new cpu accounting group */
 static struct cgroup_subsys_state *cpuacct_create(
-       struct cgroup_subsys *ss, struct cgroup *cont)
+       struct cgroup_subsys *ss, struct cgroup *cgrp)
 {
        struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
 
@@ -8020,18 +8819,18 @@ static struct cgroup_subsys_state *cpuacct_create(
 
 /* destroy an existing cpu accounting group */
 static void
-cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cont)
+cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
 {
-       struct cpuacct *ca = cgroup_ca(cont);
+       struct cpuacct *ca = cgroup_ca(cgrp);
 
        free_percpu(ca->cpuusage);
        kfree(ca);
 }
 
 /* return total cpu usage (in nanoseconds) of a group */
-static u64 cpuusage_read(struct cgroup *cont, struct cftype *cft)
+static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
 {
-       struct cpuacct *ca = cgroup_ca(cont);
+       struct cpuacct *ca = cgroup_ca(cgrp);
        u64 totalcpuusage = 0;
        int i;
 
@@ -8050,16 +8849,40 @@ static u64 cpuusage_read(struct cgroup *cont, struct cftype *cft)
        return totalcpuusage;
 }
 
+static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
+                                                               u64 reset)
+{
+       struct cpuacct *ca = cgroup_ca(cgrp);
+       int err = 0;
+       int i;
+
+       if (reset) {
+               err = -EINVAL;
+               goto out;
+       }
+
+       for_each_possible_cpu(i) {
+               u64 *cpuusage = percpu_ptr(ca->cpuusage, i);
+
+               spin_lock_irq(&cpu_rq(i)->lock);
+               *cpuusage = 0;
+               spin_unlock_irq(&cpu_rq(i)->lock);
+       }
+out:
+       return err;
+}
+
 static struct cftype files[] = {
        {
                .name = "usage",
-               .read_uint = cpuusage_read,
+               .read_u64 = cpuusage_read,
+               .write_u64 = cpuusage_write,
        },
 };
 
-static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cont)
+static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
 {
-       return cgroup_add_files(cont, ss, files, ARRAY_SIZE(files));
+       return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
 }
 
 /*