SCHED_TTWU_QUEUE is not longer needed since sparc32 now implements IPI
[linux-2.6.git] / kernel / sched.c
index 5ec2e8b..c4b3410 100644 (file)
@@ -556,6 +556,10 @@ struct rq {
        unsigned int ttwu_count;
        unsigned int ttwu_local;
 #endif
+
+#ifdef CONFIG_SMP
+       struct task_struct *wake_list;
+#endif
 };
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
@@ -2456,8 +2460,11 @@ static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
                wq_worker_waking_up(p, cpu_of(rq));
 }
 
+/*
+ * Mark the task runnable and perform wakeup-preemption.
+ */
 static void
-ttwu_post_activation(struct task_struct *p, struct rq *rq, int wake_flags)
+ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
 {
        trace_sched_wakeup(p, true);
        check_preempt_curr(rq, p, wake_flags);
@@ -2480,6 +2487,99 @@ ttwu_post_activation(struct task_struct *p, struct rq *rq, int wake_flags)
 #endif
 }
 
+static void
+ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
+{
+#ifdef CONFIG_SMP
+       if (p->sched_contributes_to_load)
+               rq->nr_uninterruptible--;
+#endif
+
+       ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
+       ttwu_do_wakeup(rq, p, wake_flags);
+}
+
+/*
+ * Called in case the task @p isn't fully descheduled from its runqueue,
+ * in this case we must do a remote wakeup. Its a 'light' wakeup though,
+ * since all we need to do is flip p->state to TASK_RUNNING, since
+ * the task is still ->on_rq.
+ */
+static int ttwu_remote(struct task_struct *p, int wake_flags)
+{
+       struct rq *rq;
+       int ret = 0;
+
+       rq = __task_rq_lock(p);
+       if (p->on_rq) {
+               ttwu_do_wakeup(rq, p, wake_flags);
+               ret = 1;
+       }
+       __task_rq_unlock(rq);
+
+       return ret;
+}
+
+#ifdef CONFIG_SMP
+static void sched_ttwu_pending(void)
+{
+       struct rq *rq = this_rq();
+       struct task_struct *list = xchg(&rq->wake_list, NULL);
+
+       if (!list)
+               return;
+
+       raw_spin_lock(&rq->lock);
+
+       while (list) {
+               struct task_struct *p = list;
+               list = list->wake_entry;
+               ttwu_do_activate(rq, p, 0);
+       }
+
+       raw_spin_unlock(&rq->lock);
+}
+
+void scheduler_ipi(void)
+{
+       sched_ttwu_pending();
+}
+
+static void ttwu_queue_remote(struct task_struct *p, int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+       struct task_struct *next = rq->wake_list;
+
+       for (;;) {
+               struct task_struct *old = next;
+
+               p->wake_entry = next;
+               next = cmpxchg(&rq->wake_list, old, p);
+               if (next == old)
+                       break;
+       }
+
+       if (!next)
+               smp_send_reschedule(cpu);
+}
+#endif
+
+static void ttwu_queue(struct task_struct *p, int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+
+#if defined(CONFIG_SMP)
+       if (sched_feat(TTWU_QUEUE) && cpu != smp_processor_id()) {
+               ttwu_queue_remote(p, cpu);
+               return;
+       }
+#endif
+
+       raw_spin_lock(&rq->lock);
+       ttwu_do_activate(rq, p, 0);
+       raw_spin_unlock(&rq->lock);
+}
+
 /**
  * try_to_wake_up - wake up a thread
  * @p: the thread to be awakened
@@ -2498,27 +2598,25 @@ ttwu_post_activation(struct task_struct *p, struct rq *rq, int wake_flags)
 static int
 try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 {
-       int cpu, this_cpu, success = 0;
        unsigned long flags;
-       struct rq *rq;
-
-       this_cpu = get_cpu();
+       int cpu, success = 0;
 
        smp_wmb();
        raw_spin_lock_irqsave(&p->pi_lock, flags);
        if (!(p->state & state))
                goto out;
 
+       success = 1; /* we're going to change ->state */
        cpu = task_cpu(p);
 
-       if (p->on_rq) {
-               rq = __task_rq_lock(p);
-               if (p->on_rq)
-                       goto out_running;
-               __task_rq_unlock(rq);
-       }
+       if (p->on_rq && ttwu_remote(p, wake_flags))
+               goto stat;
 
 #ifdef CONFIG_SMP
+       /*
+        * If the owning (remote) cpu is still in the middle of schedule() with
+        * this task as prev, wait until its done referencing the task.
+        */
        while (p->on_cpu) {
 #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
                /*
@@ -2527,8 +2625,10 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
                 * to spin on ->on_cpu if p is current, since that would
                 * deadlock.
                 */
-               if (p == current)
-                       goto out_activate;
+               if (p == current) {
+                       ttwu_queue(p, cpu);
+                       goto stat;
+               }
 #endif
                cpu_relax();
        }
@@ -2544,32 +2644,15 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
                p->sched_class->task_waking(p);
 
        cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
-#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
-out_activate:
-#endif
-#endif /* CONFIG_SMP */
-
-       rq = cpu_rq(cpu);
-       raw_spin_lock(&rq->lock);
-
-#ifdef CONFIG_SMP
-       if (cpu != task_cpu(p))
+       if (task_cpu(p) != cpu)
                set_task_cpu(p, cpu);
+#endif /* CONFIG_SMP */
 
-       if (p->sched_contributes_to_load)
-               rq->nr_uninterruptible--;
-#endif
-
-       ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
-out_running:
-       ttwu_post_activation(p, rq, wake_flags);
-       success = 1;
-       __task_rq_unlock(rq);
-
+       ttwu_queue(p, cpu);
+stat:
        ttwu_stat(p, cpu, wake_flags);
 out:
        raw_spin_unlock_irqrestore(&p->pi_lock, flags);
-       put_cpu();
 
        return success;
 }
@@ -2602,7 +2685,7 @@ static void try_to_wake_up_local(struct task_struct *p)
        if (!p->on_rq)
                ttwu_activate(rq, p, ENQUEUE_WAKEUP);
 
-       ttwu_post_activation(p, rq, 0);
+       ttwu_do_wakeup(rq, p, 0);
        ttwu_stat(p, smp_processor_id(), 0);
 out:
        raw_spin_unlock(&p->pi_lock);
@@ -4141,7 +4224,7 @@ need_resched:
                         */
                        if (blk_needs_flush_plug(prev)) {
                                raw_spin_unlock(&rq->lock);
-                               blk_flush_plug(prev);
+                               blk_schedule_flush_plug(prev);
                                raw_spin_lock(&rq->lock);
                        }
                }
@@ -6303,6 +6386,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
 
 #ifdef CONFIG_HOTPLUG_CPU
        case CPU_DYING:
+               sched_ttwu_pending();
                /* Update our root-domain */
                raw_spin_lock_irqsave(&rq->lock, flags);
                if (rq->rd) {