[PATCH] Flush MMIO writes in reset sequence
[linux-2.6.git] / kernel / futex.c
index f59003b..9d260e8 100644 (file)
@@ -297,7 +297,7 @@ static int futex_handle_fault(unsigned long address, int attempt)
        struct vm_area_struct * vma;
        struct mm_struct *mm = current->mm;
 
-       if (attempt >= 2 || !(vma = find_vma(mm, address)) ||
+       if (attempt > 2 || !(vma = find_vma(mm, address)) ||
            vma->vm_start > address || !(vma->vm_flags & VM_WRITE))
                return -EFAULT;
 
@@ -397,7 +397,7 @@ static struct task_struct * futex_find_get_task(pid_t pid)
                p = NULL;
                goto out_unlock;
        }
-       if (p->state == EXIT_ZOMBIE || p->exit_state == EXIT_ZOMBIE) {
+       if (p->exit_state != 0) {
                p = NULL;
                goto out_unlock;
        }
@@ -495,10 +495,13 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, struct futex_q *me)
        }
 
        /*
-        * We are the first waiter - try to look up the real owner and
-        * attach the new pi_state to it:
+        * We are the first waiter - try to look up the real owner and attach
+        * the new pi_state to it, but bail out when the owner died bit is set
+        * and TID = 0:
         */
        pid = uval & FUTEX_TID_MASK;
+       if (!pid && (uval & FUTEX_OWNER_DIED))
+               return -ESRCH;
        p = futex_find_get_task(pid);
        if (!p)
                return -ESRCH;
@@ -579,16 +582,17 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
         * kept enabled while there is PI state around. We must also
         * preserve the owner died bit.)
         */
-       newval = (uval & FUTEX_OWNER_DIED) | FUTEX_WAITERS | new_owner->pid;
-
-       inc_preempt_count();
-       curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
-       dec_preempt_count();
+       if (!(uval & FUTEX_OWNER_DIED)) {
+               newval = FUTEX_WAITERS | new_owner->pid;
 
-       if (curval == -EFAULT)
-               return -EFAULT;
-       if (curval != uval)
-               return -EINVAL;
+               inc_preempt_count();
+               curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
+               dec_preempt_count();
+               if (curval == -EFAULT)
+                       return -EFAULT;
+               if (curval != uval)
+                       return -EINVAL;
+       }
 
        spin_lock_irq(&pi_state->owner->pi_lock);
        WARN_ON(list_empty(&pi_state->list));
@@ -743,8 +747,10 @@ retry:
                 */
                if (attempt++) {
                        if (futex_handle_fault((unsigned long)uaddr2,
-                                              attempt))
+                                               attempt)) {
+                               ret = -EFAULT;
                                goto out;
+                       }
                        goto retry;
                }
 
@@ -944,6 +950,7 @@ static int unqueue_me(struct futex_q *q)
        /* In the common case we don't take the spinlock, which is nice. */
  retry:
        lock_ptr = q->lock_ptr;
+       barrier();
        if (lock_ptr != 0) {
                spin_lock(lock_ptr);
                /*
@@ -1113,9 +1120,10 @@ static int futex_wait(u32 __user *uaddr, u32 val, unsigned long time)
  * if there are waiters then it will block, it does PI, etc. (Due to
  * races the kernel might see a 0 value of the futex too.)
  */
-static int do_futex_lock_pi(u32 __user *uaddr, int detect, int trylock,
-                           struct hrtimer_sleeper *to)
+static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec,
+                        long nsec, int trylock)
 {
+       struct hrtimer_sleeper timeout, *to = NULL;
        struct task_struct *curr = current;
        struct futex_hash_bucket *hb;
        u32 uval, newval, curval;
@@ -1125,6 +1133,13 @@ static int do_futex_lock_pi(u32 __user *uaddr, int detect, int trylock,
        if (refill_pi_state_cache())
                return -ENOMEM;
 
+       if (sec != MAX_SCHEDULE_TIMEOUT) {
+               to = &timeout;
+               hrtimer_init(&to->timer, CLOCK_REALTIME, HRTIMER_ABS);
+               hrtimer_init_sleeper(to, current);
+               to->timer.expires = ktime_set(sec, nsec);
+       }
+
        q.pi_state = NULL;
  retry:
        down_read(&curr->mm->mmap_sem);
@@ -1300,7 +1315,7 @@ static int do_futex_lock_pi(u32 __user *uaddr, int detect, int trylock,
        if (!detect && ret == -EDEADLK && 0)
                force_sig(SIGKILL, current);
 
-       return ret;
+       return ret != -EINTR ? ret : -ERESTARTNOINTR;
 
  out_unlock_release_sem:
        queue_unlock(&q, hb);
@@ -1317,9 +1332,10 @@ static int do_futex_lock_pi(u32 __user *uaddr, int detect, int trylock,
         * still holding the mmap_sem.
         */
        if (attempt++) {
-               if (futex_handle_fault((unsigned long)uaddr, attempt))
+               if (futex_handle_fault((unsigned long)uaddr, attempt)) {
+                       ret = -EFAULT;
                        goto out_unlock_release_sem;
-
+               }
                goto retry_locked;
        }
 
@@ -1334,76 +1350,6 @@ static int do_futex_lock_pi(u32 __user *uaddr, int detect, int trylock,
 }
 
 /*
- * Restart handler
- */
-static long futex_lock_pi_restart(struct restart_block *restart)
-{
-       struct hrtimer_sleeper timeout, *to = NULL;
-       int ret;
-
-       restart->fn = do_no_restart_syscall;
-
-       if (restart->arg2 || restart->arg3) {
-               to = &timeout;
-               hrtimer_init(&to->timer, CLOCK_REALTIME, HRTIMER_ABS);
-               hrtimer_init_sleeper(to, current);
-               to->timer.expires.tv64 = ((u64)restart->arg1 << 32) |
-                       (u64) restart->arg0;
-       }
-
-       pr_debug("lock_pi restart: %p, %d (%d)\n",
-                (u32 __user *)restart->arg0, current->pid);
-
-       ret = do_futex_lock_pi((u32 __user *)restart->arg0, restart->arg1,
-                              0, to);
-
-       if (ret != -EINTR)
-               return ret;
-
-       restart->fn = futex_lock_pi_restart;
-
-       /* The other values are filled in */
-       return -ERESTART_RESTARTBLOCK;
-}
-
-/*
- * Called from the syscall entry below.
- */
-static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec,
-                        long nsec, int trylock)
-{
-       struct hrtimer_sleeper timeout, *to = NULL;
-       struct restart_block *restart;
-       int ret;
-
-       if (sec != MAX_SCHEDULE_TIMEOUT) {
-               to = &timeout;
-               hrtimer_init(&to->timer, CLOCK_REALTIME, HRTIMER_ABS);
-               hrtimer_init_sleeper(to, current);
-               to->timer.expires = ktime_set(sec, nsec);
-       }
-
-       ret = do_futex_lock_pi(uaddr, detect, trylock, to);
-
-       if (ret != -EINTR)
-               return ret;
-
-       pr_debug("lock_pi interrupted: %p, %d (%d)\n", uaddr, current->pid);
-
-       restart = &current_thread_info()->restart_block;
-       restart->fn = futex_lock_pi_restart;
-       restart->arg0 = (unsigned long) uaddr;
-       restart->arg1 = detect;
-       if (to) {
-               restart->arg2 = to->timer.expires.tv64 & 0xFFFFFFFF;
-               restart->arg3 = to->timer.expires.tv64 >> 32;
-       } else
-               restart->arg2 = restart->arg3 = 0;
-
-       return -ERESTART_RESTARTBLOCK;
-}
-
-/*
  * Userspace attempted a TID -> 0 atomic transition, and failed.
  * This is the in-kernel slowpath: we look up the PI state (if any),
  * and do the rt-mutex unlock.
@@ -1443,9 +1389,11 @@ retry_locked:
         * again. If it succeeds then we can return without waking
         * anyone else up:
         */
-       inc_preempt_count();
-       uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0);
-       dec_preempt_count();
+       if (!(uval & FUTEX_OWNER_DIED)) {
+               inc_preempt_count();
+               uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0);
+               dec_preempt_count();
+       }
 
        if (unlikely(uval == -EFAULT))
                goto pi_faulted;
@@ -1478,9 +1426,11 @@ retry_locked:
        /*
         * No waiters - kernel unlocks the futex:
         */
-       ret = unlock_futex_pi(uaddr, uval);
-       if (ret == -EFAULT)
-               goto pi_faulted;
+       if (!(uval & FUTEX_OWNER_DIED)) {
+               ret = unlock_futex_pi(uaddr, uval);
+               if (ret == -EFAULT)
+                       goto pi_faulted;
+       }
 
 out_unlock:
        spin_unlock(&hb->lock);
@@ -1497,9 +1447,10 @@ pi_faulted:
         * still holding the mmap_sem.
         */
        if (attempt++) {
-               if (futex_handle_fault((unsigned long)uaddr, attempt))
+               if (futex_handle_fault((unsigned long)uaddr, attempt)) {
+                       ret = -EFAULT;
                        goto out_unlock;
-
+               }
                goto retry_locked;
        }
 
@@ -1699,9 +1650,9 @@ err_unlock:
  * Process a futex-list entry, check whether it's owned by the
  * dying task, and do notification if so:
  */
-int handle_futex_death(u32 __user *uaddr, struct task_struct *curr)
+int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, int pi)
 {
-       u32 uval, nval;
+       u32 uval, nval, mval;
 
 retry:
        if (get_user(uval, uaddr))
@@ -1718,21 +1669,45 @@ retry:
                 * thread-death.) The rest of the cleanup is done in
                 * userspace.
                 */
-               nval = futex_atomic_cmpxchg_inatomic(uaddr, uval,
-                                                    uval | FUTEX_OWNER_DIED);
+               mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
+               nval = futex_atomic_cmpxchg_inatomic(uaddr, uval, mval);
+
                if (nval == -EFAULT)
                        return -1;
 
                if (nval != uval)
                        goto retry;
 
-               if (uval & FUTEX_WAITERS)
-                       futex_wake(uaddr, 1);
+               /*
+                * Wake robust non-PI futexes here. The wakeup of
+                * PI futexes happens in exit_pi_state():
+                */
+               if (!pi) {
+                       if (uval & FUTEX_WAITERS)
+                               futex_wake(uaddr, 1);
+               }
        }
        return 0;
 }
 
 /*
+ * Fetch a robust-list pointer. Bit 0 signals PI futexes:
+ */
+static inline int fetch_robust_entry(struct robust_list __user **entry,
+                                    struct robust_list __user **head, int *pi)
+{
+       unsigned long uentry;
+
+       if (get_user(uentry, (unsigned long *)head))
+               return -EFAULT;
+
+       *entry = (void *)(uentry & ~1UL);
+       *pi = uentry & 1;
+
+       return 0;
+}
+
+/*
  * Walk curr->robust_list (very carefully, it's a userspace list!)
  * and mark any locks found there dead, and notify any waiters.
  *
@@ -1742,14 +1717,14 @@ void exit_robust_list(struct task_struct *curr)
 {
        struct robust_list_head __user *head = curr->robust_list;
        struct robust_list __user *entry, *pending;
-       unsigned int limit = ROBUST_LIST_LIMIT;
+       unsigned int limit = ROBUST_LIST_LIMIT, pi, pip;
        unsigned long futex_offset;
 
        /*
         * Fetch the list head (which was registered earlier, via
         * sys_set_robust_list()):
         */
-       if (get_user(entry, &head->list.next))
+       if (fetch_robust_entry(&entry, &head->list.next, &pi))
                return;
        /*
         * Fetch the relative futex offset:
@@ -1760,10 +1735,11 @@ void exit_robust_list(struct task_struct *curr)
         * Fetch any possibly pending lock-add first, and handle it
         * if it exists:
         */
-       if (get_user(pending, &head->list_op_pending))
+       if (fetch_robust_entry(&pending, &head->list_op_pending, &pip))
                return;
+
        if (pending)
-               handle_futex_death((void *)pending + futex_offset, curr);
+               handle_futex_death((void *)pending + futex_offset, curr, pip);
 
        while (entry != &head->list) {
                /*
@@ -1772,12 +1748,12 @@ void exit_robust_list(struct task_struct *curr)
                 */
                if (entry != pending)
                        if (handle_futex_death((void *)entry + futex_offset,
-                                               curr))
+                                               curr, pi))
                                return;
                /*
                 * Fetch the next entry in the list:
                 */
-               if (get_user(entry, &entry->next))
+               if (fetch_robust_entry(&entry, &entry->next, &pi))
                        return;
                /*
                 * Avoid excessively long or circular lists: