ksym_tracer: Remove trace_stat
[linux-2.6.git] / kernel / exit.c
index 826e1dc..6f50ef5 100644 (file)
@@ -12,7 +12,6 @@
 #include <linux/completion.h>
 #include <linux/personality.h>
 #include <linux/tty.h>
-#include <linux/mnt_namespace.h>
 #include <linux/iocontext.h>
 #include <linux/key.h>
 #include <linux/security.h>
@@ -48,8 +47,9 @@
 #include <linux/tracehook.h>
 #include <linux/fs_struct.h>
 #include <linux/init_task.h>
-#include <linux/perf_counter.h>
+#include <linux/perf_event.h>
 #include <trace/events/sched.h>
+#include <linux/hw_breakpoint.h>
 
 #include <asm/uaccess.h>
 #include <asm/unistd.h>
@@ -111,9 +111,9 @@ static void __exit_signal(struct task_struct *tsk)
                 * We won't ever get here for the group leader, since it
                 * will have been the last reference on the signal_struct.
                 */
-               sig->utime = cputime_add(sig->utime, task_utime(tsk));
-               sig->stime = cputime_add(sig->stime, task_stime(tsk));
-               sig->gtime = cputime_add(sig->gtime, task_gtime(tsk));
+               sig->utime = cputime_add(sig->utime, tsk->utime);
+               sig->stime = cputime_add(sig->stime, tsk->stime);
+               sig->gtime = cputime_add(sig->gtime, tsk->gtime);
                sig->min_flt += tsk->min_flt;
                sig->maj_flt += tsk->maj_flt;
                sig->nvcsw += tsk->nvcsw;
@@ -155,8 +155,8 @@ static void delayed_put_task_struct(struct rcu_head *rhp)
 {
        struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
 
-#ifdef CONFIG_PERF_COUNTERS
-       WARN_ON_ONCE(tsk->perf_counter_ctxp);
+#ifdef CONFIG_PERF_EVENTS
+       WARN_ON_ONCE(tsk->perf_event_ctxp);
 #endif
        trace_sched_process_free(tsk);
        put_task_struct(tsk);
@@ -902,6 +902,8 @@ NORET_TYPE void do_exit(long code)
 
        tracehook_report_exit(&code);
 
+       validate_creds_for_do_exit(tsk);
+
        /*
         * We're taking recursive faults here in do_exit. Safest is to just
         * leave this task alone and wait for reboot.
@@ -944,6 +946,8 @@ NORET_TYPE void do_exit(long code)
        if (group_dead) {
                hrtimer_cancel(&tsk->signal->real_timer);
                exit_itimers(tsk->signal);
+               if (tsk->mm)
+                       setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm);
        }
        acct_collect(code, group_dead);
        if (group_dead)
@@ -967,20 +971,22 @@ NORET_TYPE void do_exit(long code)
        exit_thread();
        cgroup_exit(tsk, 1);
 
-       if (group_dead && tsk->signal->leader)
+       if (group_dead)
                disassociate_ctty(1);
 
        module_put(task_thread_info(tsk)->exec_domain->module);
-       if (tsk->binfmt)
-               module_put(tsk->binfmt->module);
 
        proc_exit_connector(tsk);
 
        /*
+        * FIXME: do that only when needed, using sched_exit tracepoint
+        */
+       flush_ptrace_hw_breakpoint(tsk);
+       /*
         * Flush inherited counters to the parent - before the parent
         * gets woken up by child-exit notifications.
         */
-       perf_counter_exit_task(tsk);
+       perf_event_exit_task(tsk);
 
        exit_notify(tsk, group_dead);
 #ifdef CONFIG_NUMA
@@ -988,8 +994,6 @@ NORET_TYPE void do_exit(long code)
        tsk->mempolicy = NULL;
 #endif
 #ifdef CONFIG_FUTEX
-       if (unlikely(!list_empty(&tsk->pi_state_list)))
-               exit_pi_state_list(tsk);
        if (unlikely(current->pi_state_cache))
                kfree(current->pi_state_cache);
 #endif
@@ -1005,12 +1009,15 @@ NORET_TYPE void do_exit(long code)
        tsk->flags |= PF_EXITPIDONE;
 
        if (tsk->io_context)
-               exit_io_context();
+               exit_io_context(tsk);
 
        if (tsk->splice_pipe)
                __free_pipe_info(tsk->splice_pipe);
 
+       validate_creds_for_do_exit(tsk);
+
        preempt_disable();
+       exit_rcu();
        /* causes final put_task_struct in finish_task_switch(). */
        tsk->state = TASK_DEAD;
        schedule();
@@ -1080,62 +1087,72 @@ SYSCALL_DEFINE1(exit_group, int, error_code)
        return 0;
 }
 
-static struct pid *task_pid_type(struct task_struct *task, enum pid_type type)
+struct wait_opts {
+       enum pid_type           wo_type;
+       int                     wo_flags;
+       struct pid              *wo_pid;
+
+       struct siginfo __user   *wo_info;
+       int __user              *wo_stat;
+       struct rusage __user    *wo_rusage;
+
+       wait_queue_t            child_wait;
+       int                     notask_error;
+};
+
+static inline
+struct pid *task_pid_type(struct task_struct *task, enum pid_type type)
 {
-       struct pid *pid = NULL;
-       if (type == PIDTYPE_PID)
-               pid = task->pids[type].pid;
-       else if (type < PIDTYPE_MAX)
-               pid = task->group_leader->pids[type].pid;
-       return pid;
+       if (type != PIDTYPE_PID)
+               task = task->group_leader;
+       return task->pids[type].pid;
 }
 
-static int eligible_child(enum pid_type type, struct pid *pid, int options,
-                         struct task_struct *p)
+static int eligible_pid(struct wait_opts *wo, struct task_struct *p)
 {
-       int err;
-
-       if (type < PIDTYPE_MAX) {
-               if (task_pid_type(p, type) != pid)
-                       return 0;
-       }
+       return  wo->wo_type == PIDTYPE_MAX ||
+               task_pid_type(p, wo->wo_type) == wo->wo_pid;
+}
 
+static int eligible_child(struct wait_opts *wo, struct task_struct *p)
+{
+       if (!eligible_pid(wo, p))
+               return 0;
        /* Wait for all children (clone and not) if __WALL is set;
         * otherwise, wait for clone children *only* if __WCLONE is
         * set; otherwise, wait for non-clone children *only*.  (Note:
         * A "clone" child here is one that reports to its parent
         * using a signal other than SIGCHLD.) */
-       if (((p->exit_signal != SIGCHLD) ^ ((options & __WCLONE) != 0))
-           && !(options & __WALL))
+       if (((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
+           && !(wo->wo_flags & __WALL))
                return 0;
 
-       err = security_task_wait(p);
-       if (err)
-               return err;
-
        return 1;
 }
 
-static int wait_noreap_copyout(struct task_struct *p, pid_t pid, uid_t uid,
-                              int why, int status,
-                              struct siginfo __user *infop,
-                              struct rusage __user *rusagep)
+static int wait_noreap_copyout(struct wait_opts *wo, struct task_struct *p,
+                               pid_t pid, uid_t uid, int why, int status)
 {
-       int retval = rusagep ? getrusage(p, RUSAGE_BOTH, rusagep) : 0;
+       struct siginfo __user *infop;
+       int retval = wo->wo_rusage
+               ? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
 
        put_task_struct(p);
-       if (!retval)
-               retval = put_user(SIGCHLD, &infop->si_signo);
-       if (!retval)
-               retval = put_user(0, &infop->si_errno);
-       if (!retval)
-               retval = put_user((short)why, &infop->si_code);
-       if (!retval)
-               retval = put_user(pid, &infop->si_pid);
-       if (!retval)
-               retval = put_user(uid, &infop->si_uid);
-       if (!retval)
-               retval = put_user(status, &infop->si_status);
+       infop = wo->wo_info;
+       if (infop) {
+               if (!retval)
+                       retval = put_user(SIGCHLD, &infop->si_signo);
+               if (!retval)
+                       retval = put_user(0, &infop->si_errno);
+               if (!retval)
+                       retval = put_user((short)why, &infop->si_code);
+               if (!retval)
+                       retval = put_user(pid, &infop->si_pid);
+               if (!retval)
+                       retval = put_user(uid, &infop->si_uid);
+               if (!retval)
+                       retval = put_user(status, &infop->si_status);
+       }
        if (!retval)
                retval = pid;
        return retval;
@@ -1147,19 +1164,18 @@ static int wait_noreap_copyout(struct task_struct *p, pid_t pid, uid_t uid,
  * the lock and this task is uninteresting.  If we return nonzero, we have
  * released the lock and the system call should return.
  */
-static int wait_task_zombie(struct task_struct *p, int options,
-                           struct siginfo __user *infop,
-                           int __user *stat_addr, struct rusage __user *ru)
+static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
 {
        unsigned long state;
        int retval, status, traced;
        pid_t pid = task_pid_vnr(p);
        uid_t uid = __task_cred(p)->uid;
+       struct siginfo __user *infop;
 
-       if (!likely(options & WEXITED))
+       if (!likely(wo->wo_flags & WEXITED))
                return 0;
 
-       if (unlikely(options & WNOWAIT)) {
+       if (unlikely(wo->wo_flags & WNOWAIT)) {
                int exit_code = p->exit_code;
                int why, status;
 
@@ -1172,8 +1188,7 @@ static int wait_task_zombie(struct task_struct *p, int options,
                        why = (exit_code & 0x80) ? CLD_DUMPED : CLD_KILLED;
                        status = exit_code & 0x7f;
                }
-               return wait_noreap_copyout(p, pid, uid, why,
-                                          status, infop, ru);
+               return wait_noreap_copyout(wo, p, pid, uid, why, status);
        }
 
        /*
@@ -1187,11 +1202,15 @@ static int wait_task_zombie(struct task_struct *p, int options,
        }
 
        traced = ptrace_reparented(p);
-
-       if (likely(!traced)) {
+       /*
+        * It can be ptraced but not reparented, check
+        * !task_detached() to filter out sub-threads.
+        */
+       if (likely(!traced) && likely(!task_detached(p))) {
                struct signal_struct *psig;
                struct signal_struct *sig;
-               struct task_cputime cputime;
+               unsigned long maxrss;
+               cputime_t tgutime, tgstime;
 
                /*
                 * The resource counters for the group leader are in its
@@ -1208,21 +1227,21 @@ static int wait_task_zombie(struct task_struct *p, int options,
                 * as other threads in the parent group can be right
                 * here reaping other children at the same time.
                 *
-                * We use thread_group_cputime() to get times for the thread
+                * We use thread_group_times() to get times for the thread
                 * group, which consolidates times for all threads in the
                 * group including the group leader.
                 */
-               thread_group_cputime(p, &cputime);
+               thread_group_times(p, &tgutime, &tgstime);
                spin_lock_irq(&p->real_parent->sighand->siglock);
                psig = p->real_parent->signal;
                sig = p->signal;
                psig->cutime =
                        cputime_add(psig->cutime,
-                       cputime_add(cputime.utime,
+                       cputime_add(tgutime,
                                    sig->cutime));
                psig->cstime =
                        cputime_add(psig->cstime,
-                       cputime_add(cputime.stime,
+                       cputime_add(tgstime,
                                    sig->cstime));
                psig->cgtime =
                        cputime_add(psig->cgtime,
@@ -1243,6 +1262,9 @@ static int wait_task_zombie(struct task_struct *p, int options,
                psig->coublock +=
                        task_io_get_oublock(p) +
                        sig->oublock + sig->coublock;
+               maxrss = max(sig->maxrss, sig->cmaxrss);
+               if (psig->cmaxrss < maxrss)
+                       psig->cmaxrss = maxrss;
                task_io_accounting_add(&psig->ioac, &p->ioac);
                task_io_accounting_add(&psig->ioac, &sig->ioac);
                spin_unlock_irq(&p->real_parent->sighand->siglock);
@@ -1254,11 +1276,14 @@ static int wait_task_zombie(struct task_struct *p, int options,
         */
        read_unlock(&tasklist_lock);
 
-       retval = ru ? getrusage(p, RUSAGE_BOTH, ru) : 0;
+       retval = wo->wo_rusage
+               ? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
        status = (p->signal->flags & SIGNAL_GROUP_EXIT)
                ? p->signal->group_exit_code : p->exit_code;
-       if (!retval && stat_addr)
-               retval = put_user(status, stat_addr);
+       if (!retval && wo->wo_stat)
+               retval = put_user(status, wo->wo_stat);
+
+       infop = wo->wo_info;
        if (!retval && infop)
                retval = put_user(SIGCHLD, &infop->si_signo);
        if (!retval && infop)
@@ -1326,15 +1351,18 @@ static int *task_stopped_code(struct task_struct *p, bool ptrace)
  * the lock and this task is uninteresting.  If we return nonzero, we have
  * released the lock and the system call should return.
  */
-static int wait_task_stopped(int ptrace, struct task_struct *p,
-                            int options, struct siginfo __user *infop,
-                            int __user *stat_addr, struct rusage __user *ru)
+static int wait_task_stopped(struct wait_opts *wo,
+                               int ptrace, struct task_struct *p)
 {
+       struct siginfo __user *infop;
        int retval, exit_code, *p_code, why;
        uid_t uid = 0; /* unneeded, required by compiler */
        pid_t pid;
 
-       if (!(options & WUNTRACED))
+       /*
+        * Traditionally we see ptrace'd stopped tasks regardless of options.
+        */
+       if (!ptrace && !(wo->wo_flags & WUNTRACED))
                return 0;
 
        exit_code = 0;
@@ -1348,7 +1376,7 @@ static int wait_task_stopped(int ptrace, struct task_struct *p,
        if (!exit_code)
                goto unlock_sig;
 
-       if (!unlikely(options & WNOWAIT))
+       if (!unlikely(wo->wo_flags & WNOWAIT))
                *p_code = 0;
 
        /* don't need the RCU readlock here as we're holding a spinlock */
@@ -1370,14 +1398,15 @@ unlock_sig:
        why = ptrace ? CLD_TRAPPED : CLD_STOPPED;
        read_unlock(&tasklist_lock);
 
-       if (unlikely(options & WNOWAIT))
-               return wait_noreap_copyout(p, pid, uid,
-                                          why, exit_code,
-                                          infop, ru);
+       if (unlikely(wo->wo_flags & WNOWAIT))
+               return wait_noreap_copyout(wo, p, pid, uid, why, exit_code);
 
-       retval = ru ? getrusage(p, RUSAGE_BOTH, ru) : 0;
-       if (!retval && stat_addr)
-               retval = put_user((exit_code << 8) | 0x7f, stat_addr);
+       retval = wo->wo_rusage
+               ? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
+       if (!retval && wo->wo_stat)
+               retval = put_user((exit_code << 8) | 0x7f, wo->wo_stat);
+
+       infop = wo->wo_info;
        if (!retval && infop)
                retval = put_user(SIGCHLD, &infop->si_signo);
        if (!retval && infop)
@@ -1404,15 +1433,13 @@ unlock_sig:
  * the lock and this task is uninteresting.  If we return nonzero, we have
  * released the lock and the system call should return.
  */
-static int wait_task_continued(struct task_struct *p, int options,
-                              struct siginfo __user *infop,
-                              int __user *stat_addr, struct rusage __user *ru)
+static int wait_task_continued(struct wait_opts *wo, struct task_struct *p)
 {
        int retval;
        pid_t pid;
        uid_t uid;
 
-       if (!unlikely(options & WCONTINUED))
+       if (!unlikely(wo->wo_flags & WCONTINUED))
                return 0;
 
        if (!(p->signal->flags & SIGNAL_STOP_CONTINUED))
@@ -1424,7 +1451,7 @@ static int wait_task_continued(struct task_struct *p, int options,
                spin_unlock_irq(&p->sighand->siglock);
                return 0;
        }
-       if (!unlikely(options & WNOWAIT))
+       if (!unlikely(wo->wo_flags & WNOWAIT))
                p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
        uid = __task_cred(p)->uid;
        spin_unlock_irq(&p->sighand->siglock);
@@ -1433,17 +1460,17 @@ static int wait_task_continued(struct task_struct *p, int options,
        get_task_struct(p);
        read_unlock(&tasklist_lock);
 
-       if (!infop) {
-               retval = ru ? getrusage(p, RUSAGE_BOTH, ru) : 0;
+       if (!wo->wo_info) {
+               retval = wo->wo_rusage
+                       ? getrusage(p, RUSAGE_BOTH, wo->wo_rusage) : 0;
                put_task_struct(p);
-               if (!retval && stat_addr)
-                       retval = put_user(0xffff, stat_addr);
+               if (!retval && wo->wo_stat)
+                       retval = put_user(0xffff, wo->wo_stat);
                if (!retval)
                        retval = pid;
        } else {
-               retval = wait_noreap_copyout(p, pid, uid,
-                                            CLD_CONTINUED, SIGCONT,
-                                            infop, ru);
+               retval = wait_noreap_copyout(wo, p, pid, uid,
+                                            CLD_CONTINUED, SIGCONT);
                BUG_ON(retval == 0);
        }
 
@@ -1453,22 +1480,20 @@ static int wait_task_continued(struct task_struct *p, int options,
 /*
  * Consider @p for a wait by @parent.
  *
- * -ECHILD should be in *@notask_error before the first call.
+ * -ECHILD should be in ->notask_error before the first call.
  * Returns nonzero for a final return, when we have unlocked tasklist_lock.
  * Returns zero if the search for a child should continue;
- * then *@notask_error is 0 if @p is an eligible child,
+ * then ->notask_error is 0 if @p is an eligible child,
  * or another error from security_task_wait(), or still -ECHILD.
  */
-static int wait_consider_task(struct task_struct *parent, int ptrace,
-                             struct task_struct *p, int *notask_error,
-                             enum pid_type type, struct pid *pid, int options,
-                             struct siginfo __user *infop,
-                             int __user *stat_addr, struct rusage __user *ru)
+static int wait_consider_task(struct wait_opts *wo, int ptrace,
+                               struct task_struct *p)
 {
-       int ret = eligible_child(type, pid, options, p);
+       int ret = eligible_child(wo, p);
        if (!ret)
                return ret;
 
+       ret = security_task_wait(p);
        if (unlikely(ret < 0)) {
                /*
                 * If we have not yet seen any eligible child,
@@ -1477,8 +1502,8 @@ static int wait_consider_task(struct task_struct *parent, int ptrace,
                 * to look for security policy problems, rather
                 * than for mysterious wait bugs.
                 */
-               if (*notask_error)
-                       *notask_error = ret;
+               if (wo->notask_error)
+                       wo->notask_error = ret;
                return 0;
        }
 
@@ -1487,7 +1512,7 @@ static int wait_consider_task(struct task_struct *parent, int ptrace,
                 * This child is hidden by ptrace.
                 * We aren't allowed to see it now, but eventually we will.
                 */
-               *notask_error = 0;
+               wo->notask_error = 0;
                return 0;
        }
 
@@ -1498,34 +1523,30 @@ static int wait_consider_task(struct task_struct *parent, int ptrace,
         * We don't reap group leaders with subthreads.
         */
        if (p->exit_state == EXIT_ZOMBIE && !delay_group_leader(p))
-               return wait_task_zombie(p, options, infop, stat_addr, ru);
+               return wait_task_zombie(wo, p);
 
        /*
         * It's stopped or running now, so it might
         * later continue, exit, or stop again.
         */
-       *notask_error = 0;
+       wo->notask_error = 0;
 
        if (task_stopped_code(p, ptrace))
-               return wait_task_stopped(ptrace, p, options,
-                                        infop, stat_addr, ru);
+               return wait_task_stopped(wo, ptrace, p);
 
-       return wait_task_continued(p, options, infop, stat_addr, ru);
+       return wait_task_continued(wo, p);
 }
 
 /*
  * Do the work of do_wait() for one thread in the group, @tsk.
  *
- * -ECHILD should be in *@notask_error before the first call.
+ * -ECHILD should be in ->notask_error before the first call.
  * Returns nonzero for a final return, when we have unlocked tasklist_lock.
  * Returns zero if the search for a child should continue; then
- * *@notask_error is 0 if there were any eligible children,
+ * ->notask_error is 0 if there were any eligible children,
  * or another error from security_task_wait(), or still -ECHILD.
  */
-static int do_wait_thread(struct task_struct *tsk, int *notask_error,
-                         enum pid_type type, struct pid *pid, int options,
-                         struct siginfo __user *infop, int __user *stat_addr,
-                         struct rusage __user *ru)
+static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
 {
        struct task_struct *p;
 
@@ -1534,9 +1555,7 @@ static int do_wait_thread(struct task_struct *tsk, int *notask_error,
                 * Do not consider detached threads.
                 */
                if (!task_detached(p)) {
-                       int ret = wait_consider_task(tsk, 0, p, notask_error,
-                                                    type, pid, options,
-                                                    infop, stat_addr, ru);
+                       int ret = wait_consider_task(wo, 0, p);
                        if (ret)
                                return ret;
                }
@@ -1545,22 +1564,12 @@ static int do_wait_thread(struct task_struct *tsk, int *notask_error,
        return 0;
 }
 
-static int ptrace_do_wait(struct task_struct *tsk, int *notask_error,
-                         enum pid_type type, struct pid *pid, int options,
-                         struct siginfo __user *infop, int __user *stat_addr,
-                         struct rusage __user *ru)
+static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk)
 {
        struct task_struct *p;
 
-       /*
-        * Traditionally we see ptrace'd stopped tasks regardless of options.
-        */
-       options |= WUNTRACED;
-
        list_for_each_entry(p, &tsk->ptraced, ptrace_entry) {
-               int ret = wait_consider_task(tsk, 1, p, notask_error,
-                                            type, pid, options,
-                                            infop, stat_addr, ru);
+               int ret = wait_consider_task(wo, 1, p);
                if (ret)
                        return ret;
        }
@@ -1568,93 +1577,86 @@ static int ptrace_do_wait(struct task_struct *tsk, int *notask_error,
        return 0;
 }
 
-static long do_wait(enum pid_type type, struct pid *pid, int options,
-                   struct siginfo __user *infop, int __user *stat_addr,
-                   struct rusage __user *ru)
+static int child_wait_callback(wait_queue_t *wait, unsigned mode,
+                               int sync, void *key)
+{
+       struct wait_opts *wo = container_of(wait, struct wait_opts,
+                                               child_wait);
+       struct task_struct *p = key;
+
+       if (!eligible_pid(wo, p))
+               return 0;
+
+       if ((wo->wo_flags & __WNOTHREAD) && wait->private != p->parent)
+               return 0;
+
+       return default_wake_function(wait, mode, sync, key);
+}
+
+void __wake_up_parent(struct task_struct *p, struct task_struct *parent)
+{
+       __wake_up_sync_key(&parent->signal->wait_chldexit,
+                               TASK_INTERRUPTIBLE, 1, p);
+}
+
+static long do_wait(struct wait_opts *wo)
 {
-       DECLARE_WAITQUEUE(wait, current);
        struct task_struct *tsk;
        int retval;
 
-       trace_sched_process_wait(pid);
+       trace_sched_process_wait(wo->wo_pid);
 
-       add_wait_queue(&current->signal->wait_chldexit,&wait);
+       init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
+       wo->child_wait.private = current;
+       add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
 repeat:
        /*
         * If there is nothing that can match our critiera just get out.
-        * We will clear @retval to zero if we see any child that might later
-        * match our criteria, even if we are not able to reap it yet.
+        * We will clear ->notask_error to zero if we see any child that
+        * might later match our criteria, even if we are not able to reap
+        * it yet.
         */
-       retval = -ECHILD;
-       if ((type < PIDTYPE_MAX) && (!pid || hlist_empty(&pid->tasks[type])))
-               goto end;
+       wo->notask_error = -ECHILD;
+       if ((wo->wo_type < PIDTYPE_MAX) &&
+          (!wo->wo_pid || hlist_empty(&wo->wo_pid->tasks[wo->wo_type])))
+               goto notask;
 
-       current->state = TASK_INTERRUPTIBLE;
+       set_current_state(TASK_INTERRUPTIBLE);
        read_lock(&tasklist_lock);
        tsk = current;
        do {
-               int tsk_result = do_wait_thread(tsk, &retval,
-                                               type, pid, options,
-                                               infop, stat_addr, ru);
-               if (!tsk_result)
-                       tsk_result = ptrace_do_wait(tsk, &retval,
-                                                   type, pid, options,
-                                                   infop, stat_addr, ru);
-               if (tsk_result) {
-                       /*
-                        * tasklist_lock is unlocked and we have a final result.
-                        */
-                       retval = tsk_result;
+               retval = do_wait_thread(wo, tsk);
+               if (retval)
+                       goto end;
+
+               retval = ptrace_do_wait(wo, tsk);
+               if (retval)
                        goto end;
-               }
 
-               if (options & __WNOTHREAD)
+               if (wo->wo_flags & __WNOTHREAD)
                        break;
-               tsk = next_thread(tsk);
-               BUG_ON(tsk->signal != current->signal);
-       } while (tsk != current);
+       } while_each_thread(current, tsk);
        read_unlock(&tasklist_lock);
 
-       if (!retval && !(options & WNOHANG)) {
+notask:
+       retval = wo->notask_error;
+       if (!retval && !(wo->wo_flags & WNOHANG)) {
                retval = -ERESTARTSYS;
                if (!signal_pending(current)) {
                        schedule();
                        goto repeat;
                }
        }
-
 end:
-       current->state = TASK_RUNNING;
-       remove_wait_queue(&current->signal->wait_chldexit,&wait);
-       if (infop) {
-               if (retval > 0)
-                       retval = 0;
-               else {
-                       /*
-                        * For a WNOHANG return, clear out all the fields
-                        * we would set so the user can easily tell the
-                        * difference.
-                        */
-                       if (!retval)
-                               retval = put_user(0, &infop->si_signo);
-                       if (!retval)
-                               retval = put_user(0, &infop->si_errno);
-                       if (!retval)
-                               retval = put_user(0, &infop->si_code);
-                       if (!retval)
-                               retval = put_user(0, &infop->si_pid);
-                       if (!retval)
-                               retval = put_user(0, &infop->si_uid);
-                       if (!retval)
-                               retval = put_user(0, &infop->si_status);
-               }
-       }
+       __set_current_state(TASK_RUNNING);
+       remove_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
        return retval;
 }
 
 SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
                infop, int, options, struct rusage __user *, ru)
 {
+       struct wait_opts wo;
        struct pid *pid = NULL;
        enum pid_type type;
        long ret;
@@ -1684,7 +1686,37 @@ SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
 
        if (type < PIDTYPE_MAX)
                pid = find_get_pid(upid);
-       ret = do_wait(type, pid, options, infop, NULL, ru);
+
+       wo.wo_type      = type;
+       wo.wo_pid       = pid;
+       wo.wo_flags     = options;
+       wo.wo_info      = infop;
+       wo.wo_stat      = NULL;
+       wo.wo_rusage    = ru;
+       ret = do_wait(&wo);
+
+       if (ret > 0) {
+               ret = 0;
+       } else if (infop) {
+               /*
+                * For a WNOHANG return, clear out all the fields
+                * we would set so the user can easily tell the
+                * difference.
+                */
+               if (!ret)
+                       ret = put_user(0, &infop->si_signo);
+               if (!ret)
+                       ret = put_user(0, &infop->si_errno);
+               if (!ret)
+                       ret = put_user(0, &infop->si_code);
+               if (!ret)
+                       ret = put_user(0, &infop->si_pid);
+               if (!ret)
+                       ret = put_user(0, &infop->si_uid);
+               if (!ret)
+                       ret = put_user(0, &infop->si_status);
+       }
+
        put_pid(pid);
 
        /* avoid REGPARM breakage on x86: */
@@ -1695,6 +1727,7 @@ SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
 SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
                int, options, struct rusage __user *, ru)
 {
+       struct wait_opts wo;
        struct pid *pid = NULL;
        enum pid_type type;
        long ret;
@@ -1716,7 +1749,13 @@ SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
                pid = find_get_pid(upid);
        }
 
-       ret = do_wait(type, pid, options | WEXITED, NULL, stat_addr, ru);
+       wo.wo_type      = type;
+       wo.wo_pid       = pid;
+       wo.wo_flags     = options | WEXITED;
+       wo.wo_info      = NULL;
+       wo.wo_stat      = stat_addr;
+       wo.wo_rusage    = ru;
+       ret = do_wait(&wo);
        put_pid(pid);
 
        /* avoid REGPARM breakage on x86: */