* Copyright (C) 1992 Krishna Balasubramanian
* Copyright (C) 1995 Eric Schenk, Bruno Haible
*
- * IMPLEMENTATION NOTES ON CODE REWRITE (Eric Schenk, January 1995):
- * This code underwent a massive rewrite in order to solve some problems
- * with the original code. In particular the original code failed to
- * wake up processes that were waiting for semval to go to 0 if the
- * value went to 0 and was then incremented rapidly enough. In solving
- * this problem I have also modified the implementation so that it
- * processes pending operations in a FIFO manner, thus give a guarantee
- * that processes waiting for a lock on the semaphore won't starve
- * unless another locking process fails to unlock.
- * In addition the following two changes in behavior have been introduced:
- * - The original implementation of semop returned the value
- * last semaphore element examined on success. This does not
- * match the manual page specifications, and effectively
- * allows the user to read the semaphore even if they do not
- * have read permissions. The implementation now returns 0
- * on success as stated in the manual page.
- * - There is some confusion over whether the set of undo adjustments
- * to be performed at exit should be done in an atomic manner.
- * That is, if we are attempting to decrement the semval should we queue
- * up and wait until we can do so legally?
- * The original implementation attempted to do this.
- * The current implementation does not do so. This is because I don't
- * think it is the right thing (TM) to do, and because I couldn't
- * see a clean way to get the old behavior with the new design.
- * The POSIX standard and SVID should be consulted to determine
- * what behavior is mandated.
- *
- * Further notes on refinement (Christoph Rohland, December 1998):
- * - The POSIX standard says, that the undo adjustments simply should
- * redo. So the current implementation is o.K.
- * - The previous code had two flaws:
- * 1) It actively gave the semaphore to the next waiting process
- * sleeping on the semaphore. Since this process did not have the
- * cpu this led to many unnecessary context switches and bad
- * performance. Now we only check which process should be able to
- * get the semaphore and if this process wants to reduce some
- * semaphore value we simply wake it up without doing the
- * operation. So it has to try to get it later. Thus e.g. the
- * running process may reacquire the semaphore during the current
- * time slice. If it only waits for zero or increases the semaphore,
- * we do the operation in advance and wake it up.
- * 2) It did not wake up all zero waiting processes. We try to do
- * better but only get the semops right which only wait for zero or
- * increase. If there are decrement operations in the operations
- * array we do the same as before.
- *
- * With the incarnation of O(1) scheduler, it becomes unnecessary to perform
- * check/retry algorithm for waking up blocked processes as the new scheduler
- * is better at handling thread switch than the old one.
- *
* /proc/sysvipc/sem support (c) 1999 Dragos Acostachioaie <dragos@iname.com>
*
* SMP-threaded, sysctl's added
* (c) 1999 Manfred Spraul <manfred@colorfullife.com>
* Enforced range limit on SEM_UNDO
- * (c) 2001 Red Hat Inc <alan@redhat.com>
+ * (c) 2001 Red Hat Inc
* Lockless wakeup
* (c) 2003 Manfred Spraul <manfred@colorfullife.com>
+ * Further wakeup optimizations, documentation
+ * (c) 2010 Manfred Spraul <manfred@colorfullife.com>
*
* support for audit of ipc object properties and permission changes
* Dustin Kirkland <dustin.kirkland@us.ibm.com>
* namespaces support
* OpenVZ, SWsoft Inc.
* Pavel Emelianov <xemul@openvz.org>
+ *
+ * Implementation notes: (May 2010)
+ * This file implements System V semaphores.
+ *
+ * User space visible behavior:
+ * - FIFO ordering for semop() operations (just FIFO, not starvation
+ * protection)
+ * - multiple semaphore operations that alter the same semaphore in
+ * one semop() are handled.
+ * - sem_ctime (time of last semctl()) is updated in the IPC_SET, SETVAL and
+ * SETALL calls.
+ * - two Linux specific semctl() commands: SEM_STAT, SEM_INFO.
+ * - undo adjustments at process exit are limited to 0..SEMVMX.
+ * - namespace are supported.
+ * - SEMMSL, SEMMNS, SEMOPM and SEMMNI can be configured at runtine by writing
+ * to /proc/sys/kernel/sem.
+ * - statistics about the usage are reported in /proc/sysvipc/sem.
+ *
+ * Internals:
+ * - scalability:
+ * - all global variables are read-mostly.
+ * - semop() calls and semctl(RMID) are synchronized by RCU.
+ * - most operations do write operations (actually: spin_lock calls) to
+ * the per-semaphore array structure.
+ * Thus: Perfect SMP scaling between independent semaphore arrays.
+ * If multiple semaphores in one array are used, then cache line
+ * trashing on the semaphore array spinlock will limit the scaling.
+ * - semncnt and semzcnt are calculated on demand in count_semncnt() and
+ * count_semzcnt()
+ * - the task that performs a successful semop() scans the list of all
+ * sleeping tasks and completes any pending operations that can be fulfilled.
+ * Semaphores are actively given to waiting tasks (necessary for FIFO).
+ * (see update_queue())
+ * - To improve the scalability, the actual wake-up calls are performed after
+ * dropping all locks. (see wake_up_sem_queue_prepare(),
+ * wake_up_sem_queue_do())
+ * - All work is done by the waker, the woken up task does not have to do
+ * anything - not even acquiring a lock or dropping a refcount.
+ * - A woken up task may not even touch the semaphore array anymore, it may
+ * have been destroyed already by a semctl(RMID).
+ * - The synchronizations between wake-ups due to a timeout/signal and a
+ * wake-up due to a completed semaphore operation is achieved by using an
+ * intermediate state (IN_WAKEUP).
+ * - UNDO values are stored in an array (one per process and per
+ * semaphore array, lazily allocated). For backwards compatibility, multiple
+ * modes for the UNDO variables are supported (per process, per thread)
+ * (see copy_semundo, CLONE_SYSVSEM)
+ * - There are two lists of the pending operations: a per-array list
+ * and per-semaphore list (stored in the array). This allows to achieve FIFO
+ * ordering without always scanning all pending operations.
+ * The worst-case behavior is nevertheless O(N^2) for N wakeups.
*/
#include <linux/slab.h>
#include <asm/uaccess.h>
#include "util.h"
+/* One semaphore structure for each semaphore in the system. */
+struct sem {
+ int semval; /* current value */
+ int sempid; /* pid of last operation */
+ struct list_head sem_pending; /* pending single-sop operations */
+};
+
+/* One queue for each sleeping process in the system. */
+struct sem_queue {
+ struct list_head simple_list; /* queue of pending operations */
+ struct list_head list; /* queue of pending operations */
+ struct task_struct *sleeper; /* this process */
+ struct sem_undo *undo; /* undo structure */
+ int pid; /* process id of requesting process */
+ int status; /* completion status of operation */
+ struct sembuf *sops; /* array of pending operations */
+ int nsops; /* number of operations */
+ int alter; /* does *sops alter the array? */
+};
+
+/* Each task has a list of undo requests. They are executed automatically
+ * when the process exits.
+ */
+struct sem_undo {
+ struct list_head list_proc; /* per-process list: *
+ * all undos from one process
+ * rcu protected */
+ struct rcu_head rcu; /* rcu struct for sem_undo */
+ struct sem_undo_list *ulp; /* back ptr to sem_undo_list */
+ struct list_head list_id; /* per semaphore array list:
+ * all undos for one array */
+ int semid; /* semaphore set identifier */
+ short *semadj; /* array of adjustments */
+ /* one per semaphore */
+};
+
+/* sem_undo_list controls shared access to the list of sem_undo structures
+ * that may be shared among all a CLONE_SYSVSEM task group.
+ */
+struct sem_undo_list {
+ atomic_t refcnt;
+ spinlock_t lock;
+ struct list_head list_proc;
+};
+
+
#define sem_ids(ns) ((ns)->ids[IPC_SEM_IDS])
#define sem_unlock(sma) ipc_unlock(&(sma)->sem_perm)
void sem_exit_ns(struct ipc_namespace *ns)
{
free_ipcs(ns, &sem_ids(ns), freeary);
+ idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);
}
#endif
IPC_SEM_IDS, sysvipc_sem_proc_show);
}
-/*
- * This routine is called in the paths where the rw_mutex is held to protect
- * access to the idr tree.
- */
-static inline struct sem_array *sem_lock_check_down(struct ipc_namespace *ns,
- int id)
-{
- struct kern_ipc_perm *ipcp = ipc_lock_check_down(&sem_ids(ns), id);
-
- if (IS_ERR(ipcp))
- return (struct sem_array *)ipcp;
-
- return container_of(ipcp, struct sem_array, sem_perm);
-}
-
/*
* sem_lock_(check_) routines are called in the paths where the rw_mutex
* is not held.
key_t key = params->key;
int nsems = params->u.nsems;
int semflg = params->flg;
+ int i;
if (!nsems)
return -EINVAL;
ns->used_sems += nsems;
sma->sem_base = (struct sem *) &sma[1];
- /* sma->sem_pending = NULL; */
- sma->sem_pending_last = &sma->sem_pending;
- /* sma->undo = NULL; */
+
+ for (i = 0; i < nsems; i++)
+ INIT_LIST_HEAD(&sma->sem_base[i].sem_pending);
+
+ sma->complex_count = 0;
+ INIT_LIST_HEAD(&sma->sem_pending);
+ INIT_LIST_HEAD(&sma->list_id);
sma->sem_nsems = nsems;
sma->sem_ctime = get_seconds();
sem_unlock(sma);
return 0;
}
-asmlinkage long sys_semget(key_t key, int nsems, int semflg)
+SYSCALL_DEFINE3(semget, key_t, key, int, nsems, int, semflg)
{
struct ipc_namespace *ns;
struct ipc_ops sem_ops;
return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params);
}
-/* Manage the doubly linked list sma->sem_pending as a FIFO:
- * insert new queue elements at the tail sma->sem_pending_last.
- */
-static inline void append_to_queue (struct sem_array * sma,
- struct sem_queue * q)
-{
- *(q->prev = sma->sem_pending_last) = q;
- *(sma->sem_pending_last = &q->next) = NULL;
-}
-
-static inline void prepend_to_queue (struct sem_array * sma,
- struct sem_queue * q)
-{
- q->next = sma->sem_pending;
- *(q->prev = &sma->sem_pending) = q;
- if (q->next)
- q->next->prev = &q->next;
- else /* sma->sem_pending_last == &sma->sem_pending */
- sma->sem_pending_last = &q->next;
-}
-
-static inline void remove_from_queue (struct sem_array * sma,
- struct sem_queue * q)
-{
- *(q->prev) = q->next;
- if (q->next)
- q->next->prev = q->prev;
- else /* sma->sem_pending_last == &q->next */
- sma->sem_pending_last = q->prev;
- q->prev = NULL; /* mark as removed */
-}
-
/*
* Determine whether a sequence of semaphore operations would succeed
* all at once. Return 0 if yes, 1 if need to sleep, else return error code.
sop--;
}
- sma->sem_otime = get_seconds();
return 0;
out_of_range:
return result;
}
-/* Go through the pending queue for the indicated semaphore
- * looking for tasks that can be completed.
+/** wake_up_sem_queue_prepare(q, error): Prepare wake-up
+ * @q: queue entry that must be signaled
+ * @error: Error value for the signal
+ *
+ * Prepare the wake-up of the queue entry q.
*/
-static void update_queue (struct sem_array * sma)
+static void wake_up_sem_queue_prepare(struct list_head *pt,
+ struct sem_queue *q, int error)
{
- int error;
- struct sem_queue * q;
+ if (list_empty(pt)) {
+ /*
+ * Hold preempt off so that we don't get preempted and have the
+ * wakee busy-wait until we're scheduled back on.
+ */
+ preempt_disable();
+ }
+ q->status = IN_WAKEUP;
+ q->pid = error;
+
+ list_add_tail(&q->simple_list, pt);
+}
+
+/**
+ * wake_up_sem_queue_do(pt) - do the actual wake-up
+ * @pt: list of tasks to be woken up
+ *
+ * Do the actual wake-up.
+ * The function is called without any locks held, thus the semaphore array
+ * could be destroyed already and the tasks can disappear as soon as the
+ * status is set to the actual return code.
+ */
+static void wake_up_sem_queue_do(struct list_head *pt)
+{
+ struct sem_queue *q, *t;
+ int did_something;
+
+ did_something = !list_empty(pt);
+ list_for_each_entry_safe(q, t, pt, simple_list) {
+ wake_up_process(q->sleeper);
+ /* q can disappear immediately after writing q->status. */
+ smp_wmb();
+ q->status = q->pid;
+ }
+ if (did_something)
+ preempt_enable();
+}
+
+static void unlink_queue(struct sem_array *sma, struct sem_queue *q)
+{
+ list_del(&q->list);
+ if (q->nsops == 1)
+ list_del(&q->simple_list);
+ else
+ sma->complex_count--;
+}
+
+/** check_restart(sma, q)
+ * @sma: semaphore array
+ * @q: the operation that just completed
+ *
+ * update_queue is O(N^2) when it restarts scanning the whole queue of
+ * waiting operations. Therefore this function checks if the restart is
+ * really necessary. It is called after a previously waiting operation
+ * was completed.
+ */
+static int check_restart(struct sem_array *sma, struct sem_queue *q)
+{
+ struct sem *curr;
+ struct sem_queue *h;
+
+ /* if the operation didn't modify the array, then no restart */
+ if (q->alter == 0)
+ return 0;
+
+ /* pending complex operations are too difficult to analyse */
+ if (sma->complex_count)
+ return 1;
+
+ /* we were a sleeping complex operation. Too difficult */
+ if (q->nsops > 1)
+ return 1;
+
+ curr = sma->sem_base + q->sops[0].sem_num;
+
+ /* No-one waits on this queue */
+ if (list_empty(&curr->sem_pending))
+ return 0;
+
+ /* the new semaphore value */
+ if (curr->semval) {
+ /* It is impossible that someone waits for the new value:
+ * - q is a previously sleeping simple operation that
+ * altered the array. It must be a decrement, because
+ * simple increments never sleep.
+ * - The value is not 0, thus wait-for-zero won't proceed.
+ * - If there are older (higher priority) decrements
+ * in the queue, then they have observed the original
+ * semval value and couldn't proceed. The operation
+ * decremented to value - thus they won't proceed either.
+ */
+ BUG_ON(q->sops[0].sem_op >= 0);
+ return 0;
+ }
+ /*
+ * semval is 0. Check if there are wait-for-zero semops.
+ * They must be the first entries in the per-semaphore simple queue
+ */
+ h = list_first_entry(&curr->sem_pending, struct sem_queue, simple_list);
+ BUG_ON(h->nsops != 1);
+ BUG_ON(h->sops[0].sem_num != q->sops[0].sem_num);
+
+ /* Yes, there is a wait-for-zero semop. Restart */
+ if (h->sops[0].sem_op == 0)
+ return 1;
+
+ /* Again - no-one is waiting for the new value. */
+ return 0;
+}
+
+
+/**
+ * update_queue(sma, semnum): Look for tasks that can be completed.
+ * @sma: semaphore array.
+ * @semnum: semaphore that was modified.
+ * @pt: list head for the tasks that must be woken up.
+ *
+ * update_queue must be called after a semaphore in a semaphore array
+ * was modified. If multiple semaphore were modified, then @semnum
+ * must be set to -1.
+ * The tasks that must be woken up are added to @pt. The return code
+ * is stored in q->pid.
+ * The function return 1 if at least one semop was completed successfully.
+ */
+static int update_queue(struct sem_array *sma, int semnum, struct list_head *pt)
+{
+ struct sem_queue *q;
+ struct list_head *walk;
+ struct list_head *pending_list;
+ int offset;
+ int semop_completed = 0;
+
+ /* if there are complex operations around, then knowing the semaphore
+ * that was modified doesn't help us. Assume that multiple semaphores
+ * were modified.
+ */
+ if (sma->complex_count)
+ semnum = -1;
+
+ if (semnum == -1) {
+ pending_list = &sma->sem_pending;
+ offset = offsetof(struct sem_queue, list);
+ } else {
+ pending_list = &sma->sem_base[semnum].sem_pending;
+ offset = offsetof(struct sem_queue, simple_list);
+ }
+
+again:
+ walk = pending_list->next;
+ while (walk != pending_list) {
+ int error, restart;
+
+ q = (struct sem_queue *)((char *)walk - offset);
+ walk = walk->next;
+
+ /* If we are scanning the single sop, per-semaphore list of
+ * one semaphore and that semaphore is 0, then it is not
+ * necessary to scan the "alter" entries: simple increments
+ * that affect only one entry succeed immediately and cannot
+ * be in the per semaphore pending queue, and decrements
+ * cannot be successful if the value is already 0.
+ */
+ if (semnum != -1 && sma->sem_base[semnum].semval == 0 &&
+ q->alter)
+ break;
- q = sma->sem_pending;
- while(q) {
error = try_atomic_semop(sma, q->sops, q->nsops,
q->undo, q->pid);
/* Does q->sleeper still need to sleep? */
- if (error <= 0) {
- struct sem_queue *n;
- remove_from_queue(sma,q);
- q->status = IN_WAKEUP;
- /*
- * Continue scanning. The next operation
- * that must be checked depends on the type of the
- * completed operation:
- * - if the operation modified the array, then
- * restart from the head of the queue and
- * check for threads that might be waiting
- * for semaphore values to become 0.
- * - if the operation didn't modify the array,
- * then just continue.
- */
- if (q->alter)
- n = sma->sem_pending;
- else
- n = q->next;
- wake_up_process(q->sleeper);
- /* hands-off: q will disappear immediately after
- * writing q->status.
- */
- smp_wmb();
- q->status = error;
- q = n;
+ if (error > 0)
+ continue;
+
+ unlink_queue(sma, q);
+
+ if (error) {
+ restart = 0;
} else {
- q = q->next;
+ semop_completed = 1;
+ restart = check_restart(sma, q);
}
+
+ wake_up_sem_queue_prepare(pt, q, error);
+ if (restart)
+ goto again;
+ }
+ return semop_completed;
+}
+
+/**
+ * do_smart_update(sma, sops, nsops, otime, pt) - optimized update_queue
+ * @sma: semaphore array
+ * @sops: operations that were performed
+ * @nsops: number of operations
+ * @otime: force setting otime
+ * @pt: list head of the tasks that must be woken up.
+ *
+ * do_smart_update() does the required called to update_queue, based on the
+ * actual changes that were performed on the semaphore array.
+ * Note that the function does not do the actual wake-up: the caller is
+ * responsible for calling wake_up_sem_queue_do(@pt).
+ * It is safe to perform this call after dropping all locks.
+ */
+static void do_smart_update(struct sem_array *sma, struct sembuf *sops, int nsops,
+ int otime, struct list_head *pt)
+{
+ int i;
+
+ if (sma->complex_count || sops == NULL) {
+ if (update_queue(sma, -1, pt))
+ otime = 1;
+ goto done;
+ }
+
+ for (i = 0; i < nsops; i++) {
+ if (sops[i].sem_op > 0 ||
+ (sops[i].sem_op < 0 &&
+ sma->sem_base[sops[i].sem_num].semval == 0))
+ if (update_queue(sma, sops[i].sem_num, pt))
+ otime = 1;
}
+done:
+ if (otime)
+ sma->sem_otime = get_seconds();
}
+
/* The following counts are associated to each semaphore:
* semncnt number of tasks waiting on semval being nonzero
* semzcnt number of tasks waiting on semval being zero
struct sem_queue * q;
semncnt = 0;
- for (q = sma->sem_pending; q; q = q->next) {
+ list_for_each_entry(q, &sma->sem_pending, list) {
struct sembuf * sops = q->sops;
int nsops = q->nsops;
int i;
}
return semncnt;
}
+
static int count_semzcnt (struct sem_array * sma, ushort semnum)
{
int semzcnt;
struct sem_queue * q;
semzcnt = 0;
- for (q = sma->sem_pending; q; q = q->next) {
+ list_for_each_entry(q, &sma->sem_pending, list) {
struct sembuf * sops = q->sops;
int nsops = q->nsops;
int i;
*/
static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
{
- struct sem_undo *un;
- struct sem_queue *q;
+ struct sem_undo *un, *tu;
+ struct sem_queue *q, *tq;
struct sem_array *sma = container_of(ipcp, struct sem_array, sem_perm);
+ struct list_head tasks;
- /* Invalidate the existing undo structures for this semaphore set.
- * (They will be freed without any further action in exit_sem()
- * or during the next semop.)
- */
- for (un = sma->undo; un; un = un->id_next)
+ /* Free the existing undo structures for this semaphore set. */
+ assert_spin_locked(&sma->sem_perm.lock);
+ list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {
+ list_del(&un->list_id);
+ spin_lock(&un->ulp->lock);
un->semid = -1;
+ list_del_rcu(&un->list_proc);
+ spin_unlock(&un->ulp->lock);
+ kfree_rcu(un, rcu);
+ }
/* Wake up all pending processes and let them fail with EIDRM. */
- q = sma->sem_pending;
- while(q) {
- struct sem_queue *n;
- /* lazy remove_from_queue: we are killing the whole queue */
- q->prev = NULL;
- n = q->next;
- q->status = IN_WAKEUP;
- wake_up_process(q->sleeper); /* doesn't sleep */
- smp_wmb();
- q->status = -EIDRM; /* hands-off q */
- q = n;
+ INIT_LIST_HEAD(&tasks);
+ list_for_each_entry_safe(q, tq, &sma->sem_pending, list) {
+ unlink_queue(sma, q);
+ wake_up_sem_queue_prepare(&tasks, q, -EIDRM);
}
/* Remove the semaphore set from the IDR */
sem_rmid(ns, sma);
sem_unlock(sma);
+ wake_up_sem_queue_do(&tasks);
ns->used_sems -= sma->sem_nsems;
security_sem_free(sma);
ipc_rcu_putref(sma);
{
struct semid_ds out;
+ memset(&out, 0, sizeof(out));
+
ipc64_perm_to_ipc_perm(&in->sem_perm, &out.sem_perm);
out.sem_otime = in->sem_otime;
static int semctl_nolock(struct ipc_namespace *ns, int semid,
int cmd, int version, union semun arg)
{
- int err = -EINVAL;
+ int err;
struct sem_array *sma;
switch(cmd) {
}
err = -EACCES;
- if (ipcperms (&sma->sem_perm, S_IRUGO))
+ if (ipcperms(ns, &sma->sem_perm, S_IRUGO))
goto out_unlock;
err = security_sem_semctl(sma, cmd);
default:
return -EINVAL;
}
- return err;
out_unlock:
sem_unlock(sma);
return err;
ushort fast_sem_io[SEMMSL_FAST];
ushort* sem_io = fast_sem_io;
int nsems;
+ struct list_head tasks;
sma = sem_lock_check(ns, semid);
if (IS_ERR(sma))
return PTR_ERR(sma);
+ INIT_LIST_HEAD(&tasks);
nsems = sma->sem_nsems;
err = -EACCES;
- if (ipcperms (&sma->sem_perm, (cmd==SETVAL||cmd==SETALL)?S_IWUGO:S_IRUGO))
+ if (ipcperms(ns, &sma->sem_perm,
+ (cmd == SETVAL || cmd == SETALL) ? S_IWUGO : S_IRUGO))
goto out_unlock;
err = security_sem_semctl(sma, cmd);
for (i = 0; i < nsems; i++)
sma->sem_base[i].semval = sem_io[i];
- for (un = sma->undo; un; un = un->id_next)
+
+ assert_spin_locked(&sma->sem_perm.lock);
+ list_for_each_entry(un, &sma->list_id, list_id) {
for (i = 0; i < nsems; i++)
un->semadj[i] = 0;
+ }
sma->sem_ctime = get_seconds();
/* maybe some queued-up processes were waiting for this */
- update_queue(sma);
+ do_smart_update(sma, NULL, 0, 0, &tasks);
err = 0;
goto out_unlock;
}
{
int val = arg.val;
struct sem_undo *un;
+
err = -ERANGE;
if (val > SEMVMX || val < 0)
goto out_unlock;
- for (un = sma->undo; un; un = un->id_next)
+ assert_spin_locked(&sma->sem_perm.lock);
+ list_for_each_entry(un, &sma->list_id, list_id)
un->semadj[semnum] = 0;
+
curr->semval = val;
curr->sempid = task_tgid_vnr(current);
sma->sem_ctime = get_seconds();
/* maybe some queued-up processes were waiting for this */
- update_queue(sma);
+ do_smart_update(sma, NULL, 0, 0, &tasks);
err = 0;
goto out_unlock;
}
}
out_unlock:
sem_unlock(sma);
+ wake_up_sem_queue_do(&tasks);
+
out_free:
if(sem_io != fast_sem_io)
ipc_free(sem_io, sizeof(ushort)*nsems);
return err;
}
-struct sem_setbuf {
- uid_t uid;
- gid_t gid;
- mode_t mode;
-};
-
-static inline unsigned long copy_semid_from_user(struct sem_setbuf *out, void __user *buf, int version)
+static inline unsigned long
+copy_semid_from_user(struct semid64_ds *out, void __user *buf, int version)
{
switch(version) {
case IPC_64:
- {
- struct semid64_ds tbuf;
-
- if(copy_from_user(&tbuf, buf, sizeof(tbuf)))
+ if (copy_from_user(out, buf, sizeof(*out)))
return -EFAULT;
-
- out->uid = tbuf.sem_perm.uid;
- out->gid = tbuf.sem_perm.gid;
- out->mode = tbuf.sem_perm.mode;
-
return 0;
- }
case IPC_OLD:
{
struct semid_ds tbuf_old;
if(copy_from_user(&tbuf_old, buf, sizeof(tbuf_old)))
return -EFAULT;
- out->uid = tbuf_old.sem_perm.uid;
- out->gid = tbuf_old.sem_perm.gid;
- out->mode = tbuf_old.sem_perm.mode;
+ out->sem_perm.uid = tbuf_old.sem_perm.uid;
+ out->sem_perm.gid = tbuf_old.sem_perm.gid;
+ out->sem_perm.mode = tbuf_old.sem_perm.mode;
return 0;
}
{
struct sem_array *sma;
int err;
- struct sem_setbuf uninitialized_var(setbuf);
+ struct semid64_ds semid64;
struct kern_ipc_perm *ipcp;
if(cmd == IPC_SET) {
- if(copy_semid_from_user (&setbuf, arg.buf, version))
+ if (copy_semid_from_user(&semid64, arg.buf, version))
return -EFAULT;
}
- down_write(&sem_ids(ns).rw_mutex);
- sma = sem_lock_check_down(ns, semid);
- if (IS_ERR(sma)) {
- err = PTR_ERR(sma);
- goto out_up;
- }
-
- ipcp = &sma->sem_perm;
- err = audit_ipc_obj(ipcp);
- if (err)
- goto out_unlock;
+ ipcp = ipcctl_pre_down(ns, &sem_ids(ns), semid, cmd,
+ &semid64.sem_perm, 0);
+ if (IS_ERR(ipcp))
+ return PTR_ERR(ipcp);
- if (cmd == IPC_SET) {
- err = audit_ipc_set_perm(0, setbuf.uid, setbuf.gid, setbuf.mode);
- if (err)
- goto out_unlock;
- }
- if (current->euid != ipcp->cuid &&
- current->euid != ipcp->uid && !capable(CAP_SYS_ADMIN)) {
- err=-EPERM;
- goto out_unlock;
- }
+ sma = container_of(ipcp, struct sem_array, sem_perm);
err = security_sem_semctl(sma, cmd);
if (err)
freeary(ns, ipcp);
goto out_up;
case IPC_SET:
- ipcp->uid = setbuf.uid;
- ipcp->gid = setbuf.gid;
- ipcp->mode = (ipcp->mode & ~S_IRWXUGO)
- | (setbuf.mode & S_IRWXUGO);
+ err = ipc_update_perm(&semid64.sem_perm, ipcp);
+ if (err)
+ goto out_unlock;
sma->sem_ctime = get_seconds();
break;
default:
return err;
}
-asmlinkage long sys_semctl (int semid, int semnum, int cmd, union semun arg)
+SYSCALL_DEFINE(semctl)(int semid, int semnum, int cmd, union semun arg)
{
int err = -EINVAL;
int version;
return -EINVAL;
}
}
+#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
+asmlinkage long SyS_semctl(int semid, int semnum, int cmd, union semun arg)
+{
+ return SYSC_semctl((int) semid, (int) semnum, (int) cmd, arg);
+}
+SYSCALL_ALIAS(sys_semctl, SyS_semctl);
+#endif
/* If the task doesn't already have a undo_list, then allocate one
* here. We guarantee there is only one thread using this undo list,
return -ENOMEM;
spin_lock_init(&undo_list->lock);
atomic_set(&undo_list->refcnt, 1);
+ INIT_LIST_HEAD(&undo_list->list_proc);
+
current->sysvsem.undo_list = undo_list;
}
*undo_listp = undo_list;
return 0;
}
+static struct sem_undo *__lookup_undo(struct sem_undo_list *ulp, int semid)
+{
+ struct sem_undo *un;
+
+ list_for_each_entry_rcu(un, &ulp->list_proc, list_proc) {
+ if (un->semid == semid)
+ return un;
+ }
+ return NULL;
+}
+
static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)
{
- struct sem_undo **last, *un;
+ struct sem_undo *un;
- last = &ulp->proc_list;
- un = *last;
- while(un != NULL) {
- if(un->semid==semid)
- break;
- if(un->semid==-1) {
- *last=un->proc_next;
- kfree(un);
- } else {
- last=&un->proc_next;
- }
- un=*last;
+ assert_spin_locked(&ulp->lock);
+
+ un = __lookup_undo(ulp, semid);
+ if (un) {
+ list_del_rcu(&un->list_proc);
+ list_add_rcu(&un->list_proc, &ulp->list_proc);
}
return un;
}
-static struct sem_undo *find_undo(struct ipc_namespace *ns, int semid)
+/**
+ * find_alloc_undo - Lookup (and if not present create) undo array
+ * @ns: namespace
+ * @semid: semaphore array id
+ *
+ * The function looks up (and if not present creates) the undo structure.
+ * The size of the undo structure depends on the size of the semaphore
+ * array, thus the alloc path is not that straightforward.
+ * Lifetime-rules: sem_undo is rcu-protected, on success, the function
+ * performs a rcu_read_lock().
+ */
+static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
{
struct sem_array *sma;
struct sem_undo_list *ulp;
if (error)
return ERR_PTR(error);
+ rcu_read_lock();
spin_lock(&ulp->lock);
un = lookup_undo(ulp, semid);
spin_unlock(&ulp->lock);
if (likely(un!=NULL))
goto out;
+ rcu_read_unlock();
/* no undo structure around - allocate one. */
+ /* step 1: figure out the size of the semaphore array */
sma = sem_lock_check(ns, semid);
if (IS_ERR(sma))
- return ERR_PTR(PTR_ERR(sma));
+ return ERR_CAST(sma);
nsems = sma->sem_nsems;
sem_getref_and_unlock(sma);
+ /* step 2: allocate new undo structure */
new = kzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems, GFP_KERNEL);
if (!new) {
sem_putref(sma);
return ERR_PTR(-ENOMEM);
}
- new->semadj = (short *) &new[1];
- new->semid = semid;
- spin_lock(&ulp->lock);
- un = lookup_undo(ulp, semid);
- if (un) {
- spin_unlock(&ulp->lock);
- kfree(new);
- sem_putref(sma);
- goto out;
- }
+ /* step 3: Acquire the lock on semaphore array */
sem_lock_and_putref(sma);
if (sma->sem_perm.deleted) {
sem_unlock(sma);
- spin_unlock(&ulp->lock);
kfree(new);
un = ERR_PTR(-EIDRM);
goto out;
}
- new->proc_next = ulp->proc_list;
- ulp->proc_list = new;
- new->id_next = sma->undo;
- sma->undo = new;
- sem_unlock(sma);
+ spin_lock(&ulp->lock);
+
+ /*
+ * step 4: check for races: did someone else allocate the undo struct?
+ */
+ un = lookup_undo(ulp, semid);
+ if (un) {
+ kfree(new);
+ goto success;
+ }
+ /* step 5: initialize & link new undo structure */
+ new->semadj = (short *) &new[1];
+ new->ulp = ulp;
+ new->semid = semid;
+ assert_spin_locked(&ulp->lock);
+ list_add_rcu(&new->list_proc, &ulp->list_proc);
+ assert_spin_locked(&sma->sem_perm.lock);
+ list_add(&new->list_id, &sma->list_id);
un = new;
+
+success:
spin_unlock(&ulp->lock);
+ rcu_read_lock();
+ sem_unlock(sma);
out:
return un;
}
-asmlinkage long sys_semtimedop(int semid, struct sembuf __user *tsops,
- unsigned nsops, const struct timespec __user *timeout)
+
+/**
+ * get_queue_result - Retrieve the result code from sem_queue
+ * @q: Pointer to queue structure
+ *
+ * Retrieve the return code from the pending queue. If IN_WAKEUP is found in
+ * q->status, then we must loop until the value is replaced with the final
+ * value: This may happen if a task is woken up by an unrelated event (e.g.
+ * signal) and in parallel the task is woken up by another task because it got
+ * the requested semaphores.
+ *
+ * The function can be called with or without holding the semaphore spinlock.
+ */
+static int get_queue_result(struct sem_queue *q)
+{
+ int error;
+
+ error = q->status;
+ while (unlikely(error == IN_WAKEUP)) {
+ cpu_relax();
+ error = q->status;
+ }
+
+ return error;
+}
+
+
+SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
+ unsigned, nsops, const struct timespec __user *, timeout)
{
int error = -EINVAL;
struct sem_array *sma;
struct sem_queue queue;
unsigned long jiffies_left = 0;
struct ipc_namespace *ns;
+ struct list_head tasks;
ns = current->nsproxy->ipc_ns;
alter = 1;
}
-retry_undos:
if (undos) {
- un = find_undo(ns, semid);
+ un = find_alloc_undo(ns, semid);
if (IS_ERR(un)) {
error = PTR_ERR(un);
goto out_free;
} else
un = NULL;
+ INIT_LIST_HEAD(&tasks);
+
sma = sem_lock_check(ns, semid);
if (IS_ERR(sma)) {
+ if (un)
+ rcu_read_unlock();
error = PTR_ERR(sma);
goto out_free;
}
/*
- * semid identifiers are not unique - find_undo may have
+ * semid identifiers are not unique - find_alloc_undo may have
* allocated an undo structure, it was invalidated by an RMID
- * and now a new array with received the same id. Check and retry.
+ * and now a new array with received the same id. Check and fail.
+ * This case can be detected checking un->semid. The existence of
+ * "un" itself is guaranteed by rcu.
*/
- if (un && un->semid == -1) {
- sem_unlock(sma);
- goto retry_undos;
+ error = -EIDRM;
+ if (un) {
+ if (un->semid == -1) {
+ rcu_read_unlock();
+ goto out_unlock_free;
+ } else {
+ /*
+ * rcu lock can be released, "un" cannot disappear:
+ * - sem_lock is acquired, thus IPC_RMID is
+ * impossible.
+ * - exit_sem is impossible, it always operates on
+ * current (or a dead task).
+ */
+
+ rcu_read_unlock();
+ }
}
+
error = -EFBIG;
if (max >= sma->sem_nsems)
goto out_unlock_free;
error = -EACCES;
- if (ipcperms(&sma->sem_perm, alter ? S_IWUGO : S_IRUGO))
+ if (ipcperms(ns, &sma->sem_perm, alter ? S_IWUGO : S_IRUGO))
goto out_unlock_free;
error = security_sem_semop(sma, sops, nsops, alter);
error = try_atomic_semop (sma, sops, nsops, un, task_tgid_vnr(current));
if (error <= 0) {
if (alter && error == 0)
- update_queue (sma);
+ do_smart_update(sma, sops, nsops, 1, &tasks);
+
goto out_unlock_free;
}
* task into the pending queue and go to sleep.
*/
- queue.sma = sma;
queue.sops = sops;
queue.nsops = nsops;
queue.undo = un;
queue.pid = task_tgid_vnr(current);
- queue.id = semid;
queue.alter = alter;
if (alter)
- append_to_queue(sma ,&queue);
+ list_add_tail(&queue.list, &sma->sem_pending);
else
- prepend_to_queue(sma ,&queue);
+ list_add(&queue.list, &sma->sem_pending);
+
+ if (nsops == 1) {
+ struct sem *curr;
+ curr = &sma->sem_base[sops->sem_num];
+
+ if (alter)
+ list_add_tail(&queue.simple_list, &curr->sem_pending);
+ else
+ list_add(&queue.simple_list, &curr->sem_pending);
+ } else {
+ INIT_LIST_HEAD(&queue.simple_list);
+ sma->complex_count++;
+ }
queue.status = -EINTR;
queue.sleeper = current;
+
+sleep_again:
current->state = TASK_INTERRUPTIBLE;
sem_unlock(sma);
else
schedule();
- error = queue.status;
- while(unlikely(error == IN_WAKEUP)) {
- cpu_relax();
- error = queue.status;
- }
+ error = get_queue_result(&queue);
if (error != -EINTR) {
/* fast path: update_queue already obtained all requested
- * resources */
+ * resources.
+ * Perform a smp_mb(): User space could assume that semop()
+ * is a memory barrier: Without the mb(), the cpu could
+ * speculatively read in user space stale data that was
+ * overwritten by the previous owner of the semaphore.
+ */
+ smp_mb();
+
goto out_free;
}
sma = sem_lock(ns, semid);
+
+ /*
+ * Wait until it's guaranteed that no wakeup_sem_queue_do() is ongoing.
+ */
+ error = get_queue_result(&queue);
+
+ /*
+ * Array removed? If yes, leave without sem_unlock().
+ */
if (IS_ERR(sma)) {
- BUG_ON(queue.prev != NULL);
- error = -EIDRM;
goto out_free;
}
+
/*
- * If queue.status != -EINTR we are woken up by another process
+ * If queue.status != -EINTR we are woken up by another process.
+ * Leave without unlink_queue(), but with sem_unlock().
*/
- error = queue.status;
+
if (error != -EINTR) {
goto out_unlock_free;
}
*/
if (timeout && jiffies_left == 0)
error = -EAGAIN;
- remove_from_queue(sma,&queue);
- goto out_unlock_free;
+
+ /*
+ * If the wakeup was spurious, just retry
+ */
+ if (error == -EINTR && !signal_pending(current))
+ goto sleep_again;
+
+ unlink_queue(sma, &queue);
out_unlock_free:
sem_unlock(sma);
+
+ wake_up_sem_queue_do(&tasks);
out_free:
if(sops != fast_sops)
kfree(sops);
return error;
}
-asmlinkage long sys_semop (int semid, struct sembuf __user *tsops, unsigned nsops)
+SYSCALL_DEFINE3(semop, int, semid, struct sembuf __user *, tsops,
+ unsigned, nsops)
{
return sys_semtimedop(semid, tsops, nsops, NULL);
}
*/
void exit_sem(struct task_struct *tsk)
{
- struct sem_undo_list *undo_list;
- struct sem_undo *u, **up;
- struct ipc_namespace *ns;
+ struct sem_undo_list *ulp;
- undo_list = tsk->sysvsem.undo_list;
- if (!undo_list)
+ ulp = tsk->sysvsem.undo_list;
+ if (!ulp)
return;
+ tsk->sysvsem.undo_list = NULL;
- if (!atomic_dec_and_test(&undo_list->refcnt))
+ if (!atomic_dec_and_test(&ulp->refcnt))
return;
- ns = tsk->nsproxy->ipc_ns;
- /* There's no need to hold the semundo list lock, as current
- * is the last task exiting for this undo list.
- */
- for (up = &undo_list->proc_list; (u = *up); *up = u->proc_next, kfree(u)) {
+ for (;;) {
struct sem_array *sma;
- int nsems, i;
- struct sem_undo *un, **unp;
+ struct sem_undo *un;
+ struct list_head tasks;
int semid;
-
- semid = u->semid;
+ int i;
- if(semid == -1)
- continue;
- sma = sem_lock(ns, semid);
+ rcu_read_lock();
+ un = list_entry_rcu(ulp->list_proc.next,
+ struct sem_undo, list_proc);
+ if (&un->list_proc == &ulp->list_proc)
+ semid = -1;
+ else
+ semid = un->semid;
+ rcu_read_unlock();
+
+ if (semid == -1)
+ break;
+
+ sma = sem_lock_check(tsk->nsproxy->ipc_ns, un->semid);
+
+ /* exit_sem raced with IPC_RMID, nothing to do */
if (IS_ERR(sma))
continue;
- if (u->semid == -1)
- goto next_entry;
+ un = __lookup_undo(ulp, semid);
+ if (un == NULL) {
+ /* exit_sem raced with IPC_RMID+semget() that created
+ * exactly the same semid. Nothing to do.
+ */
+ sem_unlock(sma);
+ continue;
+ }
- BUG_ON(sem_checkid(sma, u->semid));
+ /* remove un from the linked lists */
+ assert_spin_locked(&sma->sem_perm.lock);
+ list_del(&un->list_id);
- /* remove u from the sma->undo list */
- for (unp = &sma->undo; (un = *unp); unp = &un->id_next) {
- if (u == un)
- goto found;
- }
- printk ("exit_sem undo list error id=%d\n", u->semid);
- goto next_entry;
-found:
- *unp = un->id_next;
- /* perform adjustments registered in u */
- nsems = sma->sem_nsems;
- for (i = 0; i < nsems; i++) {
+ spin_lock(&ulp->lock);
+ list_del_rcu(&un->list_proc);
+ spin_unlock(&ulp->lock);
+
+ /* perform adjustments registered in un */
+ for (i = 0; i < sma->sem_nsems; i++) {
struct sem * semaphore = &sma->sem_base[i];
- if (u->semadj[i]) {
- semaphore->semval += u->semadj[i];
+ if (un->semadj[i]) {
+ semaphore->semval += un->semadj[i];
/*
* Range checks of the new semaphore value,
* not defined by sus:
semaphore->sempid = task_tgid_vnr(current);
}
}
- sma->sem_otime = get_seconds();
/* maybe some queued-up processes were waiting for this */
- update_queue(sma);
-next_entry:
+ INIT_LIST_HEAD(&tasks);
+ do_smart_update(sma, NULL, 0, 1, &tasks);
sem_unlock(sma);
+ wake_up_sem_queue_do(&tasks);
+
+ kfree_rcu(un, rcu);
}
- kfree(undo_list);
+ kfree(ulp);
}
#ifdef CONFIG_PROC_FS
static int sysvipc_sem_proc_show(struct seq_file *s, void *it)
{
+ struct user_namespace *user_ns = seq_user_ns(s);
struct sem_array *sma = it;
return seq_printf(s,
- "%10d %10d %4o %10lu %5u %5u %5u %5u %10lu %10lu\n",
+ "%10d %10d %4o %10u %5u %5u %5u %5u %10lu %10lu\n",
sma->sem_perm.key,
sma->sem_perm.id,
sma->sem_perm.mode,
sma->sem_nsems,
- sma->sem_perm.uid,
- sma->sem_perm.gid,
- sma->sem_perm.cuid,
- sma->sem_perm.cgid,
+ from_kuid_munged(user_ns, sma->sem_perm.uid),
+ from_kgid_munged(user_ns, sma->sem_perm.gid),
+ from_kuid_munged(user_ns, sma->sem_perm.cuid),
+ from_kgid_munged(user_ns, sma->sem_perm.cgid),
sma->sem_otime,
sma->sem_ctime);
}
Code Review / linux-3.10.git / blobdiff