Merge branch 'perf/urgent' into perf/core
[linux-2.6.git] / kernel / rcutorture.c
index 328a825..9bb5217 100644 (file)
@@ -18,7 +18,7 @@
  * Copyright (C) IBM Corporation, 2005, 2006
  *
  * Authors: Paul E. McKenney <paulmck@us.ibm.com>
- *          Josh Triplett <josh@freedesktop.org>
+ *       Josh Triplett <josh@freedesktop.org>
  *
  * See also:  Documentation/RCU/torture.txt
  */
@@ -50,7 +50,7 @@
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "
-              "Josh Triplett <josh@freedesktop.org>");
+             "Josh Triplett <josh@freedesktop.org>");
 
 static int nreaders = -1;      /* # reader threads, defaults to 2*ncpus */
 static int nfakewriters = 4;   /* # fake writer threads */
@@ -110,8 +110,8 @@ struct rcu_torture {
 };
 
 static LIST_HEAD(rcu_torture_freelist);
-static struct rcu_torture *rcu_torture_current = NULL;
-static long rcu_torture_current_version = 0;
+static struct rcu_torture *rcu_torture_current;
+static long rcu_torture_current_version;
 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 static DEFINE_SPINLOCK(rcu_torture_lock);
 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
@@ -124,11 +124,11 @@ static atomic_t n_rcu_torture_alloc_fail;
 static atomic_t n_rcu_torture_free;
 static atomic_t n_rcu_torture_mberror;
 static atomic_t n_rcu_torture_error;
-static long n_rcu_torture_timers = 0;
+static long n_rcu_torture_timers;
 static struct list_head rcu_torture_removed;
 static cpumask_var_t shuffle_tmp_mask;
 
-static int stutter_pause_test = 0;
+static int stutter_pause_test;
 
 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
 #define RCUTORTURE_RUNNABLE_INIT 1
@@ -267,7 +267,8 @@ struct rcu_torture_ops {
        int irq_capable;
        char *name;
 };
-static struct rcu_torture_ops *cur_ops = NULL;
+
+static struct rcu_torture_ops *cur_ops;
 
 /*
  * Definitions for rcu torture testing.
@@ -326,6 +327,11 @@ rcu_torture_cb(struct rcu_head *p)
                cur_ops->deferred_free(rp);
 }
 
+static int rcu_no_completed(void)
+{
+       return 0;
+}
+
 static void rcu_torture_deferred_free(struct rcu_torture *p)
 {
        call_rcu(&p->rtort_rcu, rcu_torture_cb);
@@ -342,8 +348,8 @@ static struct rcu_torture_ops rcu_ops = {
        .sync           = synchronize_rcu,
        .cb_barrier     = rcu_barrier,
        .stats          = NULL,
-       .irq_capable    = 1,
-       .name           = "rcu"
+       .irq_capable    = 1,
+       .name           = "rcu"
 };
 
 static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
@@ -387,6 +393,21 @@ static struct rcu_torture_ops rcu_sync_ops = {
        .name           = "rcu_sync"
 };
 
+static struct rcu_torture_ops rcu_expedited_ops = {
+       .init           = rcu_sync_torture_init,
+       .cleanup        = NULL,
+       .readlock       = rcu_torture_read_lock,
+       .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
+       .readunlock     = rcu_torture_read_unlock,
+       .completed      = rcu_no_completed,
+       .deferred_free  = rcu_sync_torture_deferred_free,
+       .sync           = synchronize_rcu_expedited,
+       .cb_barrier     = NULL,
+       .stats          = NULL,
+       .irq_capable    = 1,
+       .name           = "rcu_expedited"
+};
+
 /*
  * Definitions for rcu_bh torture testing.
  */
@@ -546,6 +567,25 @@ static struct rcu_torture_ops srcu_ops = {
        .name           = "srcu"
 };
 
+static void srcu_torture_synchronize_expedited(void)
+{
+       synchronize_srcu_expedited(&srcu_ctl);
+}
+
+static struct rcu_torture_ops srcu_expedited_ops = {
+       .init           = srcu_torture_init,
+       .cleanup        = srcu_torture_cleanup,
+       .readlock       = srcu_torture_read_lock,
+       .read_delay     = srcu_read_delay,
+       .readunlock     = srcu_torture_read_unlock,
+       .completed      = srcu_torture_completed,
+       .deferred_free  = rcu_sync_torture_deferred_free,
+       .sync           = srcu_torture_synchronize_expedited,
+       .cb_barrier     = NULL,
+       .stats          = srcu_torture_stats,
+       .name           = "srcu_expedited"
+};
+
 /*
  * Definitions for sched torture testing.
  */
@@ -561,11 +601,6 @@ static void sched_torture_read_unlock(int idx)
        preempt_enable();
 }
 
-static int sched_torture_completed(void)
-{
-       return 0;
-}
-
 static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
 {
        call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
@@ -582,7 +617,7 @@ static struct rcu_torture_ops sched_ops = {
        .readlock       = sched_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = sched_torture_read_unlock,
-       .completed      = sched_torture_completed,
+       .completed      = rcu_no_completed,
        .deferred_free  = rcu_sched_torture_deferred_free,
        .sync           = sched_torture_synchronize,
        .cb_barrier     = rcu_barrier_sched,
@@ -591,13 +626,13 @@ static struct rcu_torture_ops sched_ops = {
        .name           = "sched"
 };
 
-static struct rcu_torture_ops sched_ops_sync = {
+static struct rcu_torture_ops sched_sync_ops = {
        .init           = rcu_sync_torture_init,
        .cleanup        = NULL,
        .readlock       = sched_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = sched_torture_read_unlock,
-       .completed      = sched_torture_completed,
+       .completed      = rcu_no_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = sched_torture_synchronize,
        .cb_barrier     = NULL,
@@ -605,15 +640,13 @@ static struct rcu_torture_ops sched_ops_sync = {
        .name           = "sched_sync"
 };
 
-extern int rcu_expedited_torture_stats(char *page);
-
 static struct rcu_torture_ops sched_expedited_ops = {
        .init           = rcu_sync_torture_init,
        .cleanup        = NULL,
        .readlock       = sched_torture_read_lock,
        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
        .readunlock     = sched_torture_read_unlock,
-       .completed      = sched_torture_completed,
+       .completed      = rcu_no_completed,
        .deferred_free  = rcu_sync_torture_deferred_free,
        .sync           = synchronize_sched_expedited,
        .cb_barrier     = NULL,
@@ -641,14 +674,15 @@ rcu_torture_writer(void *arg)
 
        do {
                schedule_timeout_uninterruptible(1);
-               if ((rp = rcu_torture_alloc()) == NULL)
+               rp = rcu_torture_alloc();
+               if (rp == NULL)
                        continue;
                rp->rtort_pipe_count = 0;
                udelay(rcu_random(&rand) & 0x3ff);
                old_rp = rcu_torture_current;
                rp->rtort_mbtest = 1;
                rcu_assign_pointer(rcu_torture_current, rp);
-               smp_wmb();
+               smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
                if (old_rp) {
                        i = old_rp->rtort_pipe_count;
                        if (i > RCU_TORTURE_PIPE_LEN)
@@ -729,13 +763,13 @@ static void rcu_torture_timer(unsigned long unused)
                /* Should not happen, but... */
                pipe_count = RCU_TORTURE_PIPE_LEN;
        }
-       ++__get_cpu_var(rcu_torture_count)[pipe_count];
+       __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]);
        completed = cur_ops->completed() - completed;
        if (completed > RCU_TORTURE_PIPE_LEN) {
                /* Should not happen, but... */
                completed = RCU_TORTURE_PIPE_LEN;
        }
-       ++__get_cpu_var(rcu_torture_batch)[completed];
+       __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]);
        preempt_enable();
        cur_ops->readunlock(idx);
 }
@@ -784,13 +818,13 @@ rcu_torture_reader(void *arg)
                        /* Should not happen, but... */
                        pipe_count = RCU_TORTURE_PIPE_LEN;
                }
-               ++__get_cpu_var(rcu_torture_count)[pipe_count];
+               __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]);
                completed = cur_ops->completed() - completed;
                if (completed > RCU_TORTURE_PIPE_LEN) {
                        /* Should not happen, but... */
                        completed = RCU_TORTURE_PIPE_LEN;
                }
-               ++__get_cpu_var(rcu_torture_batch)[completed];
+               __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]);
                preempt_enable();
                cur_ops->readunlock(idx);
                schedule();
@@ -1097,9 +1131,10 @@ rcu_torture_init(void)
        int cpu;
        int firsterr = 0;
        static struct rcu_torture_ops *torture_ops[] =
-               { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
-                 &sched_expedited_ops,
-                 &srcu_ops, &sched_ops, &sched_ops_sync, };
+               { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
+                 &rcu_bh_ops, &rcu_bh_sync_ops,
+                 &srcu_ops, &srcu_expedited_ops,
+                 &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
 
        mutex_lock(&fullstop_mutex);
 
@@ -1110,10 +1145,14 @@ rcu_torture_init(void)
                        break;
        }
        if (i == ARRAY_SIZE(torture_ops)) {
-               printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
+               printk(KERN_ALERT "rcu-torture: invalid torture type: \"%s\"\n",
                       torture_type);
+               printk(KERN_ALERT "rcu-torture types:");
+               for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
+                       printk(KERN_ALERT " %s", torture_ops[i]->name);
+               printk(KERN_ALERT "\n");
                mutex_unlock(&fullstop_mutex);
-               return (-EINVAL);
+               return -EINVAL;
        }
        if (cur_ops->init)
                cur_ops->init(); /* no "goto unwind" prior to this point!!! */
@@ -1164,7 +1203,7 @@ rcu_torture_init(void)
                goto unwind;
        }
        fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
-                                  GFP_KERNEL);
+                                  GFP_KERNEL);
        if (fakewriter_tasks == NULL) {
                VERBOSE_PRINTK_ERRSTRING("out of memory");
                firsterr = -ENOMEM;
@@ -1173,7 +1212,7 @@ rcu_torture_init(void)
        for (i = 0; i < nfakewriters; i++) {
                VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
                fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
-                                                 "rcu_torture_fakewriter");
+                                                 "rcu_torture_fakewriter");
                if (IS_ERR(fakewriter_tasks[i])) {
                        firsterr = PTR_ERR(fakewriter_tasks[i]);
                        VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");