rcu: fix rcutorture behavior during reboot
[linux-2.6.git] / kernel / rcutorture.c
1 /*
2  * Read-Copy Update module-based torture test facility
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17  *
18  * Copyright (C) IBM Corporation, 2005, 2006
19  *
20  * Authors: Paul E. McKenney <paulmck@us.ibm.com>
21  *          Josh Triplett <josh@freedesktop.org>
22  *
23  * See also:  Documentation/RCU/torture.txt
24  */
25 #include <linux/types.h>
26 #include <linux/kernel.h>
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/kthread.h>
30 #include <linux/err.h>
31 #include <linux/spinlock.h>
32 #include <linux/smp.h>
33 #include <linux/rcupdate.h>
34 #include <linux/interrupt.h>
35 #include <linux/sched.h>
36 #include <asm/atomic.h>
37 #include <linux/bitops.h>
38 #include <linux/completion.h>
39 #include <linux/moduleparam.h>
40 #include <linux/percpu.h>
41 #include <linux/notifier.h>
42 #include <linux/reboot.h>
43 #include <linux/freezer.h>
44 #include <linux/cpu.h>
45 #include <linux/delay.h>
46 #include <linux/stat.h>
47 #include <linux/srcu.h>
48 #include <linux/slab.h>
49 #include <asm/byteorder.h>
50
51 MODULE_LICENSE("GPL");
52 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "
53               "Josh Triplett <josh@freedesktop.org>");
54
55 static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
56 static int nfakewriters = 4;    /* # fake writer threads */
57 static int stat_interval;       /* Interval between stats, in seconds. */
58                                 /*  Defaults to "only at end of test". */
59 static int verbose;             /* Print more debug info. */
60 static int test_no_idle_hz;     /* Test RCU's support for tickless idle CPUs. */
61 static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
62 static int stutter = 5;         /* Start/stop testing interval (in sec) */
63 static int irqreader = 1;       /* RCU readers from irq (timers). */
64 static char *torture_type = "rcu"; /* What RCU implementation to torture. */
65
66 module_param(nreaders, int, 0444);
67 MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
68 module_param(nfakewriters, int, 0444);
69 MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
70 module_param(stat_interval, int, 0444);
71 MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
72 module_param(verbose, bool, 0444);
73 MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
74 module_param(test_no_idle_hz, bool, 0444);
75 MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
76 module_param(shuffle_interval, int, 0444);
77 MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
78 module_param(stutter, int, 0444);
79 MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
80 module_param(irqreader, int, 0444);
81 MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
82 module_param(torture_type, charp, 0444);
83 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
84
85 #define TORTURE_FLAG "-torture:"
86 #define PRINTK_STRING(s) \
87         do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
88 #define VERBOSE_PRINTK_STRING(s) \
89         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
90 #define VERBOSE_PRINTK_ERRSTRING(s) \
91         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
92
93 static char printk_buf[4096];
94
95 static int nrealreaders;
96 static struct task_struct *writer_task;
97 static struct task_struct **fakewriter_tasks;
98 static struct task_struct **reader_tasks;
99 static struct task_struct *stats_task;
100 static struct task_struct *shuffler_task;
101 static struct task_struct *stutter_task;
102
103 #define RCU_TORTURE_PIPE_LEN 10
104
105 struct rcu_torture {
106         struct rcu_head rtort_rcu;
107         int rtort_pipe_count;
108         struct list_head rtort_free;
109         int rtort_mbtest;
110 };
111
112 static LIST_HEAD(rcu_torture_freelist);
113 static struct rcu_torture *rcu_torture_current = NULL;
114 static long rcu_torture_current_version = 0;
115 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
116 static DEFINE_SPINLOCK(rcu_torture_lock);
117 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
118         { 0 };
119 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
120         { 0 };
121 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
122 static atomic_t n_rcu_torture_alloc;
123 static atomic_t n_rcu_torture_alloc_fail;
124 static atomic_t n_rcu_torture_free;
125 static atomic_t n_rcu_torture_mberror;
126 static atomic_t n_rcu_torture_error;
127 static long n_rcu_torture_timers = 0;
128 static struct list_head rcu_torture_removed;
129
130 static int stutter_pause_test = 0;
131
132 #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
133 #define RCUTORTURE_RUNNABLE_INIT 1
134 #else
135 #define RCUTORTURE_RUNNABLE_INIT 0
136 #endif
137 int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
138
139 #define FULLSTOP_SIGNALED 1     /* Bail due to signal. */
140 #define FULLSTOP_CLEANUP  2     /* Orderly shutdown. */
141 static int fullstop;            /* stop generating callbacks at test end. */
142 DEFINE_MUTEX(fullstop_mutex);   /* protect fullstop transitions and */
143                                 /*  spawning of kthreads. */
144
145 /*
146  * Detect and respond to a signal-based shutdown.
147  */
148 static int
149 rcutorture_shutdown_notify(struct notifier_block *unused1,
150                            unsigned long unused2, void *unused3)
151 {
152         if (fullstop)
153                 return NOTIFY_DONE;
154         if (signal_pending(current)) {
155                 mutex_lock(&fullstop_mutex);
156                 if (!ACCESS_ONCE(fullstop))
157                         fullstop = FULLSTOP_SIGNALED;
158                 mutex_unlock(&fullstop_mutex);
159         }
160         return NOTIFY_DONE;
161 }
162
163 /*
164  * Allocate an element from the rcu_tortures pool.
165  */
166 static struct rcu_torture *
167 rcu_torture_alloc(void)
168 {
169         struct list_head *p;
170
171         spin_lock_bh(&rcu_torture_lock);
172         if (list_empty(&rcu_torture_freelist)) {
173                 atomic_inc(&n_rcu_torture_alloc_fail);
174                 spin_unlock_bh(&rcu_torture_lock);
175                 return NULL;
176         }
177         atomic_inc(&n_rcu_torture_alloc);
178         p = rcu_torture_freelist.next;
179         list_del_init(p);
180         spin_unlock_bh(&rcu_torture_lock);
181         return container_of(p, struct rcu_torture, rtort_free);
182 }
183
184 /*
185  * Free an element to the rcu_tortures pool.
186  */
187 static void
188 rcu_torture_free(struct rcu_torture *p)
189 {
190         atomic_inc(&n_rcu_torture_free);
191         spin_lock_bh(&rcu_torture_lock);
192         list_add_tail(&p->rtort_free, &rcu_torture_freelist);
193         spin_unlock_bh(&rcu_torture_lock);
194 }
195
196 struct rcu_random_state {
197         unsigned long rrs_state;
198         long rrs_count;
199 };
200
201 #define RCU_RANDOM_MULT 39916801  /* prime */
202 #define RCU_RANDOM_ADD  479001701 /* prime */
203 #define RCU_RANDOM_REFRESH 10000
204
205 #define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
206
207 /*
208  * Crude but fast random-number generator.  Uses a linear congruential
209  * generator, with occasional help from cpu_clock().
210  */
211 static unsigned long
212 rcu_random(struct rcu_random_state *rrsp)
213 {
214         if (--rrsp->rrs_count < 0) {
215                 rrsp->rrs_state +=
216                         (unsigned long)cpu_clock(raw_smp_processor_id());
217                 rrsp->rrs_count = RCU_RANDOM_REFRESH;
218         }
219         rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
220         return swahw32(rrsp->rrs_state);
221 }
222
223 static void
224 rcu_stutter_wait(void)
225 {
226         while ((stutter_pause_test || !rcutorture_runnable) && !fullstop) {
227                 if (rcutorture_runnable)
228                         schedule_timeout_interruptible(1);
229                 else
230                         schedule_timeout_interruptible(round_jiffies_relative(HZ));
231         }
232 }
233
234 /*
235  * Operations vector for selecting different types of tests.
236  */
237
238 struct rcu_torture_ops {
239         void (*init)(void);
240         void (*cleanup)(void);
241         int (*readlock)(void);
242         void (*readdelay)(struct rcu_random_state *rrsp);
243         void (*readunlock)(int idx);
244         int (*completed)(void);
245         void (*deferredfree)(struct rcu_torture *p);
246         void (*sync)(void);
247         void (*cb_barrier)(void);
248         int (*stats)(char *page);
249         int irqcapable;
250         char *name;
251 };
252 static struct rcu_torture_ops *cur_ops = NULL;
253
254 /*
255  * Definitions for rcu torture testing.
256  */
257
258 static int rcu_torture_read_lock(void) __acquires(RCU)
259 {
260         rcu_read_lock();
261         return 0;
262 }
263
264 static void rcu_read_delay(struct rcu_random_state *rrsp)
265 {
266         long delay;
267         const long longdelay = 200;
268
269         /* We want there to be long-running readers, but not all the time. */
270
271         delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay);
272         if (!delay)
273                 udelay(longdelay);
274 }
275
276 static void rcu_torture_read_unlock(int idx) __releases(RCU)
277 {
278         rcu_read_unlock();
279 }
280
281 static int rcu_torture_completed(void)
282 {
283         return rcu_batches_completed();
284 }
285
286 static void
287 rcu_torture_cb(struct rcu_head *p)
288 {
289         int i;
290         struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
291
292         if (fullstop) {
293                 /* Test is ending, just drop callbacks on the floor. */
294                 /* The next initialization will pick up the pieces. */
295                 return;
296         }
297         i = rp->rtort_pipe_count;
298         if (i > RCU_TORTURE_PIPE_LEN)
299                 i = RCU_TORTURE_PIPE_LEN;
300         atomic_inc(&rcu_torture_wcount[i]);
301         if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
302                 rp->rtort_mbtest = 0;
303                 rcu_torture_free(rp);
304         } else
305                 cur_ops->deferredfree(rp);
306 }
307
308 static void rcu_torture_deferred_free(struct rcu_torture *p)
309 {
310         call_rcu(&p->rtort_rcu, rcu_torture_cb);
311 }
312
313 static struct rcu_torture_ops rcu_ops = {
314         .init = NULL,
315         .cleanup = NULL,
316         .readlock = rcu_torture_read_lock,
317         .readdelay = rcu_read_delay,
318         .readunlock = rcu_torture_read_unlock,
319         .completed = rcu_torture_completed,
320         .deferredfree = rcu_torture_deferred_free,
321         .sync = synchronize_rcu,
322         .cb_barrier = rcu_barrier,
323         .stats = NULL,
324         .irqcapable = 1,
325         .name = "rcu"
326 };
327
328 static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
329 {
330         int i;
331         struct rcu_torture *rp;
332         struct rcu_torture *rp1;
333
334         cur_ops->sync();
335         list_add(&p->rtort_free, &rcu_torture_removed);
336         list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
337                 i = rp->rtort_pipe_count;
338                 if (i > RCU_TORTURE_PIPE_LEN)
339                         i = RCU_TORTURE_PIPE_LEN;
340                 atomic_inc(&rcu_torture_wcount[i]);
341                 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
342                         rp->rtort_mbtest = 0;
343                         list_del(&rp->rtort_free);
344                         rcu_torture_free(rp);
345                 }
346         }
347 }
348
349 static void rcu_sync_torture_init(void)
350 {
351         INIT_LIST_HEAD(&rcu_torture_removed);
352 }
353
354 static struct rcu_torture_ops rcu_sync_ops = {
355         .init = rcu_sync_torture_init,
356         .cleanup = NULL,
357         .readlock = rcu_torture_read_lock,
358         .readdelay = rcu_read_delay,
359         .readunlock = rcu_torture_read_unlock,
360         .completed = rcu_torture_completed,
361         .deferredfree = rcu_sync_torture_deferred_free,
362         .sync = synchronize_rcu,
363         .cb_barrier = NULL,
364         .stats = NULL,
365         .irqcapable = 1,
366         .name = "rcu_sync"
367 };
368
369 /*
370  * Definitions for rcu_bh torture testing.
371  */
372
373 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
374 {
375         rcu_read_lock_bh();
376         return 0;
377 }
378
379 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
380 {
381         rcu_read_unlock_bh();
382 }
383
384 static int rcu_bh_torture_completed(void)
385 {
386         return rcu_batches_completed_bh();
387 }
388
389 static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
390 {
391         call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
392 }
393
394 struct rcu_bh_torture_synchronize {
395         struct rcu_head head;
396         struct completion completion;
397 };
398
399 static void rcu_bh_torture_wakeme_after_cb(struct rcu_head *head)
400 {
401         struct rcu_bh_torture_synchronize *rcu;
402
403         rcu = container_of(head, struct rcu_bh_torture_synchronize, head);
404         complete(&rcu->completion);
405 }
406
407 static void rcu_bh_torture_synchronize(void)
408 {
409         struct rcu_bh_torture_synchronize rcu;
410
411         init_completion(&rcu.completion);
412         call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb);
413         wait_for_completion(&rcu.completion);
414 }
415
416 static struct rcu_torture_ops rcu_bh_ops = {
417         .init = NULL,
418         .cleanup = NULL,
419         .readlock = rcu_bh_torture_read_lock,
420         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
421         .readunlock = rcu_bh_torture_read_unlock,
422         .completed = rcu_bh_torture_completed,
423         .deferredfree = rcu_bh_torture_deferred_free,
424         .sync = rcu_bh_torture_synchronize,
425         .cb_barrier = rcu_barrier_bh,
426         .stats = NULL,
427         .irqcapable = 1,
428         .name = "rcu_bh"
429 };
430
431 static struct rcu_torture_ops rcu_bh_sync_ops = {
432         .init = rcu_sync_torture_init,
433         .cleanup = NULL,
434         .readlock = rcu_bh_torture_read_lock,
435         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
436         .readunlock = rcu_bh_torture_read_unlock,
437         .completed = rcu_bh_torture_completed,
438         .deferredfree = rcu_sync_torture_deferred_free,
439         .sync = rcu_bh_torture_synchronize,
440         .cb_barrier = NULL,
441         .stats = NULL,
442         .irqcapable = 1,
443         .name = "rcu_bh_sync"
444 };
445
446 /*
447  * Definitions for srcu torture testing.
448  */
449
450 static struct srcu_struct srcu_ctl;
451
452 static void srcu_torture_init(void)
453 {
454         init_srcu_struct(&srcu_ctl);
455         rcu_sync_torture_init();
456 }
457
458 static void srcu_torture_cleanup(void)
459 {
460         synchronize_srcu(&srcu_ctl);
461         cleanup_srcu_struct(&srcu_ctl);
462 }
463
464 static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
465 {
466         return srcu_read_lock(&srcu_ctl);
467 }
468
469 static void srcu_read_delay(struct rcu_random_state *rrsp)
470 {
471         long delay;
472         const long uspertick = 1000000 / HZ;
473         const long longdelay = 10;
474
475         /* We want there to be long-running readers, but not all the time. */
476
477         delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
478         if (!delay)
479                 schedule_timeout_interruptible(longdelay);
480 }
481
482 static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
483 {
484         srcu_read_unlock(&srcu_ctl, idx);
485 }
486
487 static int srcu_torture_completed(void)
488 {
489         return srcu_batches_completed(&srcu_ctl);
490 }
491
492 static void srcu_torture_synchronize(void)
493 {
494         synchronize_srcu(&srcu_ctl);
495 }
496
497 static int srcu_torture_stats(char *page)
498 {
499         int cnt = 0;
500         int cpu;
501         int idx = srcu_ctl.completed & 0x1;
502
503         cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
504                        torture_type, TORTURE_FLAG, idx);
505         for_each_possible_cpu(cpu) {
506                 cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
507                                per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
508                                per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
509         }
510         cnt += sprintf(&page[cnt], "\n");
511         return cnt;
512 }
513
514 static struct rcu_torture_ops srcu_ops = {
515         .init = srcu_torture_init,
516         .cleanup = srcu_torture_cleanup,
517         .readlock = srcu_torture_read_lock,
518         .readdelay = srcu_read_delay,
519         .readunlock = srcu_torture_read_unlock,
520         .completed = srcu_torture_completed,
521         .deferredfree = rcu_sync_torture_deferred_free,
522         .sync = srcu_torture_synchronize,
523         .cb_barrier = NULL,
524         .stats = srcu_torture_stats,
525         .name = "srcu"
526 };
527
528 /*
529  * Definitions for sched torture testing.
530  */
531
532 static int sched_torture_read_lock(void)
533 {
534         preempt_disable();
535         return 0;
536 }
537
538 static void sched_torture_read_unlock(int idx)
539 {
540         preempt_enable();
541 }
542
543 static int sched_torture_completed(void)
544 {
545         return 0;
546 }
547
548 static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
549 {
550         call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
551 }
552
553 static void sched_torture_synchronize(void)
554 {
555         synchronize_sched();
556 }
557
558 static struct rcu_torture_ops sched_ops = {
559         .init = rcu_sync_torture_init,
560         .cleanup = NULL,
561         .readlock = sched_torture_read_lock,
562         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
563         .readunlock = sched_torture_read_unlock,
564         .completed = sched_torture_completed,
565         .deferredfree = rcu_sched_torture_deferred_free,
566         .sync = sched_torture_synchronize,
567         .cb_barrier = rcu_barrier_sched,
568         .stats = NULL,
569         .irqcapable = 1,
570         .name = "sched"
571 };
572
573 static struct rcu_torture_ops sched_ops_sync = {
574         .init = rcu_sync_torture_init,
575         .cleanup = NULL,
576         .readlock = sched_torture_read_lock,
577         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
578         .readunlock = sched_torture_read_unlock,
579         .completed = sched_torture_completed,
580         .deferredfree = rcu_sync_torture_deferred_free,
581         .sync = sched_torture_synchronize,
582         .cb_barrier = NULL,
583         .stats = NULL,
584         .name = "sched_sync"
585 };
586
587 /*
588  * RCU torture writer kthread.  Repeatedly substitutes a new structure
589  * for that pointed to by rcu_torture_current, freeing the old structure
590  * after a series of grace periods (the "pipeline").
591  */
592 static int
593 rcu_torture_writer(void *arg)
594 {
595         int i;
596         long oldbatch = rcu_batches_completed();
597         struct rcu_torture *rp;
598         struct rcu_torture *old_rp;
599         static DEFINE_RCU_RANDOM(rand);
600
601         VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
602         set_user_nice(current, 19);
603
604         do {
605                 schedule_timeout_uninterruptible(1);
606                 if ((rp = rcu_torture_alloc()) == NULL)
607                         continue;
608                 rp->rtort_pipe_count = 0;
609                 udelay(rcu_random(&rand) & 0x3ff);
610                 old_rp = rcu_torture_current;
611                 rp->rtort_mbtest = 1;
612                 rcu_assign_pointer(rcu_torture_current, rp);
613                 smp_wmb();
614                 if (old_rp) {
615                         i = old_rp->rtort_pipe_count;
616                         if (i > RCU_TORTURE_PIPE_LEN)
617                                 i = RCU_TORTURE_PIPE_LEN;
618                         atomic_inc(&rcu_torture_wcount[i]);
619                         old_rp->rtort_pipe_count++;
620                         cur_ops->deferredfree(old_rp);
621                 }
622                 rcu_torture_current_version++;
623                 oldbatch = cur_ops->completed();
624                 rcu_stutter_wait();
625         } while (!kthread_should_stop() && !fullstop);
626         VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
627         while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
628                 schedule_timeout_uninterruptible(1);
629         return 0;
630 }
631
632 /*
633  * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
634  * delay between calls.
635  */
636 static int
637 rcu_torture_fakewriter(void *arg)
638 {
639         DEFINE_RCU_RANDOM(rand);
640
641         VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
642         set_user_nice(current, 19);
643
644         do {
645                 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
646                 udelay(rcu_random(&rand) & 0x3ff);
647                 cur_ops->sync();
648                 rcu_stutter_wait();
649         } while (!kthread_should_stop() && !fullstop);
650
651         VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
652         while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
653                 schedule_timeout_uninterruptible(1);
654         return 0;
655 }
656
657 /*
658  * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
659  * incrementing the corresponding element of the pipeline array.  The
660  * counter in the element should never be greater than 1, otherwise, the
661  * RCU implementation is broken.
662  */
663 static void rcu_torture_timer(unsigned long unused)
664 {
665         int idx;
666         int completed;
667         static DEFINE_RCU_RANDOM(rand);
668         static DEFINE_SPINLOCK(rand_lock);
669         struct rcu_torture *p;
670         int pipe_count;
671
672         idx = cur_ops->readlock();
673         completed = cur_ops->completed();
674         p = rcu_dereference(rcu_torture_current);
675         if (p == NULL) {
676                 /* Leave because rcu_torture_writer is not yet underway */
677                 cur_ops->readunlock(idx);
678                 return;
679         }
680         if (p->rtort_mbtest == 0)
681                 atomic_inc(&n_rcu_torture_mberror);
682         spin_lock(&rand_lock);
683         cur_ops->readdelay(&rand);
684         n_rcu_torture_timers++;
685         spin_unlock(&rand_lock);
686         preempt_disable();
687         pipe_count = p->rtort_pipe_count;
688         if (pipe_count > RCU_TORTURE_PIPE_LEN) {
689                 /* Should not happen, but... */
690                 pipe_count = RCU_TORTURE_PIPE_LEN;
691         }
692         ++__get_cpu_var(rcu_torture_count)[pipe_count];
693         completed = cur_ops->completed() - completed;
694         if (completed > RCU_TORTURE_PIPE_LEN) {
695                 /* Should not happen, but... */
696                 completed = RCU_TORTURE_PIPE_LEN;
697         }
698         ++__get_cpu_var(rcu_torture_batch)[completed];
699         preempt_enable();
700         cur_ops->readunlock(idx);
701 }
702
703 /*
704  * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
705  * incrementing the corresponding element of the pipeline array.  The
706  * counter in the element should never be greater than 1, otherwise, the
707  * RCU implementation is broken.
708  */
709 static int
710 rcu_torture_reader(void *arg)
711 {
712         int completed;
713         int idx;
714         DEFINE_RCU_RANDOM(rand);
715         struct rcu_torture *p;
716         int pipe_count;
717         struct timer_list t;
718
719         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
720         set_user_nice(current, 19);
721         if (irqreader && cur_ops->irqcapable)
722                 setup_timer_on_stack(&t, rcu_torture_timer, 0);
723
724         do {
725                 if (irqreader && cur_ops->irqcapable) {
726                         if (!timer_pending(&t))
727                                 mod_timer(&t, 1);
728                 }
729                 idx = cur_ops->readlock();
730                 completed = cur_ops->completed();
731                 p = rcu_dereference(rcu_torture_current);
732                 if (p == NULL) {
733                         /* Wait for rcu_torture_writer to get underway */
734                         cur_ops->readunlock(idx);
735                         schedule_timeout_interruptible(HZ);
736                         continue;
737                 }
738                 if (p->rtort_mbtest == 0)
739                         atomic_inc(&n_rcu_torture_mberror);
740                 cur_ops->readdelay(&rand);
741                 preempt_disable();
742                 pipe_count = p->rtort_pipe_count;
743                 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
744                         /* Should not happen, but... */
745                         pipe_count = RCU_TORTURE_PIPE_LEN;
746                 }
747                 ++__get_cpu_var(rcu_torture_count)[pipe_count];
748                 completed = cur_ops->completed() - completed;
749                 if (completed > RCU_TORTURE_PIPE_LEN) {
750                         /* Should not happen, but... */
751                         completed = RCU_TORTURE_PIPE_LEN;
752                 }
753                 ++__get_cpu_var(rcu_torture_batch)[completed];
754                 preempt_enable();
755                 cur_ops->readunlock(idx);
756                 schedule();
757                 rcu_stutter_wait();
758         } while (!kthread_should_stop() && !fullstop);
759         VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
760         if (irqreader && cur_ops->irqcapable)
761                 del_timer_sync(&t);
762         while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
763                 schedule_timeout_uninterruptible(1);
764         return 0;
765 }
766
767 /*
768  * Create an RCU-torture statistics message in the specified buffer.
769  */
770 static int
771 rcu_torture_printk(char *page)
772 {
773         int cnt = 0;
774         int cpu;
775         int i;
776         long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
777         long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
778
779         for_each_possible_cpu(cpu) {
780                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
781                         pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
782                         batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
783                 }
784         }
785         for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
786                 if (pipesummary[i] != 0)
787                         break;
788         }
789         cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
790         cnt += sprintf(&page[cnt],
791                        "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
792                        "rtmbe: %d nt: %ld",
793                        rcu_torture_current,
794                        rcu_torture_current_version,
795                        list_empty(&rcu_torture_freelist),
796                        atomic_read(&n_rcu_torture_alloc),
797                        atomic_read(&n_rcu_torture_alloc_fail),
798                        atomic_read(&n_rcu_torture_free),
799                        atomic_read(&n_rcu_torture_mberror),
800                        n_rcu_torture_timers);
801         if (atomic_read(&n_rcu_torture_mberror) != 0)
802                 cnt += sprintf(&page[cnt], " !!!");
803         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
804         if (i > 1) {
805                 cnt += sprintf(&page[cnt], "!!! ");
806                 atomic_inc(&n_rcu_torture_error);
807                 WARN_ON_ONCE(1);
808         }
809         cnt += sprintf(&page[cnt], "Reader Pipe: ");
810         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
811                 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
812         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
813         cnt += sprintf(&page[cnt], "Reader Batch: ");
814         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
815                 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
816         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
817         cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
818         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
819                 cnt += sprintf(&page[cnt], " %d",
820                                atomic_read(&rcu_torture_wcount[i]));
821         }
822         cnt += sprintf(&page[cnt], "\n");
823         if (cur_ops->stats)
824                 cnt += cur_ops->stats(&page[cnt]);
825         return cnt;
826 }
827
828 /*
829  * Print torture statistics.  Caller must ensure that there is only
830  * one call to this function at a given time!!!  This is normally
831  * accomplished by relying on the module system to only have one copy
832  * of the module loaded, and then by giving the rcu_torture_stats
833  * kthread full control (or the init/cleanup functions when rcu_torture_stats
834  * thread is not running).
835  */
836 static void
837 rcu_torture_stats_print(void)
838 {
839         int cnt;
840
841         cnt = rcu_torture_printk(printk_buf);
842         printk(KERN_ALERT "%s", printk_buf);
843 }
844
845 /*
846  * Periodically prints torture statistics, if periodic statistics printing
847  * was specified via the stat_interval module parameter.
848  *
849  * No need to worry about fullstop here, since this one doesn't reference
850  * volatile state or register callbacks.
851  */
852 static int
853 rcu_torture_stats(void *arg)
854 {
855         VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
856         do {
857                 schedule_timeout_interruptible(stat_interval * HZ);
858                 rcu_torture_stats_print();
859         } while (!kthread_should_stop() && !fullstop);
860         VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
861         return 0;
862 }
863
864 static int rcu_idle_cpu;        /* Force all torture tasks off this CPU */
865
866 /* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
867  * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
868  */
869 static void rcu_torture_shuffle_tasks(void)
870 {
871         cpumask_t tmp_mask;
872         int i;
873
874         cpus_setall(tmp_mask);
875         get_online_cpus();
876
877         /* No point in shuffling if there is only one online CPU (ex: UP) */
878         if (num_online_cpus() == 1) {
879                 put_online_cpus();
880                 return;
881         }
882
883         if (rcu_idle_cpu != -1)
884                 cpu_clear(rcu_idle_cpu, tmp_mask);
885
886         set_cpus_allowed_ptr(current, &tmp_mask);
887
888         if (reader_tasks) {
889                 for (i = 0; i < nrealreaders; i++)
890                         if (reader_tasks[i])
891                                 set_cpus_allowed_ptr(reader_tasks[i],
892                                                      &tmp_mask);
893         }
894
895         if (fakewriter_tasks) {
896                 for (i = 0; i < nfakewriters; i++)
897                         if (fakewriter_tasks[i])
898                                 set_cpus_allowed_ptr(fakewriter_tasks[i],
899                                                      &tmp_mask);
900         }
901
902         if (writer_task)
903                 set_cpus_allowed_ptr(writer_task, &tmp_mask);
904
905         if (stats_task)
906                 set_cpus_allowed_ptr(stats_task, &tmp_mask);
907
908         if (rcu_idle_cpu == -1)
909                 rcu_idle_cpu = num_online_cpus() - 1;
910         else
911                 rcu_idle_cpu--;
912
913         put_online_cpus();
914 }
915
916 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
917  * system to become idle at a time and cut off its timer ticks. This is meant
918  * to test the support for such tickless idle CPU in RCU.
919  */
920 static int
921 rcu_torture_shuffle(void *arg)
922 {
923         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
924         do {
925                 schedule_timeout_interruptible(shuffle_interval * HZ);
926                 rcu_torture_shuffle_tasks();
927         } while (!kthread_should_stop() && !fullstop);
928         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
929         return 0;
930 }
931
932 /* Cause the rcutorture test to "stutter", starting and stopping all
933  * threads periodically.
934  */
935 static int
936 rcu_torture_stutter(void *arg)
937 {
938         VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
939         do {
940                 schedule_timeout_interruptible(stutter * HZ);
941                 stutter_pause_test = 1;
942                 if (!kthread_should_stop() && !fullstop)
943                         schedule_timeout_interruptible(stutter * HZ);
944                 stutter_pause_test = 0;
945         } while (!kthread_should_stop() && !fullstop);
946         VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
947         return 0;
948 }
949
950 static inline void
951 rcu_torture_print_module_parms(char *tag)
952 {
953         printk(KERN_ALERT "%s" TORTURE_FLAG
954                 "--- %s: nreaders=%d nfakewriters=%d "
955                 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
956                 "shuffle_interval=%d stutter=%d irqreader=%d\n",
957                 torture_type, tag, nrealreaders, nfakewriters,
958                 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
959                 stutter, irqreader);
960 }
961
962 static struct notifier_block rcutorture_nb = {
963         .notifier_call = rcutorture_shutdown_notify,
964 };
965
966 static void
967 rcu_torture_cleanup(void)
968 {
969         int i;
970
971         mutex_lock(&fullstop_mutex);
972         if (!fullstop) {
973                 /* If being signaled, let it happen, then exit. */
974                 mutex_unlock(&fullstop_mutex);
975                 schedule_timeout_interruptible(10 * HZ);
976                 if (cur_ops->cb_barrier != NULL)
977                         cur_ops->cb_barrier();
978                 return;
979         }
980         fullstop = FULLSTOP_CLEANUP;
981         mutex_unlock(&fullstop_mutex);
982         unregister_reboot_notifier(&rcutorture_nb);
983         if (stutter_task) {
984                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
985                 kthread_stop(stutter_task);
986         }
987         stutter_task = NULL;
988         if (shuffler_task) {
989                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
990                 kthread_stop(shuffler_task);
991         }
992         shuffler_task = NULL;
993
994         if (writer_task) {
995                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
996                 kthread_stop(writer_task);
997         }
998         writer_task = NULL;
999
1000         if (reader_tasks) {
1001                 for (i = 0; i < nrealreaders; i++) {
1002                         if (reader_tasks[i]) {
1003                                 VERBOSE_PRINTK_STRING(
1004                                         "Stopping rcu_torture_reader task");
1005                                 kthread_stop(reader_tasks[i]);
1006                         }
1007                         reader_tasks[i] = NULL;
1008                 }
1009                 kfree(reader_tasks);
1010                 reader_tasks = NULL;
1011         }
1012         rcu_torture_current = NULL;
1013
1014         if (fakewriter_tasks) {
1015                 for (i = 0; i < nfakewriters; i++) {
1016                         if (fakewriter_tasks[i]) {
1017                                 VERBOSE_PRINTK_STRING(
1018                                         "Stopping rcu_torture_fakewriter task");
1019                                 kthread_stop(fakewriter_tasks[i]);
1020                         }
1021                         fakewriter_tasks[i] = NULL;
1022                 }
1023                 kfree(fakewriter_tasks);
1024                 fakewriter_tasks = NULL;
1025         }
1026
1027         if (stats_task) {
1028                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
1029                 kthread_stop(stats_task);
1030         }
1031         stats_task = NULL;
1032
1033         /* Wait for all RCU callbacks to fire.  */
1034
1035         if (cur_ops->cb_barrier != NULL)
1036                 cur_ops->cb_barrier();
1037
1038         rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
1039
1040         if (cur_ops->cleanup)
1041                 cur_ops->cleanup();
1042         if (atomic_read(&n_rcu_torture_error))
1043                 rcu_torture_print_module_parms("End of test: FAILURE");
1044         else
1045                 rcu_torture_print_module_parms("End of test: SUCCESS");
1046 }
1047
1048 static int __init
1049 rcu_torture_init(void)
1050 {
1051         int i;
1052         int cpu;
1053         int firsterr = 0;
1054         static struct rcu_torture_ops *torture_ops[] =
1055                 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
1056                   &srcu_ops, &sched_ops, &sched_ops_sync, };
1057
1058         mutex_lock(&fullstop_mutex);
1059
1060         /* Process args and tell the world that the torturer is on the job. */
1061         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1062                 cur_ops = torture_ops[i];
1063                 if (strcmp(torture_type, cur_ops->name) == 0)
1064                         break;
1065         }
1066         if (i == ARRAY_SIZE(torture_ops)) {
1067                 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
1068                        torture_type);
1069                 mutex_unlock(&fullstop_mutex);
1070                 return (-EINVAL);
1071         }
1072         if (cur_ops->init)
1073                 cur_ops->init(); /* no "goto unwind" prior to this point!!! */
1074
1075         if (nreaders >= 0)
1076                 nrealreaders = nreaders;
1077         else
1078                 nrealreaders = 2 * num_online_cpus();
1079         rcu_torture_print_module_parms("Start of test");
1080         fullstop = 0;
1081
1082         /* Set up the freelist. */
1083
1084         INIT_LIST_HEAD(&rcu_torture_freelist);
1085         for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
1086                 rcu_tortures[i].rtort_mbtest = 0;
1087                 list_add_tail(&rcu_tortures[i].rtort_free,
1088                               &rcu_torture_freelist);
1089         }
1090
1091         /* Initialize the statistics so that each run gets its own numbers. */
1092
1093         rcu_torture_current = NULL;
1094         rcu_torture_current_version = 0;
1095         atomic_set(&n_rcu_torture_alloc, 0);
1096         atomic_set(&n_rcu_torture_alloc_fail, 0);
1097         atomic_set(&n_rcu_torture_free, 0);
1098         atomic_set(&n_rcu_torture_mberror, 0);
1099         atomic_set(&n_rcu_torture_error, 0);
1100         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1101                 atomic_set(&rcu_torture_wcount[i], 0);
1102         for_each_possible_cpu(cpu) {
1103                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1104                         per_cpu(rcu_torture_count, cpu)[i] = 0;
1105                         per_cpu(rcu_torture_batch, cpu)[i] = 0;
1106                 }
1107         }
1108
1109         /* Start up the kthreads. */
1110
1111         VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
1112         writer_task = kthread_run(rcu_torture_writer, NULL,
1113                                   "rcu_torture_writer");
1114         if (IS_ERR(writer_task)) {
1115                 firsterr = PTR_ERR(writer_task);
1116                 VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
1117                 writer_task = NULL;
1118                 goto unwind;
1119         }
1120         fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
1121                                    GFP_KERNEL);
1122         if (fakewriter_tasks == NULL) {
1123                 VERBOSE_PRINTK_ERRSTRING("out of memory");
1124                 firsterr = -ENOMEM;
1125                 goto unwind;
1126         }
1127         for (i = 0; i < nfakewriters; i++) {
1128                 VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
1129                 fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
1130                                                   "rcu_torture_fakewriter");
1131                 if (IS_ERR(fakewriter_tasks[i])) {
1132                         firsterr = PTR_ERR(fakewriter_tasks[i]);
1133                         VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
1134                         fakewriter_tasks[i] = NULL;
1135                         goto unwind;
1136                 }
1137         }
1138         reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
1139                                GFP_KERNEL);
1140         if (reader_tasks == NULL) {
1141                 VERBOSE_PRINTK_ERRSTRING("out of memory");
1142                 firsterr = -ENOMEM;
1143                 goto unwind;
1144         }
1145         for (i = 0; i < nrealreaders; i++) {
1146                 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
1147                 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
1148                                               "rcu_torture_reader");
1149                 if (IS_ERR(reader_tasks[i])) {
1150                         firsterr = PTR_ERR(reader_tasks[i]);
1151                         VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
1152                         reader_tasks[i] = NULL;
1153                         goto unwind;
1154                 }
1155         }
1156         if (stat_interval > 0) {
1157                 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
1158                 stats_task = kthread_run(rcu_torture_stats, NULL,
1159                                         "rcu_torture_stats");
1160                 if (IS_ERR(stats_task)) {
1161                         firsterr = PTR_ERR(stats_task);
1162                         VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
1163                         stats_task = NULL;
1164                         goto unwind;
1165                 }
1166         }
1167         if (test_no_idle_hz) {
1168                 rcu_idle_cpu = num_online_cpus() - 1;
1169                 /* Create the shuffler thread */
1170                 shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
1171                                           "rcu_torture_shuffle");
1172                 if (IS_ERR(shuffler_task)) {
1173                         firsterr = PTR_ERR(shuffler_task);
1174                         VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
1175                         shuffler_task = NULL;
1176                         goto unwind;
1177                 }
1178         }
1179         if (stutter < 0)
1180                 stutter = 0;
1181         if (stutter) {
1182                 /* Create the stutter thread */
1183                 stutter_task = kthread_run(rcu_torture_stutter, NULL,
1184                                           "rcu_torture_stutter");
1185                 if (IS_ERR(stutter_task)) {
1186                         firsterr = PTR_ERR(stutter_task);
1187                         VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
1188                         stutter_task = NULL;
1189                         goto unwind;
1190                 }
1191         }
1192         register_reboot_notifier(&rcutorture_nb);
1193         mutex_unlock(&fullstop_mutex);
1194         return 0;
1195
1196 unwind:
1197         mutex_unlock(&fullstop_mutex);
1198         rcu_torture_cleanup();
1199         return firsterr;
1200 }
1201
1202 module_init(rcu_torture_init);
1203 module_exit(rcu_torture_cleanup);