linux/kernel/rcutorture.c
<<
>>
Prefs
   1/*
   2 * Read-Copy Update module-based torture test facility
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License as published by
   6 * the Free Software Foundation; either version 2 of the License, or
   7 * (at your option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write to the Free Software
  16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17 *
  18 * Copyright (C) IBM Corporation, 2005, 2006
  19 *
  20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21 *        Josh Triplett <josh@freedesktop.org>
  22 *
  23 * See also:  Documentation/RCU/torture.txt
  24 */
  25#include <linux/types.h>
  26#include <linux/kernel.h>
  27#include <linux/init.h>
  28#include <linux/module.h>
  29#include <linux/kthread.h>
  30#include <linux/err.h>
  31#include <linux/spinlock.h>
  32#include <linux/smp.h>
  33#include <linux/rcupdate.h>
  34#include <linux/interrupt.h>
  35#include <linux/sched.h>
  36#include <asm/atomic.h>
  37#include <linux/bitops.h>
  38#include <linux/completion.h>
  39#include <linux/moduleparam.h>
  40#include <linux/percpu.h>
  41#include <linux/notifier.h>
  42#include <linux/reboot.h>
  43#include <linux/freezer.h>
  44#include <linux/cpu.h>
  45#include <linux/delay.h>
  46#include <linux/stat.h>
  47#include <linux/srcu.h>
  48#include <linux/slab.h>
  49#include <asm/byteorder.h>
  50
  51MODULE_LICENSE("GPL");
  52MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "
  53              "Josh Triplett <josh@freedesktop.org>");
  54
  55static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
  56static int nfakewriters = 4;    /* # fake writer threads */
  57static int stat_interval;       /* Interval between stats, in seconds. */
  58                                /*  Defaults to "only at end of test". */
  59static int verbose;             /* Print more debug info. */
  60static int test_no_idle_hz;     /* Test RCU's support for tickless idle CPUs. */
  61static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
  62static int stutter = 5;         /* Start/stop testing interval (in sec) */
  63static int irqreader = 1;       /* RCU readers from irq (timers). */
  64static int fqs_duration = 0;    /* Duration of bursts (us), 0 to disable. */
  65static int fqs_holdoff = 0;     /* Hold time within burst (us). */
  66static int fqs_stutter = 3;     /* Wait time between bursts (s). */
  67static char *torture_type = "rcu"; /* What RCU implementation to torture. */
  68
  69module_param(nreaders, int, 0444);
  70MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
  71module_param(nfakewriters, int, 0444);
  72MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
  73module_param(stat_interval, int, 0444);
  74MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
  75module_param(verbose, bool, 0444);
  76MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
  77module_param(test_no_idle_hz, bool, 0444);
  78MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
  79module_param(shuffle_interval, int, 0444);
  80MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
  81module_param(stutter, int, 0444);
  82MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
  83module_param(irqreader, int, 0444);
  84MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
  85module_param(fqs_duration, int, 0444);
  86MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");
  87module_param(fqs_holdoff, int, 0444);
  88MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
  89module_param(fqs_stutter, int, 0444);
  90MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
  91module_param(torture_type, charp, 0444);
  92MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
  93
  94#define TORTURE_FLAG "-torture:"
  95#define PRINTK_STRING(s) \
  96        do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  97#define VERBOSE_PRINTK_STRING(s) \
  98        do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  99#define VERBOSE_PRINTK_ERRSTRING(s) \
 100        do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
 101
 102static char printk_buf[4096];
 103
 104static int nrealreaders;
 105static struct task_struct *writer_task;
 106static struct task_struct **fakewriter_tasks;
 107static struct task_struct **reader_tasks;
 108static struct task_struct *stats_task;
 109static struct task_struct *shuffler_task;
 110static struct task_struct *stutter_task;
 111static struct task_struct *fqs_task;
 112
 113#define RCU_TORTURE_PIPE_LEN 10
 114
 115struct rcu_torture {
 116        struct rcu_head rtort_rcu;
 117        int rtort_pipe_count;
 118        struct list_head rtort_free;
 119        int rtort_mbtest;
 120};
 121
 122static LIST_HEAD(rcu_torture_freelist);
 123static struct rcu_torture __rcu *rcu_torture_current;
 124static long rcu_torture_current_version;
 125static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 126static DEFINE_SPINLOCK(rcu_torture_lock);
 127static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
 128        { 0 };
 129static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
 130        { 0 };
 131static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
 132static atomic_t n_rcu_torture_alloc;
 133static atomic_t n_rcu_torture_alloc_fail;
 134static atomic_t n_rcu_torture_free;
 135static atomic_t n_rcu_torture_mberror;
 136static atomic_t n_rcu_torture_error;
 137static long n_rcu_torture_timers;
 138static struct list_head rcu_torture_removed;
 139static cpumask_var_t shuffle_tmp_mask;
 140
 141static int stutter_pause_test;
 142
 143#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
 144#define RCUTORTURE_RUNNABLE_INIT 1
 145#else
 146#define RCUTORTURE_RUNNABLE_INIT 0
 147#endif
 148int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
 149
 150/* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
 151
 152#define FULLSTOP_DONTSTOP 0     /* Normal operation. */
 153#define FULLSTOP_SHUTDOWN 1     /* System shutdown with rcutorture running. */
 154#define FULLSTOP_RMMOD    2     /* Normal rmmod of rcutorture. */
 155static int fullstop = FULLSTOP_RMMOD;
 156/*
 157 * Protect fullstop transitions and spawning of kthreads.
 158 */
 159static DEFINE_MUTEX(fullstop_mutex);
 160
 161/*
 162 * Detect and respond to a system shutdown.
 163 */
 164static int
 165rcutorture_shutdown_notify(struct notifier_block *unused1,
 166                           unsigned long unused2, void *unused3)
 167{
 168        mutex_lock(&fullstop_mutex);
 169        if (fullstop == FULLSTOP_DONTSTOP)
 170                fullstop = FULLSTOP_SHUTDOWN;
 171        else
 172                printk(KERN_WARNING /* but going down anyway, so... */
 173                       "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
 174        mutex_unlock(&fullstop_mutex);
 175        return NOTIFY_DONE;
 176}
 177
 178/*
 179 * Absorb kthreads into a kernel function that won't return, so that
 180 * they won't ever access module text or data again.
 181 */
 182static void rcutorture_shutdown_absorb(char *title)
 183{
 184        if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 185                printk(KERN_NOTICE
 186                       "rcutorture thread %s parking due to system shutdown\n",
 187                       title);
 188                schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
 189        }
 190}
 191
 192/*
 193 * Allocate an element from the rcu_tortures pool.
 194 */
 195static struct rcu_torture *
 196rcu_torture_alloc(void)
 197{
 198        struct list_head *p;
 199
 200        spin_lock_bh(&rcu_torture_lock);
 201        if (list_empty(&rcu_torture_freelist)) {
 202                atomic_inc(&n_rcu_torture_alloc_fail);
 203                spin_unlock_bh(&rcu_torture_lock);
 204                return NULL;
 205        }
 206        atomic_inc(&n_rcu_torture_alloc);
 207        p = rcu_torture_freelist.next;
 208        list_del_init(p);
 209        spin_unlock_bh(&rcu_torture_lock);
 210        return container_of(p, struct rcu_torture, rtort_free);
 211}
 212
 213/*
 214 * Free an element to the rcu_tortures pool.
 215 */
 216static void
 217rcu_torture_free(struct rcu_torture *p)
 218{
 219        atomic_inc(&n_rcu_torture_free);
 220        spin_lock_bh(&rcu_torture_lock);
 221        list_add_tail(&p->rtort_free, &rcu_torture_freelist);
 222        spin_unlock_bh(&rcu_torture_lock);
 223}
 224
 225struct rcu_random_state {
 226        unsigned long rrs_state;
 227        long rrs_count;
 228};
 229
 230#define RCU_RANDOM_MULT 39916801  /* prime */
 231#define RCU_RANDOM_ADD  479001701 /* prime */
 232#define RCU_RANDOM_REFRESH 10000
 233
 234#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
 235
 236/*
 237 * Crude but fast random-number generator.  Uses a linear congruential
 238 * generator, with occasional help from cpu_clock().
 239 */
 240static unsigned long
 241rcu_random(struct rcu_random_state *rrsp)
 242{
 243        if (--rrsp->rrs_count < 0) {
 244                rrsp->rrs_state += (unsigned long)local_clock();
 245                rrsp->rrs_count = RCU_RANDOM_REFRESH;
 246        }
 247        rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
 248        return swahw32(rrsp->rrs_state);
 249}
 250
 251static void
 252rcu_stutter_wait(char *title)
 253{
 254        while (stutter_pause_test || !rcutorture_runnable) {
 255                if (rcutorture_runnable)
 256                        schedule_timeout_interruptible(1);
 257                else
 258                        schedule_timeout_interruptible(round_jiffies_relative(HZ));
 259                rcutorture_shutdown_absorb(title);
 260        }
 261}
 262
 263/*
 264 * Operations vector for selecting different types of tests.
 265 */
 266
 267struct rcu_torture_ops {
 268        void (*init)(void);
 269        void (*cleanup)(void);
 270        int (*readlock)(void);
 271        void (*read_delay)(struct rcu_random_state *rrsp);
 272        void (*readunlock)(int idx);
 273        int (*completed)(void);
 274        void (*deferred_free)(struct rcu_torture *p);
 275        void (*sync)(void);
 276        void (*cb_barrier)(void);
 277        void (*fqs)(void);
 278        int (*stats)(char *page);
 279        int irq_capable;
 280        char *name;
 281};
 282
 283static struct rcu_torture_ops *cur_ops;
 284
 285/*
 286 * Definitions for rcu torture testing.
 287 */
 288
 289static int rcu_torture_read_lock(void) __acquires(RCU)
 290{
 291        rcu_read_lock();
 292        return 0;
 293}
 294
 295static void rcu_read_delay(struct rcu_random_state *rrsp)
 296{
 297        const unsigned long shortdelay_us = 200;
 298        const unsigned long longdelay_ms = 50;
 299
 300        /* We want a short delay sometimes to make a reader delay the grace
 301         * period, and we want a long delay occasionally to trigger
 302         * force_quiescent_state. */
 303
 304        if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))
 305                mdelay(longdelay_ms);
 306        if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
 307                udelay(shortdelay_us);
 308#ifdef CONFIG_PREEMPT
 309        if (!preempt_count() && !(rcu_random(rrsp) % (nrealreaders * 20000)))
 310                preempt_schedule();  /* No QS if preempt_disable() in effect */
 311#endif
 312}
 313
 314static void rcu_torture_read_unlock(int idx) __releases(RCU)
 315{
 316        rcu_read_unlock();
 317}
 318
 319static int rcu_torture_completed(void)
 320{
 321        return rcu_batches_completed();
 322}
 323
 324static void
 325rcu_torture_cb(struct rcu_head *p)
 326{
 327        int i;
 328        struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
 329
 330        if (fullstop != FULLSTOP_DONTSTOP) {
 331                /* Test is ending, just drop callbacks on the floor. */
 332                /* The next initialization will pick up the pieces. */
 333                return;
 334        }
 335        i = rp->rtort_pipe_count;
 336        if (i > RCU_TORTURE_PIPE_LEN)
 337                i = RCU_TORTURE_PIPE_LEN;
 338        atomic_inc(&rcu_torture_wcount[i]);
 339        if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 340                rp->rtort_mbtest = 0;
 341                rcu_torture_free(rp);
 342        } else
 343                cur_ops->deferred_free(rp);
 344}
 345
 346static int rcu_no_completed(void)
 347{
 348        return 0;
 349}
 350
 351static void rcu_torture_deferred_free(struct rcu_torture *p)
 352{
 353        call_rcu(&p->rtort_rcu, rcu_torture_cb);
 354}
 355
 356static struct rcu_torture_ops rcu_ops = {
 357        .init           = NULL,
 358        .cleanup        = NULL,
 359        .readlock       = rcu_torture_read_lock,
 360        .read_delay     = rcu_read_delay,
 361        .readunlock     = rcu_torture_read_unlock,
 362        .completed      = rcu_torture_completed,
 363        .deferred_free  = rcu_torture_deferred_free,
 364        .sync           = synchronize_rcu,
 365        .cb_barrier     = rcu_barrier,
 366        .fqs            = rcu_force_quiescent_state,
 367        .stats          = NULL,
 368        .irq_capable    = 1,
 369        .name           = "rcu"
 370};
 371
 372static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
 373{
 374        int i;
 375        struct rcu_torture *rp;
 376        struct rcu_torture *rp1;
 377
 378        cur_ops->sync();
 379        list_add(&p->rtort_free, &rcu_torture_removed);
 380        list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
 381                i = rp->rtort_pipe_count;
 382                if (i > RCU_TORTURE_PIPE_LEN)
 383                        i = RCU_TORTURE_PIPE_LEN;
 384                atomic_inc(&rcu_torture_wcount[i]);
 385                if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 386                        rp->rtort_mbtest = 0;
 387                        list_del(&rp->rtort_free);
 388                        rcu_torture_free(rp);
 389                }
 390        }
 391}
 392
 393static void rcu_sync_torture_init(void)
 394{
 395        INIT_LIST_HEAD(&rcu_torture_removed);
 396}
 397
 398static struct rcu_torture_ops rcu_sync_ops = {
 399        .init           = rcu_sync_torture_init,
 400        .cleanup        = NULL,
 401        .readlock       = rcu_torture_read_lock,
 402        .read_delay     = rcu_read_delay,
 403        .readunlock     = rcu_torture_read_unlock,
 404        .completed      = rcu_torture_completed,
 405        .deferred_free  = rcu_sync_torture_deferred_free,
 406        .sync           = synchronize_rcu,
 407        .cb_barrier     = NULL,
 408        .fqs            = rcu_force_quiescent_state,
 409        .stats          = NULL,
 410        .irq_capable    = 1,
 411        .name           = "rcu_sync"
 412};
 413
 414static struct rcu_torture_ops rcu_expedited_ops = {
 415        .init           = rcu_sync_torture_init,
 416        .cleanup        = NULL,
 417        .readlock       = rcu_torture_read_lock,
 418        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 419        .readunlock     = rcu_torture_read_unlock,
 420        .completed      = rcu_no_completed,
 421        .deferred_free  = rcu_sync_torture_deferred_free,
 422        .sync           = synchronize_rcu_expedited,
 423        .cb_barrier     = NULL,
 424        .fqs            = rcu_force_quiescent_state,
 425        .stats          = NULL,
 426        .irq_capable    = 1,
 427        .name           = "rcu_expedited"
 428};
 429
 430/*
 431 * Definitions for rcu_bh torture testing.
 432 */
 433
 434static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
 435{
 436        rcu_read_lock_bh();
 437        return 0;
 438}
 439
 440static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
 441{
 442        rcu_read_unlock_bh();
 443}
 444
 445static int rcu_bh_torture_completed(void)
 446{
 447        return rcu_batches_completed_bh();
 448}
 449
 450static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 451{
 452        call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
 453}
 454
 455struct rcu_bh_torture_synchronize {
 456        struct rcu_head head;
 457        struct completion completion;
 458};
 459
 460static void rcu_bh_torture_wakeme_after_cb(struct rcu_head *head)
 461{
 462        struct rcu_bh_torture_synchronize *rcu;
 463
 464        rcu = container_of(head, struct rcu_bh_torture_synchronize, head);
 465        complete(&rcu->completion);
 466}
 467
 468static void rcu_bh_torture_synchronize(void)
 469{
 470        struct rcu_bh_torture_synchronize rcu;
 471
 472        init_rcu_head_on_stack(&rcu.head);
 473        init_completion(&rcu.completion);
 474        call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb);
 475        wait_for_completion(&rcu.completion);
 476        destroy_rcu_head_on_stack(&rcu.head);
 477}
 478
 479static struct rcu_torture_ops rcu_bh_ops = {
 480        .init           = NULL,
 481        .cleanup        = NULL,
 482        .readlock       = rcu_bh_torture_read_lock,
 483        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 484        .readunlock     = rcu_bh_torture_read_unlock,
 485        .completed      = rcu_bh_torture_completed,
 486        .deferred_free  = rcu_bh_torture_deferred_free,
 487        .sync           = rcu_bh_torture_synchronize,
 488        .cb_barrier     = rcu_barrier_bh,
 489        .fqs            = rcu_bh_force_quiescent_state,
 490        .stats          = NULL,
 491        .irq_capable    = 1,
 492        .name           = "rcu_bh"
 493};
 494
 495static struct rcu_torture_ops rcu_bh_sync_ops = {
 496        .init           = rcu_sync_torture_init,
 497        .cleanup        = NULL,
 498        .readlock       = rcu_bh_torture_read_lock,
 499        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 500        .readunlock     = rcu_bh_torture_read_unlock,
 501        .completed      = rcu_bh_torture_completed,
 502        .deferred_free  = rcu_sync_torture_deferred_free,
 503        .sync           = rcu_bh_torture_synchronize,
 504        .cb_barrier     = NULL,
 505        .fqs            = rcu_bh_force_quiescent_state,
 506        .stats          = NULL,
 507        .irq_capable    = 1,
 508        .name           = "rcu_bh_sync"
 509};
 510
 511/*
 512 * Definitions for srcu torture testing.
 513 */
 514
 515static struct srcu_struct srcu_ctl;
 516
 517static void srcu_torture_init(void)
 518{
 519        init_srcu_struct(&srcu_ctl);
 520        rcu_sync_torture_init();
 521}
 522
 523static void srcu_torture_cleanup(void)
 524{
 525        synchronize_srcu(&srcu_ctl);
 526        cleanup_srcu_struct(&srcu_ctl);
 527}
 528
 529static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
 530{
 531        return srcu_read_lock(&srcu_ctl);
 532}
 533
 534static void srcu_read_delay(struct rcu_random_state *rrsp)
 535{
 536        long delay;
 537        const long uspertick = 1000000 / HZ;
 538        const long longdelay = 10;
 539
 540        /* We want there to be long-running readers, but not all the time. */
 541
 542        delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
 543        if (!delay)
 544                schedule_timeout_interruptible(longdelay);
 545        else
 546                rcu_read_delay(rrsp);
 547}
 548
 549static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
 550{
 551        srcu_read_unlock(&srcu_ctl, idx);
 552}
 553
 554static int srcu_torture_completed(void)
 555{
 556        return srcu_batches_completed(&srcu_ctl);
 557}
 558
 559static void srcu_torture_synchronize(void)
 560{
 561        synchronize_srcu(&srcu_ctl);
 562}
 563
 564static int srcu_torture_stats(char *page)
 565{
 566        int cnt = 0;
 567        int cpu;
 568        int idx = srcu_ctl.completed & 0x1;
 569
 570        cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
 571                       torture_type, TORTURE_FLAG, idx);
 572        for_each_possible_cpu(cpu) {
 573                cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
 574                               per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
 575                               per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
 576        }
 577        cnt += sprintf(&page[cnt], "\n");
 578        return cnt;
 579}
 580
 581static struct rcu_torture_ops srcu_ops = {
 582        .init           = srcu_torture_init,
 583        .cleanup        = srcu_torture_cleanup,
 584        .readlock       = srcu_torture_read_lock,
 585        .read_delay     = srcu_read_delay,
 586        .readunlock     = srcu_torture_read_unlock,
 587        .completed      = srcu_torture_completed,
 588        .deferred_free  = rcu_sync_torture_deferred_free,
 589        .sync           = srcu_torture_synchronize,
 590        .cb_barrier     = NULL,
 591        .stats          = srcu_torture_stats,
 592        .name           = "srcu"
 593};
 594
 595static void srcu_torture_synchronize_expedited(void)
 596{
 597        synchronize_srcu_expedited(&srcu_ctl);
 598}
 599
 600static struct rcu_torture_ops srcu_expedited_ops = {
 601        .init           = srcu_torture_init,
 602        .cleanup        = srcu_torture_cleanup,
 603        .readlock       = srcu_torture_read_lock,
 604        .read_delay     = srcu_read_delay,
 605        .readunlock     = srcu_torture_read_unlock,
 606        .completed      = srcu_torture_completed,
 607        .deferred_free  = rcu_sync_torture_deferred_free,
 608        .sync           = srcu_torture_synchronize_expedited,
 609        .cb_barrier     = NULL,
 610        .stats          = srcu_torture_stats,
 611        .name           = "srcu_expedited"
 612};
 613
 614/*
 615 * Definitions for sched torture testing.
 616 */
 617
 618static int sched_torture_read_lock(void)
 619{
 620        preempt_disable();
 621        return 0;
 622}
 623
 624static void sched_torture_read_unlock(int idx)
 625{
 626        preempt_enable();
 627}
 628
 629static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
 630{
 631        call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
 632}
 633
 634static void sched_torture_synchronize(void)
 635{
 636        synchronize_sched();
 637}
 638
 639static struct rcu_torture_ops sched_ops = {
 640        .init           = rcu_sync_torture_init,
 641        .cleanup        = NULL,
 642        .readlock       = sched_torture_read_lock,
 643        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 644        .readunlock     = sched_torture_read_unlock,
 645        .completed      = rcu_no_completed,
 646        .deferred_free  = rcu_sched_torture_deferred_free,
 647        .sync           = sched_torture_synchronize,
 648        .cb_barrier     = rcu_barrier_sched,
 649        .fqs            = rcu_sched_force_quiescent_state,
 650        .stats          = NULL,
 651        .irq_capable    = 1,
 652        .name           = "sched"
 653};
 654
 655static struct rcu_torture_ops sched_sync_ops = {
 656        .init           = rcu_sync_torture_init,
 657        .cleanup        = NULL,
 658        .readlock       = sched_torture_read_lock,
 659        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 660        .readunlock     = sched_torture_read_unlock,
 661        .completed      = rcu_no_completed,
 662        .deferred_free  = rcu_sync_torture_deferred_free,
 663        .sync           = sched_torture_synchronize,
 664        .cb_barrier     = NULL,
 665        .fqs            = rcu_sched_force_quiescent_state,
 666        .stats          = NULL,
 667        .name           = "sched_sync"
 668};
 669
 670static struct rcu_torture_ops sched_expedited_ops = {
 671        .init           = rcu_sync_torture_init,
 672        .cleanup        = NULL,
 673        .readlock       = sched_torture_read_lock,
 674        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 675        .readunlock     = sched_torture_read_unlock,
 676        .completed      = rcu_no_completed,
 677        .deferred_free  = rcu_sync_torture_deferred_free,
 678        .sync           = synchronize_sched_expedited,
 679        .cb_barrier     = NULL,
 680        .fqs            = rcu_sched_force_quiescent_state,
 681        .stats          = NULL,
 682        .irq_capable    = 1,
 683        .name           = "sched_expedited"
 684};
 685
 686/*
 687 * RCU torture force-quiescent-state kthread.  Repeatedly induces
 688 * bursts of calls to force_quiescent_state(), increasing the probability
 689 * of occurrence of some important types of race conditions.
 690 */
 691static int
 692rcu_torture_fqs(void *arg)
 693{
 694        unsigned long fqs_resume_time;
 695        int fqs_burst_remaining;
 696
 697        VERBOSE_PRINTK_STRING("rcu_torture_fqs task started");
 698        do {
 699                fqs_resume_time = jiffies + fqs_stutter * HZ;
 700                while (jiffies - fqs_resume_time > LONG_MAX) {
 701                        schedule_timeout_interruptible(1);
 702                }
 703                fqs_burst_remaining = fqs_duration;
 704                while (fqs_burst_remaining > 0) {
 705                        cur_ops->fqs();
 706                        udelay(fqs_holdoff);
 707                        fqs_burst_remaining -= fqs_holdoff;
 708                }
 709                rcu_stutter_wait("rcu_torture_fqs");
 710        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
 711        VERBOSE_PRINTK_STRING("rcu_torture_fqs task stopping");
 712        rcutorture_shutdown_absorb("rcu_torture_fqs");
 713        while (!kthread_should_stop())
 714                schedule_timeout_uninterruptible(1);
 715        return 0;
 716}
 717
 718/*
 719 * RCU torture writer kthread.  Repeatedly substitutes a new structure
 720 * for that pointed to by rcu_torture_current, freeing the old structure
 721 * after a series of grace periods (the "pipeline").
 722 */
 723static int
 724rcu_torture_writer(void *arg)
 725{
 726        int i;
 727        long oldbatch = rcu_batches_completed();
 728        struct rcu_torture *rp;
 729        struct rcu_torture *old_rp;
 730        static DEFINE_RCU_RANDOM(rand);
 731
 732        VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
 733        set_user_nice(current, 19);
 734
 735        do {
 736                schedule_timeout_uninterruptible(1);
 737                rp = rcu_torture_alloc();
 738                if (rp == NULL)
 739                        continue;
 740                rp->rtort_pipe_count = 0;
 741                udelay(rcu_random(&rand) & 0x3ff);
 742                old_rp = rcu_dereference_check(rcu_torture_current,
 743                                               current == writer_task);
 744                rp->rtort_mbtest = 1;
 745                rcu_assign_pointer(rcu_torture_current, rp);
 746                smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
 747                if (old_rp) {
 748                        i = old_rp->rtort_pipe_count;
 749                        if (i > RCU_TORTURE_PIPE_LEN)
 750                                i = RCU_TORTURE_PIPE_LEN;
 751                        atomic_inc(&rcu_torture_wcount[i]);
 752                        old_rp->rtort_pipe_count++;
 753                        cur_ops->deferred_free(old_rp);
 754                }
 755                rcu_torture_current_version++;
 756                oldbatch = cur_ops->completed();
 757                rcu_stutter_wait("rcu_torture_writer");
 758        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
 759        VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
 760        rcutorture_shutdown_absorb("rcu_torture_writer");
 761        while (!kthread_should_stop())
 762                schedule_timeout_uninterruptible(1);
 763        return 0;
 764}
 765
 766/*
 767 * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
 768 * delay between calls.
 769 */
 770static int
 771rcu_torture_fakewriter(void *arg)
 772{
 773        DEFINE_RCU_RANDOM(rand);
 774
 775        VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
 776        set_user_nice(current, 19);
 777
 778        do {
 779                schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
 780                udelay(rcu_random(&rand) & 0x3ff);
 781                cur_ops->sync();
 782                rcu_stutter_wait("rcu_torture_fakewriter");
 783        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
 784
 785        VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
 786        rcutorture_shutdown_absorb("rcu_torture_fakewriter");
 787        while (!kthread_should_stop())
 788                schedule_timeout_uninterruptible(1);
 789        return 0;
 790}
 791
 792/*
 793 * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
 794 * incrementing the corresponding element of the pipeline array.  The
 795 * counter in the element should never be greater than 1, otherwise, the
 796 * RCU implementation is broken.
 797 */
 798static void rcu_torture_timer(unsigned long unused)
 799{
 800        int idx;
 801        int completed;
 802        static DEFINE_RCU_RANDOM(rand);
 803        static DEFINE_SPINLOCK(rand_lock);
 804        struct rcu_torture *p;
 805        int pipe_count;
 806
 807        idx = cur_ops->readlock();
 808        completed = cur_ops->completed();
 809        p = rcu_dereference_check(rcu_torture_current,
 810                                  rcu_read_lock_held() ||
 811                                  rcu_read_lock_bh_held() ||
 812                                  rcu_read_lock_sched_held() ||
 813                                  srcu_read_lock_held(&srcu_ctl));
 814        if (p == NULL) {
 815                /* Leave because rcu_torture_writer is not yet underway */
 816                cur_ops->readunlock(idx);
 817                return;
 818        }
 819        if (p->rtort_mbtest == 0)
 820                atomic_inc(&n_rcu_torture_mberror);
 821        spin_lock(&rand_lock);
 822        cur_ops->read_delay(&rand);
 823        n_rcu_torture_timers++;
 824        spin_unlock(&rand_lock);
 825        preempt_disable();
 826        pipe_count = p->rtort_pipe_count;
 827        if (pipe_count > RCU_TORTURE_PIPE_LEN) {
 828                /* Should not happen, but... */
 829                pipe_count = RCU_TORTURE_PIPE_LEN;
 830        }
 831        __this_cpu_inc(rcu_torture_count[pipe_count]);
 832        completed = cur_ops->completed() - completed;
 833        if (completed > RCU_TORTURE_PIPE_LEN) {
 834                /* Should not happen, but... */
 835                completed = RCU_TORTURE_PIPE_LEN;
 836        }
 837        __this_cpu_inc(rcu_torture_batch[completed]);
 838        preempt_enable();
 839        cur_ops->readunlock(idx);
 840}
 841
 842/*
 843 * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
 844 * incrementing the corresponding element of the pipeline array.  The
 845 * counter in the element should never be greater than 1, otherwise, the
 846 * RCU implementation is broken.
 847 */
 848static int
 849rcu_torture_reader(void *arg)
 850{
 851        int completed;
 852        int idx;
 853        DEFINE_RCU_RANDOM(rand);
 854        struct rcu_torture *p;
 855        int pipe_count;
 856        struct timer_list t;
 857
 858        VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
 859        set_user_nice(current, 19);
 860        if (irqreader && cur_ops->irq_capable)
 861                setup_timer_on_stack(&t, rcu_torture_timer, 0);
 862
 863        do {
 864                if (irqreader && cur_ops->irq_capable) {
 865                        if (!timer_pending(&t))
 866                                mod_timer(&t, jiffies + 1);
 867                }
 868                idx = cur_ops->readlock();
 869                completed = cur_ops->completed();
 870                p = rcu_dereference_check(rcu_torture_current,
 871                                          rcu_read_lock_held() ||
 872                                          rcu_read_lock_bh_held() ||
 873                                          rcu_read_lock_sched_held() ||
 874                                          srcu_read_lock_held(&srcu_ctl));
 875                if (p == NULL) {
 876                        /* Wait for rcu_torture_writer to get underway */
 877                        cur_ops->readunlock(idx);
 878                        schedule_timeout_interruptible(HZ);
 879                        continue;
 880                }
 881                if (p->rtort_mbtest == 0)
 882                        atomic_inc(&n_rcu_torture_mberror);
 883                cur_ops->read_delay(&rand);
 884                preempt_disable();
 885                pipe_count = p->rtort_pipe_count;
 886                if (pipe_count > RCU_TORTURE_PIPE_LEN) {
 887                        /* Should not happen, but... */
 888                        pipe_count = RCU_TORTURE_PIPE_LEN;
 889                }
 890                __this_cpu_inc(rcu_torture_count[pipe_count]);
 891                completed = cur_ops->completed() - completed;
 892                if (completed > RCU_TORTURE_PIPE_LEN) {
 893                        /* Should not happen, but... */
 894                        completed = RCU_TORTURE_PIPE_LEN;
 895                }
 896                __this_cpu_inc(rcu_torture_batch[completed]);
 897                preempt_enable();
 898                cur_ops->readunlock(idx);
 899                schedule();
 900                rcu_stutter_wait("rcu_torture_reader");
 901        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
 902        VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
 903        rcutorture_shutdown_absorb("rcu_torture_reader");
 904        if (irqreader && cur_ops->irq_capable)
 905                del_timer_sync(&t);
 906        while (!kthread_should_stop())
 907                schedule_timeout_uninterruptible(1);
 908        return 0;
 909}
 910
 911/*
 912 * Create an RCU-torture statistics message in the specified buffer.
 913 */
 914static int
 915rcu_torture_printk(char *page)
 916{
 917        int cnt = 0;
 918        int cpu;
 919        int i;
 920        long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 921        long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 922
 923        for_each_possible_cpu(cpu) {
 924                for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 925                        pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
 926                        batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
 927                }
 928        }
 929        for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
 930                if (pipesummary[i] != 0)
 931                        break;
 932        }
 933        cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
 934        cnt += sprintf(&page[cnt],
 935                       "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
 936                       "rtmbe: %d nt: %ld",
 937                       rcu_torture_current,
 938                       rcu_torture_current_version,
 939                       list_empty(&rcu_torture_freelist),
 940                       atomic_read(&n_rcu_torture_alloc),
 941                       atomic_read(&n_rcu_torture_alloc_fail),
 942                       atomic_read(&n_rcu_torture_free),
 943                       atomic_read(&n_rcu_torture_mberror),
 944                       n_rcu_torture_timers);
 945        if (atomic_read(&n_rcu_torture_mberror) != 0)
 946                cnt += sprintf(&page[cnt], " !!!");
 947        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 948        if (i > 1) {
 949                cnt += sprintf(&page[cnt], "!!! ");
 950                atomic_inc(&n_rcu_torture_error);
 951                WARN_ON_ONCE(1);
 952        }
 953        cnt += sprintf(&page[cnt], "Reader Pipe: ");
 954        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 955                cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
 956        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 957        cnt += sprintf(&page[cnt], "Reader Batch: ");
 958        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 959                cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
 960        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 961        cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
 962        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 963                cnt += sprintf(&page[cnt], " %d",
 964                               atomic_read(&rcu_torture_wcount[i]));
 965        }
 966        cnt += sprintf(&page[cnt], "\n");
 967        if (cur_ops->stats)
 968                cnt += cur_ops->stats(&page[cnt]);
 969        return cnt;
 970}
 971
 972/*
 973 * Print torture statistics.  Caller must ensure that there is only
 974 * one call to this function at a given time!!!  This is normally
 975 * accomplished by relying on the module system to only have one copy
 976 * of the module loaded, and then by giving the rcu_torture_stats
 977 * kthread full control (or the init/cleanup functions when rcu_torture_stats
 978 * thread is not running).
 979 */
 980static void
 981rcu_torture_stats_print(void)
 982{
 983        int cnt;
 984
 985        cnt = rcu_torture_printk(printk_buf);
 986        printk(KERN_ALERT "%s", printk_buf);
 987}
 988
 989/*
 990 * Periodically prints torture statistics, if periodic statistics printing
 991 * was specified via the stat_interval module parameter.
 992 *
 993 * No need to worry about fullstop here, since this one doesn't reference
 994 * volatile state or register callbacks.
 995 */
 996static int
 997rcu_torture_stats(void *arg)
 998{
 999        VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
1000        do {
1001                schedule_timeout_interruptible(stat_interval * HZ);
1002                rcu_torture_stats_print();
1003                rcutorture_shutdown_absorb("rcu_torture_stats");
1004        } while (!kthread_should_stop());
1005        VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
1006        return 0;
1007}
1008
1009static int rcu_idle_cpu;        /* Force all torture tasks off this CPU */
1010
1011/* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
1012 * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
1013 */
1014static void rcu_torture_shuffle_tasks(void)
1015{
1016        int i;
1017
1018        cpumask_setall(shuffle_tmp_mask);
1019        get_online_cpus();
1020
1021        /* No point in shuffling if there is only one online CPU (ex: UP) */
1022        if (num_online_cpus() == 1) {
1023                put_online_cpus();
1024                return;
1025        }
1026
1027        if (rcu_idle_cpu != -1)
1028                cpumask_clear_cpu(rcu_idle_cpu, shuffle_tmp_mask);
1029
1030        set_cpus_allowed_ptr(current, shuffle_tmp_mask);
1031
1032        if (reader_tasks) {
1033                for (i = 0; i < nrealreaders; i++)
1034                        if (reader_tasks[i])
1035                                set_cpus_allowed_ptr(reader_tasks[i],
1036                                                     shuffle_tmp_mask);
1037        }
1038
1039        if (fakewriter_tasks) {
1040                for (i = 0; i < nfakewriters; i++)
1041                        if (fakewriter_tasks[i])
1042                                set_cpus_allowed_ptr(fakewriter_tasks[i],
1043                                                     shuffle_tmp_mask);
1044        }
1045
1046        if (writer_task)
1047                set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask);
1048
1049        if (stats_task)
1050                set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask);
1051
1052        if (rcu_idle_cpu == -1)
1053                rcu_idle_cpu = num_online_cpus() - 1;
1054        else
1055                rcu_idle_cpu--;
1056
1057        put_online_cpus();
1058}
1059
1060/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
1061 * system to become idle at a time and cut off its timer ticks. This is meant
1062 * to test the support for such tickless idle CPU in RCU.
1063 */
1064static int
1065rcu_torture_shuffle(void *arg)
1066{
1067        VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
1068        do {
1069                schedule_timeout_interruptible(shuffle_interval * HZ);
1070                rcu_torture_shuffle_tasks();
1071                rcutorture_shutdown_absorb("rcu_torture_shuffle");
1072        } while (!kthread_should_stop());
1073        VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
1074        return 0;
1075}
1076
1077/* Cause the rcutorture test to "stutter", starting and stopping all
1078 * threads periodically.
1079 */
1080static int
1081rcu_torture_stutter(void *arg)
1082{
1083        VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
1084        do {
1085                schedule_timeout_interruptible(stutter * HZ);
1086                stutter_pause_test = 1;
1087                if (!kthread_should_stop())
1088                        schedule_timeout_interruptible(stutter * HZ);
1089                stutter_pause_test = 0;
1090                rcutorture_shutdown_absorb("rcu_torture_stutter");
1091        } while (!kthread_should_stop());
1092        VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
1093        return 0;
1094}
1095
1096static inline void
1097rcu_torture_print_module_parms(char *tag)
1098{
1099        printk(KERN_ALERT "%s" TORTURE_FLAG
1100                "--- %s: nreaders=%d nfakewriters=%d "
1101                "stat_interval=%d verbose=%d test_no_idle_hz=%d "
1102                "shuffle_interval=%d stutter=%d irqreader=%d "
1103                "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d\n",
1104                torture_type, tag, nrealreaders, nfakewriters,
1105                stat_interval, verbose, test_no_idle_hz, shuffle_interval,
1106                stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter);
1107}
1108
1109static struct notifier_block rcutorture_nb = {
1110        .notifier_call = rcutorture_shutdown_notify,
1111};
1112
1113static void
1114rcu_torture_cleanup(void)
1115{
1116        int i;
1117
1118        mutex_lock(&fullstop_mutex);
1119        if (fullstop == FULLSTOP_SHUTDOWN) {
1120                printk(KERN_WARNING /* but going down anyway, so... */
1121                       "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
1122                mutex_unlock(&fullstop_mutex);
1123                schedule_timeout_uninterruptible(10);
1124                if (cur_ops->cb_barrier != NULL)
1125                        cur_ops->cb_barrier();
1126                return;
1127        }
1128        fullstop = FULLSTOP_RMMOD;
1129        mutex_unlock(&fullstop_mutex);
1130        unregister_reboot_notifier(&rcutorture_nb);
1131        if (stutter_task) {
1132                VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
1133                kthread_stop(stutter_task);
1134        }
1135        stutter_task = NULL;
1136        if (shuffler_task) {
1137                VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
1138                kthread_stop(shuffler_task);
1139                free_cpumask_var(shuffle_tmp_mask);
1140        }
1141        shuffler_task = NULL;
1142
1143        if (writer_task) {
1144                VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
1145                kthread_stop(writer_task);
1146        }
1147        writer_task = NULL;
1148
1149        if (reader_tasks) {
1150                for (i = 0; i < nrealreaders; i++) {
1151                        if (reader_tasks[i]) {
1152                                VERBOSE_PRINTK_STRING(
1153                                        "Stopping rcu_torture_reader task");
1154                                kthread_stop(reader_tasks[i]);
1155                        }
1156                        reader_tasks[i] = NULL;
1157                }
1158                kfree(reader_tasks);
1159                reader_tasks = NULL;
1160        }
1161        rcu_torture_current = NULL;
1162
1163        if (fakewriter_tasks) {
1164                for (i = 0; i < nfakewriters; i++) {
1165                        if (fakewriter_tasks[i]) {
1166                                VERBOSE_PRINTK_STRING(
1167                                        "Stopping rcu_torture_fakewriter task");
1168                                kthread_stop(fakewriter_tasks[i]);
1169                        }
1170                        fakewriter_tasks[i] = NULL;
1171                }
1172                kfree(fakewriter_tasks);
1173                fakewriter_tasks = NULL;
1174        }
1175
1176        if (stats_task) {
1177                VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
1178                kthread_stop(stats_task);
1179        }
1180        stats_task = NULL;
1181
1182        if (fqs_task) {
1183                VERBOSE_PRINTK_STRING("Stopping rcu_torture_fqs task");
1184                kthread_stop(fqs_task);
1185        }
1186        fqs_task = NULL;
1187
1188        /* Wait for all RCU callbacks to fire.  */
1189
1190        if (cur_ops->cb_barrier != NULL)
1191                cur_ops->cb_barrier();
1192
1193        rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
1194
1195        if (cur_ops->cleanup)
1196                cur_ops->cleanup();
1197        if (atomic_read(&n_rcu_torture_error))
1198                rcu_torture_print_module_parms("End of test: FAILURE");
1199        else
1200                rcu_torture_print_module_parms("End of test: SUCCESS");
1201}
1202
1203static int __init
1204rcu_torture_init(void)
1205{
1206        int i;
1207        int cpu;
1208        int firsterr = 0;
1209        static struct rcu_torture_ops *torture_ops[] =
1210                { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
1211                  &rcu_bh_ops, &rcu_bh_sync_ops,
1212                  &srcu_ops, &srcu_expedited_ops,
1213                  &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
1214
1215        mutex_lock(&fullstop_mutex);
1216
1217        /* Process args and tell the world that the torturer is on the job. */
1218        for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1219                cur_ops = torture_ops[i];
1220                if (strcmp(torture_type, cur_ops->name) == 0)
1221                        break;
1222        }
1223        if (i == ARRAY_SIZE(torture_ops)) {
1224                printk(KERN_ALERT "rcu-torture: invalid torture type: \"%s\"\n",
1225                       torture_type);
1226                printk(KERN_ALERT "rcu-torture types:");
1227                for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1228                        printk(KERN_ALERT " %s", torture_ops[i]->name);
1229                printk(KERN_ALERT "\n");
1230                mutex_unlock(&fullstop_mutex);
1231                return -EINVAL;
1232        }
1233        if (cur_ops->fqs == NULL && fqs_duration != 0) {
1234                printk(KERN_ALERT "rcu-torture: ->fqs NULL and non-zero "
1235                                  "fqs_duration, fqs disabled.\n");
1236                fqs_duration = 0;
1237        }
1238        if (cur_ops->init)
1239                cur_ops->init(); /* no "goto unwind" prior to this point!!! */
1240
1241        if (nreaders >= 0)
1242                nrealreaders = nreaders;
1243        else
1244                nrealreaders = 2 * num_online_cpus();
1245        rcu_torture_print_module_parms("Start of test");
1246        fullstop = FULLSTOP_DONTSTOP;
1247
1248        /* Set up the freelist. */
1249
1250        INIT_LIST_HEAD(&rcu_torture_freelist);
1251        for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
1252                rcu_tortures[i].rtort_mbtest = 0;
1253                list_add_tail(&rcu_tortures[i].rtort_free,
1254                              &rcu_torture_freelist);
1255        }
1256
1257        /* Initialize the statistics so that each run gets its own numbers. */
1258
1259        rcu_torture_current = NULL;
1260        rcu_torture_current_version = 0;
1261        atomic_set(&n_rcu_torture_alloc, 0);
1262        atomic_set(&n_rcu_torture_alloc_fail, 0);
1263        atomic_set(&n_rcu_torture_free, 0);
1264        atomic_set(&n_rcu_torture_mberror, 0);
1265        atomic_set(&n_rcu_torture_error, 0);
1266        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1267                atomic_set(&rcu_torture_wcount[i], 0);
1268        for_each_possible_cpu(cpu) {
1269                for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1270                        per_cpu(rcu_torture_count, cpu)[i] = 0;
1271                        per_cpu(rcu_torture_batch, cpu)[i] = 0;
1272                }
1273        }
1274
1275        /* Start up the kthreads. */
1276
1277        VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
1278        writer_task = kthread_run(rcu_torture_writer, NULL,
1279                                  "rcu_torture_writer");
1280        if (IS_ERR(writer_task)) {
1281                firsterr = PTR_ERR(writer_task);
1282                VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
1283                writer_task = NULL;
1284                goto unwind;
1285        }
1286        fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
1287                                   GFP_KERNEL);
1288        if (fakewriter_tasks == NULL) {
1289                VERBOSE_PRINTK_ERRSTRING("out of memory");
1290                firsterr = -ENOMEM;
1291                goto unwind;
1292        }
1293        for (i = 0; i < nfakewriters; i++) {
1294                VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
1295                fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
1296                                                  "rcu_torture_fakewriter");
1297                if (IS_ERR(fakewriter_tasks[i])) {
1298                        firsterr = PTR_ERR(fakewriter_tasks[i]);
1299                        VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
1300                        fakewriter_tasks[i] = NULL;
1301                        goto unwind;
1302                }
1303        }
1304        reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
1305                               GFP_KERNEL);
1306        if (reader_tasks == NULL) {
1307                VERBOSE_PRINTK_ERRSTRING("out of memory");
1308                firsterr = -ENOMEM;
1309                goto unwind;
1310        }
1311        for (i = 0; i < nrealreaders; i++) {
1312                VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
1313                reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
1314                                              "rcu_torture_reader");
1315                if (IS_ERR(reader_tasks[i])) {
1316                        firsterr = PTR_ERR(reader_tasks[i]);
1317                        VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
1318                        reader_tasks[i] = NULL;
1319                        goto unwind;
1320                }
1321        }
1322        if (stat_interval > 0) {
1323                VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
1324                stats_task = kthread_run(rcu_torture_stats, NULL,
1325                                        "rcu_torture_stats");
1326                if (IS_ERR(stats_task)) {
1327                        firsterr = PTR_ERR(stats_task);
1328                        VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
1329                        stats_task = NULL;
1330                        goto unwind;
1331                }
1332        }
1333        if (test_no_idle_hz) {
1334                rcu_idle_cpu = num_online_cpus() - 1;
1335
1336                if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
1337                        firsterr = -ENOMEM;
1338                        VERBOSE_PRINTK_ERRSTRING("Failed to alloc mask");
1339                        goto unwind;
1340                }
1341
1342                /* Create the shuffler thread */
1343                shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
1344                                          "rcu_torture_shuffle");
1345                if (IS_ERR(shuffler_task)) {
1346                        free_cpumask_var(shuffle_tmp_mask);
1347                        firsterr = PTR_ERR(shuffler_task);
1348                        VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
1349                        shuffler_task = NULL;
1350                        goto unwind;
1351                }
1352        }
1353        if (stutter < 0)
1354                stutter = 0;
1355        if (stutter) {
1356                /* Create the stutter thread */
1357                stutter_task = kthread_run(rcu_torture_stutter, NULL,
1358                                          "rcu_torture_stutter");
1359                if (IS_ERR(stutter_task)) {
1360                        firsterr = PTR_ERR(stutter_task);
1361                        VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
1362                        stutter_task = NULL;
1363                        goto unwind;
1364                }
1365        }
1366        if (fqs_duration < 0)
1367                fqs_duration = 0;
1368        if (fqs_duration) {
1369                /* Create the stutter thread */
1370                fqs_task = kthread_run(rcu_torture_fqs, NULL,
1371                                       "rcu_torture_fqs");
1372                if (IS_ERR(fqs_task)) {
1373                        firsterr = PTR_ERR(fqs_task);
1374                        VERBOSE_PRINTK_ERRSTRING("Failed to create fqs");
1375                        fqs_task = NULL;
1376                        goto unwind;
1377                }
1378        }
1379        register_reboot_notifier(&rcutorture_nb);
1380        mutex_unlock(&fullstop_mutex);
1381        return 0;
1382
1383unwind:
1384        mutex_unlock(&fullstop_mutex);
1385        rcu_torture_cleanup();
1386        return firsterr;
1387}
1388
1389module_init(rcu_torture_init);
1390module_exit(rcu_torture_cleanup);
1391