linux/kernel/rcutorture.c
<<
>>
Prefs
   1/*
   2 * Read-Copy Update module-based torture test facility
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License as published by
   6 * the Free Software Foundation; either version 2 of the License, or
   7 * (at your option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write to the Free Software
  16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17 *
  18 * Copyright (C) IBM Corporation, 2005, 2006
  19 *
  20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21 *        Josh Triplett <josh@freedesktop.org>
  22 *
  23 * See also:  Documentation/RCU/torture.txt
  24 */
  25#include <linux/types.h>
  26#include <linux/kernel.h>
  27#include <linux/init.h>
  28#include <linux/module.h>
  29#include <linux/kthread.h>
  30#include <linux/err.h>
  31#include <linux/spinlock.h>
  32#include <linux/smp.h>
  33#include <linux/rcupdate.h>
  34#include <linux/interrupt.h>
  35#include <linux/sched.h>
  36#include <linux/atomic.h>
  37#include <linux/bitops.h>
  38#include <linux/completion.h>
  39#include <linux/moduleparam.h>
  40#include <linux/percpu.h>
  41#include <linux/notifier.h>
  42#include <linux/reboot.h>
  43#include <linux/freezer.h>
  44#include <linux/cpu.h>
  45#include <linux/delay.h>
  46#include <linux/stat.h>
  47#include <linux/srcu.h>
  48#include <linux/slab.h>
  49#include <linux/trace_clock.h>
  50#include <asm/byteorder.h>
  51
  52MODULE_LICENSE("GPL");
  53MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@freedesktop.org>");
  54
  55static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
  56static int nfakewriters = 4;    /* # fake writer threads */
  57static int stat_interval = 60;  /* Interval between stats, in seconds. */
  58                                /*  Zero means "only at end of test". */
  59static bool verbose;            /* Print more debug info. */
  60static bool test_no_idle_hz = true;
  61                                /* Test RCU support for tickless idle CPUs. */
  62static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
  63static int stutter = 5;         /* Start/stop testing interval (in sec) */
  64static int irqreader = 1;       /* RCU readers from irq (timers). */
  65static int fqs_duration;        /* Duration of bursts (us), 0 to disable. */
  66static int fqs_holdoff;         /* Hold time within burst (us). */
  67static int fqs_stutter = 3;     /* Wait time between bursts (s). */
  68static int n_barrier_cbs;       /* Number of callbacks to test RCU barriers. */
  69static int onoff_interval;      /* Wait time between CPU hotplugs, 0=disable. */
  70static int onoff_holdoff;       /* Seconds after boot before CPU hotplugs. */
  71static int shutdown_secs;       /* Shutdown time (s).  <=0 for no shutdown. */
  72static int stall_cpu;           /* CPU-stall duration (s).  0 for no stall. */
  73static int stall_cpu_holdoff = 10; /* Time to wait until stall (s).  */
  74static int test_boost = 1;      /* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */
  75static int test_boost_interval = 7; /* Interval between boost tests, seconds. */
  76static int test_boost_duration = 4; /* Duration of each boost test, seconds. */
  77static char *torture_type = "rcu"; /* What RCU implementation to torture. */
  78
  79module_param(nreaders, int, 0444);
  80MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
  81module_param(nfakewriters, int, 0444);
  82MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
  83module_param(stat_interval, int, 0644);
  84MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
  85module_param(verbose, bool, 0444);
  86MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
  87module_param(test_no_idle_hz, bool, 0444);
  88MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
  89module_param(shuffle_interval, int, 0444);
  90MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
  91module_param(stutter, int, 0444);
  92MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
  93module_param(irqreader, int, 0444);
  94MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
  95module_param(fqs_duration, int, 0444);
  96MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");
  97module_param(fqs_holdoff, int, 0444);
  98MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
  99module_param(fqs_stutter, int, 0444);
 100MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
 101module_param(n_barrier_cbs, int, 0444);
 102MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");
 103module_param(onoff_interval, int, 0444);
 104MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
 105module_param(onoff_holdoff, int, 0444);
 106MODULE_PARM_DESC(onoff_holdoff, "Time after boot before CPU hotplugs (s)");
 107module_param(shutdown_secs, int, 0444);
 108MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), zero to disable.");
 109module_param(stall_cpu, int, 0444);
 110MODULE_PARM_DESC(stall_cpu, "Stall duration (s), zero to disable.");
 111module_param(stall_cpu_holdoff, int, 0444);
 112MODULE_PARM_DESC(stall_cpu_holdoff, "Time to wait before starting stall (s).");
 113module_param(test_boost, int, 0444);
 114MODULE_PARM_DESC(test_boost, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
 115module_param(test_boost_interval, int, 0444);
 116MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds.");
 117module_param(test_boost_duration, int, 0444);
 118MODULE_PARM_DESC(test_boost_duration, "Duration of each boost test, seconds.");
 119module_param(torture_type, charp, 0444);
 120MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
 121
 122#define TORTURE_FLAG "-torture:"
 123#define PRINTK_STRING(s) \
 124        do { pr_alert("%s" TORTURE_FLAG s "\n", torture_type); } while (0)
 125#define VERBOSE_PRINTK_STRING(s) \
 126        do { if (verbose) pr_alert("%s" TORTURE_FLAG s "\n", torture_type); } while (0)
 127#define VERBOSE_PRINTK_ERRSTRING(s) \
 128        do { if (verbose) pr_alert("%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
 129
 130static char printk_buf[4096];
 131
 132static int nrealreaders;
 133static struct task_struct *writer_task;
 134static struct task_struct **fakewriter_tasks;
 135static struct task_struct **reader_tasks;
 136static struct task_struct *stats_task;
 137static struct task_struct *shuffler_task;
 138static struct task_struct *stutter_task;
 139static struct task_struct *fqs_task;
 140static struct task_struct *boost_tasks[NR_CPUS];
 141static struct task_struct *shutdown_task;
 142#ifdef CONFIG_HOTPLUG_CPU
 143static struct task_struct *onoff_task;
 144#endif /* #ifdef CONFIG_HOTPLUG_CPU */
 145static struct task_struct *stall_task;
 146static struct task_struct **barrier_cbs_tasks;
 147static struct task_struct *barrier_task;
 148
 149#define RCU_TORTURE_PIPE_LEN 10
 150
 151struct rcu_torture {
 152        struct rcu_head rtort_rcu;
 153        int rtort_pipe_count;
 154        struct list_head rtort_free;
 155        int rtort_mbtest;
 156};
 157
 158static LIST_HEAD(rcu_torture_freelist);
 159static struct rcu_torture __rcu *rcu_torture_current;
 160static unsigned long rcu_torture_current_version;
 161static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 162static DEFINE_SPINLOCK(rcu_torture_lock);
 163static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
 164        { 0 };
 165static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
 166        { 0 };
 167static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
 168static atomic_t n_rcu_torture_alloc;
 169static atomic_t n_rcu_torture_alloc_fail;
 170static atomic_t n_rcu_torture_free;
 171static atomic_t n_rcu_torture_mberror;
 172static atomic_t n_rcu_torture_error;
 173static long n_rcu_torture_barrier_error;
 174static long n_rcu_torture_boost_ktrerror;
 175static long n_rcu_torture_boost_rterror;
 176static long n_rcu_torture_boost_failure;
 177static long n_rcu_torture_boosts;
 178static long n_rcu_torture_timers;
 179static long n_offline_attempts;
 180static long n_offline_successes;
 181static unsigned long sum_offline;
 182static int min_offline = -1;
 183static int max_offline;
 184static long n_online_attempts;
 185static long n_online_successes;
 186static unsigned long sum_online;
 187static int min_online = -1;
 188static int max_online;
 189static long n_barrier_attempts;
 190static long n_barrier_successes;
 191static struct list_head rcu_torture_removed;
 192static cpumask_var_t shuffle_tmp_mask;
 193
 194static int stutter_pause_test;
 195
 196#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
 197#define RCUTORTURE_RUNNABLE_INIT 1
 198#else
 199#define RCUTORTURE_RUNNABLE_INIT 0
 200#endif
 201int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
 202module_param(rcutorture_runnable, int, 0444);
 203MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot");
 204
 205#if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
 206#define rcu_can_boost() 1
 207#else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
 208#define rcu_can_boost() 0
 209#endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
 210
 211#ifdef CONFIG_RCU_TRACE
 212static u64 notrace rcu_trace_clock_local(void)
 213{
 214        u64 ts = trace_clock_local();
 215        unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC);
 216        return ts;
 217}
 218#else /* #ifdef CONFIG_RCU_TRACE */
 219static u64 notrace rcu_trace_clock_local(void)
 220{
 221        return 0ULL;
 222}
 223#endif /* #else #ifdef CONFIG_RCU_TRACE */
 224
 225static unsigned long shutdown_time;     /* jiffies to system shutdown. */
 226static unsigned long boost_starttime;   /* jiffies of next boost test start. */
 227DEFINE_MUTEX(boost_mutex);              /* protect setting boost_starttime */
 228                                        /*  and boost task create/destroy. */
 229static atomic_t barrier_cbs_count;      /* Barrier callbacks registered. */
 230static bool barrier_phase;              /* Test phase. */
 231static atomic_t barrier_cbs_invoked;    /* Barrier callbacks invoked. */
 232static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
 233static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
 234
 235/* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
 236
 237#define FULLSTOP_DONTSTOP 0     /* Normal operation. */
 238#define FULLSTOP_SHUTDOWN 1     /* System shutdown with rcutorture running. */
 239#define FULLSTOP_RMMOD    2     /* Normal rmmod of rcutorture. */
 240static int fullstop = FULLSTOP_RMMOD;
 241/*
 242 * Protect fullstop transitions and spawning of kthreads.
 243 */
 244static DEFINE_MUTEX(fullstop_mutex);
 245
 246/* Forward reference. */
 247static void rcu_torture_cleanup(void);
 248
 249/*
 250 * Detect and respond to a system shutdown.
 251 */
 252static int
 253rcutorture_shutdown_notify(struct notifier_block *unused1,
 254                           unsigned long unused2, void *unused3)
 255{
 256        mutex_lock(&fullstop_mutex);
 257        if (fullstop == FULLSTOP_DONTSTOP)
 258                fullstop = FULLSTOP_SHUTDOWN;
 259        else
 260                pr_warn(/* but going down anyway, so... */
 261                       "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
 262        mutex_unlock(&fullstop_mutex);
 263        return NOTIFY_DONE;
 264}
 265
 266/*
 267 * Absorb kthreads into a kernel function that won't return, so that
 268 * they won't ever access module text or data again.
 269 */
 270static void rcutorture_shutdown_absorb(char *title)
 271{
 272        if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 273                pr_notice(
 274                       "rcutorture thread %s parking due to system shutdown\n",
 275                       title);
 276                schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
 277        }
 278}
 279
 280/*
 281 * Allocate an element from the rcu_tortures pool.
 282 */
 283static struct rcu_torture *
 284rcu_torture_alloc(void)
 285{
 286        struct list_head *p;
 287
 288        spin_lock_bh(&rcu_torture_lock);
 289        if (list_empty(&rcu_torture_freelist)) {
 290                atomic_inc(&n_rcu_torture_alloc_fail);
 291                spin_unlock_bh(&rcu_torture_lock);
 292                return NULL;
 293        }
 294        atomic_inc(&n_rcu_torture_alloc);
 295        p = rcu_torture_freelist.next;
 296        list_del_init(p);
 297        spin_unlock_bh(&rcu_torture_lock);
 298        return container_of(p, struct rcu_torture, rtort_free);
 299}
 300
 301/*
 302 * Free an element to the rcu_tortures pool.
 303 */
 304static void
 305rcu_torture_free(struct rcu_torture *p)
 306{
 307        atomic_inc(&n_rcu_torture_free);
 308        spin_lock_bh(&rcu_torture_lock);
 309        list_add_tail(&p->rtort_free, &rcu_torture_freelist);
 310        spin_unlock_bh(&rcu_torture_lock);
 311}
 312
 313struct rcu_random_state {
 314        unsigned long rrs_state;
 315        long rrs_count;
 316};
 317
 318#define RCU_RANDOM_MULT 39916801  /* prime */
 319#define RCU_RANDOM_ADD  479001701 /* prime */
 320#define RCU_RANDOM_REFRESH 10000
 321
 322#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
 323
 324/*
 325 * Crude but fast random-number generator.  Uses a linear congruential
 326 * generator, with occasional help from cpu_clock().
 327 */
 328static unsigned long
 329rcu_random(struct rcu_random_state *rrsp)
 330{
 331        if (--rrsp->rrs_count < 0) {
 332                rrsp->rrs_state += (unsigned long)local_clock();
 333                rrsp->rrs_count = RCU_RANDOM_REFRESH;
 334        }
 335        rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
 336        return swahw32(rrsp->rrs_state);
 337}
 338
 339static void
 340rcu_stutter_wait(char *title)
 341{
 342        while (stutter_pause_test || !rcutorture_runnable) {
 343                if (rcutorture_runnable)
 344                        schedule_timeout_interruptible(1);
 345                else
 346                        schedule_timeout_interruptible(round_jiffies_relative(HZ));
 347                rcutorture_shutdown_absorb(title);
 348        }
 349}
 350
 351/*
 352 * Operations vector for selecting different types of tests.
 353 */
 354
 355struct rcu_torture_ops {
 356        void (*init)(void);
 357        int (*readlock)(void);
 358        void (*read_delay)(struct rcu_random_state *rrsp);
 359        void (*readunlock)(int idx);
 360        int (*completed)(void);
 361        void (*deferred_free)(struct rcu_torture *p);
 362        void (*sync)(void);
 363        void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
 364        void (*cb_barrier)(void);
 365        void (*fqs)(void);
 366        int (*stats)(char *page);
 367        int irq_capable;
 368        int can_boost;
 369        char *name;
 370};
 371
 372static struct rcu_torture_ops *cur_ops;
 373
 374/*
 375 * Definitions for rcu torture testing.
 376 */
 377
 378static int rcu_torture_read_lock(void) __acquires(RCU)
 379{
 380        rcu_read_lock();
 381        return 0;
 382}
 383
 384static void rcu_read_delay(struct rcu_random_state *rrsp)
 385{
 386        const unsigned long shortdelay_us = 200;
 387        const unsigned long longdelay_ms = 50;
 388
 389        /* We want a short delay sometimes to make a reader delay the grace
 390         * period, and we want a long delay occasionally to trigger
 391         * force_quiescent_state. */
 392
 393        if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))
 394                mdelay(longdelay_ms);
 395        if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
 396                udelay(shortdelay_us);
 397#ifdef CONFIG_PREEMPT
 398        if (!preempt_count() && !(rcu_random(rrsp) % (nrealreaders * 20000)))
 399                preempt_schedule();  /* No QS if preempt_disable() in effect */
 400#endif
 401}
 402
 403static void rcu_torture_read_unlock(int idx) __releases(RCU)
 404{
 405        rcu_read_unlock();
 406}
 407
 408static int rcu_torture_completed(void)
 409{
 410        return rcu_batches_completed();
 411}
 412
 413static void
 414rcu_torture_cb(struct rcu_head *p)
 415{
 416        int i;
 417        struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
 418
 419        if (fullstop != FULLSTOP_DONTSTOP) {
 420                /* Test is ending, just drop callbacks on the floor. */
 421                /* The next initialization will pick up the pieces. */
 422                return;
 423        }
 424        i = rp->rtort_pipe_count;
 425        if (i > RCU_TORTURE_PIPE_LEN)
 426                i = RCU_TORTURE_PIPE_LEN;
 427        atomic_inc(&rcu_torture_wcount[i]);
 428        if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 429                rp->rtort_mbtest = 0;
 430                rcu_torture_free(rp);
 431        } else {
 432                cur_ops->deferred_free(rp);
 433        }
 434}
 435
 436static int rcu_no_completed(void)
 437{
 438        return 0;
 439}
 440
 441static void rcu_torture_deferred_free(struct rcu_torture *p)
 442{
 443        call_rcu(&p->rtort_rcu, rcu_torture_cb);
 444}
 445
 446static struct rcu_torture_ops rcu_ops = {
 447        .init           = NULL,
 448        .readlock       = rcu_torture_read_lock,
 449        .read_delay     = rcu_read_delay,
 450        .readunlock     = rcu_torture_read_unlock,
 451        .completed      = rcu_torture_completed,
 452        .deferred_free  = rcu_torture_deferred_free,
 453        .sync           = synchronize_rcu,
 454        .call           = call_rcu,
 455        .cb_barrier     = rcu_barrier,
 456        .fqs            = rcu_force_quiescent_state,
 457        .stats          = NULL,
 458        .irq_capable    = 1,
 459        .can_boost      = rcu_can_boost(),
 460        .name           = "rcu"
 461};
 462
 463static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
 464{
 465        int i;
 466        struct rcu_torture *rp;
 467        struct rcu_torture *rp1;
 468
 469        cur_ops->sync();
 470        list_add(&p->rtort_free, &rcu_torture_removed);
 471        list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
 472                i = rp->rtort_pipe_count;
 473                if (i > RCU_TORTURE_PIPE_LEN)
 474                        i = RCU_TORTURE_PIPE_LEN;
 475                atomic_inc(&rcu_torture_wcount[i]);
 476                if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 477                        rp->rtort_mbtest = 0;
 478                        list_del(&rp->rtort_free);
 479                        rcu_torture_free(rp);
 480                }
 481        }
 482}
 483
 484static void rcu_sync_torture_init(void)
 485{
 486        INIT_LIST_HEAD(&rcu_torture_removed);
 487}
 488
 489static struct rcu_torture_ops rcu_sync_ops = {
 490        .init           = rcu_sync_torture_init,
 491        .readlock       = rcu_torture_read_lock,
 492        .read_delay     = rcu_read_delay,
 493        .readunlock     = rcu_torture_read_unlock,
 494        .completed      = rcu_torture_completed,
 495        .deferred_free  = rcu_sync_torture_deferred_free,
 496        .sync           = synchronize_rcu,
 497        .call           = NULL,
 498        .cb_barrier     = NULL,
 499        .fqs            = rcu_force_quiescent_state,
 500        .stats          = NULL,
 501        .irq_capable    = 1,
 502        .can_boost      = rcu_can_boost(),
 503        .name           = "rcu_sync"
 504};
 505
 506static struct rcu_torture_ops rcu_expedited_ops = {
 507        .init           = rcu_sync_torture_init,
 508        .readlock       = rcu_torture_read_lock,
 509        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 510        .readunlock     = rcu_torture_read_unlock,
 511        .completed      = rcu_no_completed,
 512        .deferred_free  = rcu_sync_torture_deferred_free,
 513        .sync           = synchronize_rcu_expedited,
 514        .call           = NULL,
 515        .cb_barrier     = NULL,
 516        .fqs            = rcu_force_quiescent_state,
 517        .stats          = NULL,
 518        .irq_capable    = 1,
 519        .can_boost      = rcu_can_boost(),
 520        .name           = "rcu_expedited"
 521};
 522
 523/*
 524 * Definitions for rcu_bh torture testing.
 525 */
 526
 527static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
 528{
 529        rcu_read_lock_bh();
 530        return 0;
 531}
 532
 533static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
 534{
 535        rcu_read_unlock_bh();
 536}
 537
 538static int rcu_bh_torture_completed(void)
 539{
 540        return rcu_batches_completed_bh();
 541}
 542
 543static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 544{
 545        call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
 546}
 547
 548static struct rcu_torture_ops rcu_bh_ops = {
 549        .init           = NULL,
 550        .readlock       = rcu_bh_torture_read_lock,
 551        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 552        .readunlock     = rcu_bh_torture_read_unlock,
 553        .completed      = rcu_bh_torture_completed,
 554        .deferred_free  = rcu_bh_torture_deferred_free,
 555        .sync           = synchronize_rcu_bh,
 556        .call           = call_rcu_bh,
 557        .cb_barrier     = rcu_barrier_bh,
 558        .fqs            = rcu_bh_force_quiescent_state,
 559        .stats          = NULL,
 560        .irq_capable    = 1,
 561        .name           = "rcu_bh"
 562};
 563
 564static struct rcu_torture_ops rcu_bh_sync_ops = {
 565        .init           = rcu_sync_torture_init,
 566        .readlock       = rcu_bh_torture_read_lock,
 567        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 568        .readunlock     = rcu_bh_torture_read_unlock,
 569        .completed      = rcu_bh_torture_completed,
 570        .deferred_free  = rcu_sync_torture_deferred_free,
 571        .sync           = synchronize_rcu_bh,
 572        .call           = NULL,
 573        .cb_barrier     = NULL,
 574        .fqs            = rcu_bh_force_quiescent_state,
 575        .stats          = NULL,
 576        .irq_capable    = 1,
 577        .name           = "rcu_bh_sync"
 578};
 579
 580static struct rcu_torture_ops rcu_bh_expedited_ops = {
 581        .init           = rcu_sync_torture_init,
 582        .readlock       = rcu_bh_torture_read_lock,
 583        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 584        .readunlock     = rcu_bh_torture_read_unlock,
 585        .completed      = rcu_bh_torture_completed,
 586        .deferred_free  = rcu_sync_torture_deferred_free,
 587        .sync           = synchronize_rcu_bh_expedited,
 588        .call           = NULL,
 589        .cb_barrier     = NULL,
 590        .fqs            = rcu_bh_force_quiescent_state,
 591        .stats          = NULL,
 592        .irq_capable    = 1,
 593        .name           = "rcu_bh_expedited"
 594};
 595
 596/*
 597 * Definitions for srcu torture testing.
 598 */
 599
 600DEFINE_STATIC_SRCU(srcu_ctl);
 601
 602static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
 603{
 604        return srcu_read_lock(&srcu_ctl);
 605}
 606
 607static void srcu_read_delay(struct rcu_random_state *rrsp)
 608{
 609        long delay;
 610        const long uspertick = 1000000 / HZ;
 611        const long longdelay = 10;
 612
 613        /* We want there to be long-running readers, but not all the time. */
 614
 615        delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
 616        if (!delay)
 617                schedule_timeout_interruptible(longdelay);
 618        else
 619                rcu_read_delay(rrsp);
 620}
 621
 622static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
 623{
 624        srcu_read_unlock(&srcu_ctl, idx);
 625}
 626
 627static int srcu_torture_completed(void)
 628{
 629        return srcu_batches_completed(&srcu_ctl);
 630}
 631
 632static void srcu_torture_deferred_free(struct rcu_torture *rp)
 633{
 634        call_srcu(&srcu_ctl, &rp->rtort_rcu, rcu_torture_cb);
 635}
 636
 637static void srcu_torture_synchronize(void)
 638{
 639        synchronize_srcu(&srcu_ctl);
 640}
 641
 642static void srcu_torture_call(struct rcu_head *head,
 643                              void (*func)(struct rcu_head *head))
 644{
 645        call_srcu(&srcu_ctl, head, func);
 646}
 647
 648static void srcu_torture_barrier(void)
 649{
 650        srcu_barrier(&srcu_ctl);
 651}
 652
 653static int srcu_torture_stats(char *page)
 654{
 655        int cnt = 0;
 656        int cpu;
 657        int idx = srcu_ctl.completed & 0x1;
 658
 659        cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
 660                       torture_type, TORTURE_FLAG, idx);
 661        for_each_possible_cpu(cpu) {
 662                cnt += sprintf(&page[cnt], " %d(%lu,%lu)", cpu,
 663                               per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
 664                               per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
 665        }
 666        cnt += sprintf(&page[cnt], "\n");
 667        return cnt;
 668}
 669
 670static struct rcu_torture_ops srcu_ops = {
 671        .init           = rcu_sync_torture_init,
 672        .readlock       = srcu_torture_read_lock,
 673        .read_delay     = srcu_read_delay,
 674        .readunlock     = srcu_torture_read_unlock,
 675        .completed      = srcu_torture_completed,
 676        .deferred_free  = srcu_torture_deferred_free,
 677        .sync           = srcu_torture_synchronize,
 678        .call           = srcu_torture_call,
 679        .cb_barrier     = srcu_torture_barrier,
 680        .stats          = srcu_torture_stats,
 681        .name           = "srcu"
 682};
 683
 684static struct rcu_torture_ops srcu_sync_ops = {
 685        .init           = rcu_sync_torture_init,
 686        .readlock       = srcu_torture_read_lock,
 687        .read_delay     = srcu_read_delay,
 688        .readunlock     = srcu_torture_read_unlock,
 689        .completed      = srcu_torture_completed,
 690        .deferred_free  = rcu_sync_torture_deferred_free,
 691        .sync           = srcu_torture_synchronize,
 692        .call           = NULL,
 693        .cb_barrier     = NULL,
 694        .stats          = srcu_torture_stats,
 695        .name           = "srcu_sync"
 696};
 697
 698static int srcu_torture_read_lock_raw(void) __acquires(&srcu_ctl)
 699{
 700        return srcu_read_lock_raw(&srcu_ctl);
 701}
 702
 703static void srcu_torture_read_unlock_raw(int idx) __releases(&srcu_ctl)
 704{
 705        srcu_read_unlock_raw(&srcu_ctl, idx);
 706}
 707
 708static struct rcu_torture_ops srcu_raw_ops = {
 709        .init           = rcu_sync_torture_init,
 710        .readlock       = srcu_torture_read_lock_raw,
 711        .read_delay     = srcu_read_delay,
 712        .readunlock     = srcu_torture_read_unlock_raw,
 713        .completed      = srcu_torture_completed,
 714        .deferred_free  = srcu_torture_deferred_free,
 715        .sync           = srcu_torture_synchronize,
 716        .call           = NULL,
 717        .cb_barrier     = NULL,
 718        .stats          = srcu_torture_stats,
 719        .name           = "srcu_raw"
 720};
 721
 722static struct rcu_torture_ops srcu_raw_sync_ops = {
 723        .init           = rcu_sync_torture_init,
 724        .readlock       = srcu_torture_read_lock_raw,
 725        .read_delay     = srcu_read_delay,
 726        .readunlock     = srcu_torture_read_unlock_raw,
 727        .completed      = srcu_torture_completed,
 728        .deferred_free  = rcu_sync_torture_deferred_free,
 729        .sync           = srcu_torture_synchronize,
 730        .call           = NULL,
 731        .cb_barrier     = NULL,
 732        .stats          = srcu_torture_stats,
 733        .name           = "srcu_raw_sync"
 734};
 735
 736static void srcu_torture_synchronize_expedited(void)
 737{
 738        synchronize_srcu_expedited(&srcu_ctl);
 739}
 740
 741static struct rcu_torture_ops srcu_expedited_ops = {
 742        .init           = rcu_sync_torture_init,
 743        .readlock       = srcu_torture_read_lock,
 744        .read_delay     = srcu_read_delay,
 745        .readunlock     = srcu_torture_read_unlock,
 746        .completed      = srcu_torture_completed,
 747        .deferred_free  = rcu_sync_torture_deferred_free,
 748        .sync           = srcu_torture_synchronize_expedited,
 749        .call           = NULL,
 750        .cb_barrier     = NULL,
 751        .stats          = srcu_torture_stats,
 752        .name           = "srcu_expedited"
 753};
 754
 755/*
 756 * Definitions for sched torture testing.
 757 */
 758
 759static int sched_torture_read_lock(void)
 760{
 761        preempt_disable();
 762        return 0;
 763}
 764
 765static void sched_torture_read_unlock(int idx)
 766{
 767        preempt_enable();
 768}
 769
 770static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
 771{
 772        call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
 773}
 774
 775static struct rcu_torture_ops sched_ops = {
 776        .init           = rcu_sync_torture_init,
 777        .readlock       = sched_torture_read_lock,
 778        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 779        .readunlock     = sched_torture_read_unlock,
 780        .completed      = rcu_no_completed,
 781        .deferred_free  = rcu_sched_torture_deferred_free,
 782        .sync           = synchronize_sched,
 783        .cb_barrier     = rcu_barrier_sched,
 784        .fqs            = rcu_sched_force_quiescent_state,
 785        .stats          = NULL,
 786        .irq_capable    = 1,
 787        .name           = "sched"
 788};
 789
 790static struct rcu_torture_ops sched_sync_ops = {
 791        .init           = rcu_sync_torture_init,
 792        .readlock       = sched_torture_read_lock,
 793        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 794        .readunlock     = sched_torture_read_unlock,
 795        .completed      = rcu_no_completed,
 796        .deferred_free  = rcu_sync_torture_deferred_free,
 797        .sync           = synchronize_sched,
 798        .cb_barrier     = NULL,
 799        .fqs            = rcu_sched_force_quiescent_state,
 800        .stats          = NULL,
 801        .name           = "sched_sync"
 802};
 803
 804static struct rcu_torture_ops sched_expedited_ops = {
 805        .init           = rcu_sync_torture_init,
 806        .readlock       = sched_torture_read_lock,
 807        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 808        .readunlock     = sched_torture_read_unlock,
 809        .completed      = rcu_no_completed,
 810        .deferred_free  = rcu_sync_torture_deferred_free,
 811        .sync           = synchronize_sched_expedited,
 812        .cb_barrier     = NULL,
 813        .fqs            = rcu_sched_force_quiescent_state,
 814        .stats          = NULL,
 815        .irq_capable    = 1,
 816        .name           = "sched_expedited"
 817};
 818
 819/*
 820 * RCU torture priority-boost testing.  Runs one real-time thread per
 821 * CPU for moderate bursts, repeatedly registering RCU callbacks and
 822 * spinning waiting for them to be invoked.  If a given callback takes
 823 * too long to be invoked, we assume that priority inversion has occurred.
 824 */
 825
 826struct rcu_boost_inflight {
 827        struct rcu_head rcu;
 828        int inflight;
 829};
 830
 831static void rcu_torture_boost_cb(struct rcu_head *head)
 832{
 833        struct rcu_boost_inflight *rbip =
 834                container_of(head, struct rcu_boost_inflight, rcu);
 835
 836        smp_mb(); /* Ensure RCU-core accesses precede clearing ->inflight */
 837        rbip->inflight = 0;
 838}
 839
 840static int rcu_torture_boost(void *arg)
 841{
 842        unsigned long call_rcu_time;
 843        unsigned long endtime;
 844        unsigned long oldstarttime;
 845        struct rcu_boost_inflight rbi = { .inflight = 0 };
 846        struct sched_param sp;
 847
 848        VERBOSE_PRINTK_STRING("rcu_torture_boost started");
 849
 850        /* Set real-time priority. */
 851        sp.sched_priority = 1;
 852        if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) {
 853                VERBOSE_PRINTK_STRING("rcu_torture_boost RT prio failed!");
 854                n_rcu_torture_boost_rterror++;
 855        }
 856
 857        init_rcu_head_on_stack(&rbi.rcu);
 858        /* Each pass through the following loop does one boost-test cycle. */
 859        do {
 860                /* Wait for the next test interval. */
 861                oldstarttime = boost_starttime;
 862                while (ULONG_CMP_LT(jiffies, oldstarttime)) {
 863                        schedule_timeout_interruptible(oldstarttime - jiffies);
 864                        rcu_stutter_wait("rcu_torture_boost");
 865                        if (kthread_should_stop() ||
 866                            fullstop != FULLSTOP_DONTSTOP)
 867                                goto checkwait;
 868                }
 869
 870                /* Do one boost-test interval. */
 871                endtime = oldstarttime + test_boost_duration * HZ;
 872                call_rcu_time = jiffies;
 873                while (ULONG_CMP_LT(jiffies, endtime)) {
 874                        /* If we don't have a callback in flight, post one. */
 875                        if (!rbi.inflight) {
 876                                smp_mb(); /* RCU core before ->inflight = 1. */
 877                                rbi.inflight = 1;
 878                                call_rcu(&rbi.rcu, rcu_torture_boost_cb);
 879                                if (jiffies - call_rcu_time >
 880                                         test_boost_duration * HZ - HZ / 2) {
 881                                        VERBOSE_PRINTK_STRING("rcu_torture_boost boosting failed");
 882                                        n_rcu_torture_boost_failure++;
 883                                }
 884                                call_rcu_time = jiffies;
 885                        }
 886                        cond_resched();
 887                        rcu_stutter_wait("rcu_torture_boost");
 888                        if (kthread_should_stop() ||
 889                            fullstop != FULLSTOP_DONTSTOP)
 890                                goto checkwait;
 891                }
 892
 893                /*
 894                 * Set the start time of the next test interval.
 895                 * Yes, this is vulnerable to long delays, but such
 896                 * delays simply cause a false negative for the next
 897                 * interval.  Besides, we are running at RT priority,
 898                 * so delays should be relatively rare.
 899                 */
 900                while (oldstarttime == boost_starttime &&
 901                       !kthread_should_stop()) {
 902                        if (mutex_trylock(&boost_mutex)) {
 903                                boost_starttime = jiffies +
 904                                                  test_boost_interval * HZ;
 905                                n_rcu_torture_boosts++;
 906                                mutex_unlock(&boost_mutex);
 907                                break;
 908                        }
 909                        schedule_timeout_uninterruptible(1);
 910                }
 911
 912                /* Go do the stutter. */
 913checkwait:      rcu_stutter_wait("rcu_torture_boost");
 914        } while (!kthread_should_stop() && fullstop  == FULLSTOP_DONTSTOP);
 915
 916        /* Clean up and exit. */
 917        VERBOSE_PRINTK_STRING("rcu_torture_boost task stopping");
 918        rcutorture_shutdown_absorb("rcu_torture_boost");
 919        while (!kthread_should_stop() || rbi.inflight)
 920                schedule_timeout_uninterruptible(1);
 921        smp_mb(); /* order accesses to ->inflight before stack-frame death. */
 922        destroy_rcu_head_on_stack(&rbi.rcu);
 923        return 0;
 924}
 925
 926/*
 927 * RCU torture force-quiescent-state kthread.  Repeatedly induces
 928 * bursts of calls to force_quiescent_state(), increasing the probability
 929 * of occurrence of some important types of race conditions.
 930 */
 931static int
 932rcu_torture_fqs(void *arg)
 933{
 934        unsigned long fqs_resume_time;
 935        int fqs_burst_remaining;
 936
 937        VERBOSE_PRINTK_STRING("rcu_torture_fqs task started");
 938        do {
 939                fqs_resume_time = jiffies + fqs_stutter * HZ;
 940                while (ULONG_CMP_LT(jiffies, fqs_resume_time) &&
 941                       !kthread_should_stop()) {
 942                        schedule_timeout_interruptible(1);
 943                }
 944                fqs_burst_remaining = fqs_duration;
 945                while (fqs_burst_remaining > 0 &&
 946                       !kthread_should_stop()) {
 947                        cur_ops->fqs();
 948                        udelay(fqs_holdoff);
 949                        fqs_burst_remaining -= fqs_holdoff;
 950                }
 951                rcu_stutter_wait("rcu_torture_fqs");
 952        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
 953        VERBOSE_PRINTK_STRING("rcu_torture_fqs task stopping");
 954        rcutorture_shutdown_absorb("rcu_torture_fqs");
 955        while (!kthread_should_stop())
 956                schedule_timeout_uninterruptible(1);
 957        return 0;
 958}
 959
 960/*
 961 * RCU torture writer kthread.  Repeatedly substitutes a new structure
 962 * for that pointed to by rcu_torture_current, freeing the old structure
 963 * after a series of grace periods (the "pipeline").
 964 */
 965static int
 966rcu_torture_writer(void *arg)
 967{
 968        int i;
 969        long oldbatch = rcu_batches_completed();
 970        struct rcu_torture *rp;
 971        struct rcu_torture *old_rp;
 972        static DEFINE_RCU_RANDOM(rand);
 973
 974        VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
 975        set_user_nice(current, 19);
 976
 977        do {
 978                schedule_timeout_uninterruptible(1);
 979                rp = rcu_torture_alloc();
 980                if (rp == NULL)
 981                        continue;
 982                rp->rtort_pipe_count = 0;
 983                udelay(rcu_random(&rand) & 0x3ff);
 984                old_rp = rcu_dereference_check(rcu_torture_current,
 985                                               current == writer_task);
 986                rp->rtort_mbtest = 1;
 987                rcu_assign_pointer(rcu_torture_current, rp);
 988                smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
 989                if (old_rp) {
 990                        i = old_rp->rtort_pipe_count;
 991                        if (i > RCU_TORTURE_PIPE_LEN)
 992                                i = RCU_TORTURE_PIPE_LEN;
 993                        atomic_inc(&rcu_torture_wcount[i]);
 994                        old_rp->rtort_pipe_count++;
 995                        cur_ops->deferred_free(old_rp);
 996                }
 997                rcutorture_record_progress(++rcu_torture_current_version);
 998                oldbatch = cur_ops->completed();
 999                rcu_stutter_wait("rcu_torture_writer");
1000        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
1001        VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
1002        rcutorture_shutdown_absorb("rcu_torture_writer");
1003        while (!kthread_should_stop())
1004                schedule_timeout_uninterruptible(1);
1005        return 0;
1006}
1007
1008/*
1009 * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
1010 * delay between calls.
1011 */
1012static int
1013rcu_torture_fakewriter(void *arg)
1014{
1015        DEFINE_RCU_RANDOM(rand);
1016
1017        VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
1018        set_user_nice(current, 19);
1019
1020        do {
1021                schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
1022                udelay(rcu_random(&rand) & 0x3ff);
1023                if (cur_ops->cb_barrier != NULL &&
1024                    rcu_random(&rand) % (nfakewriters * 8) == 0)
1025                        cur_ops->cb_barrier();
1026                else
1027                        cur_ops->sync();
1028                rcu_stutter_wait("rcu_torture_fakewriter");
1029        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
1030
1031        VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
1032        rcutorture_shutdown_absorb("rcu_torture_fakewriter");
1033        while (!kthread_should_stop())
1034                schedule_timeout_uninterruptible(1);
1035        return 0;
1036}
1037
1038void rcutorture_trace_dump(void)
1039{
1040        static atomic_t beenhere = ATOMIC_INIT(0);
1041
1042        if (atomic_read(&beenhere))
1043                return;
1044        if (atomic_xchg(&beenhere, 1) != 0)
1045                return;
1046        ftrace_dump(DUMP_ALL);
1047}
1048
1049/*
1050 * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
1051 * incrementing the corresponding element of the pipeline array.  The
1052 * counter in the element should never be greater than 1, otherwise, the
1053 * RCU implementation is broken.
1054 */
1055static void rcu_torture_timer(unsigned long unused)
1056{
1057        int idx;
1058        int completed;
1059        int completed_end;
1060        static DEFINE_RCU_RANDOM(rand);
1061        static DEFINE_SPINLOCK(rand_lock);
1062        struct rcu_torture *p;
1063        int pipe_count;
1064        unsigned long long ts;
1065
1066        idx = cur_ops->readlock();
1067        completed = cur_ops->completed();
1068        ts = rcu_trace_clock_local();
1069        p = rcu_dereference_check(rcu_torture_current,
1070                                  rcu_read_lock_bh_held() ||
1071                                  rcu_read_lock_sched_held() ||
1072                                  srcu_read_lock_held(&srcu_ctl));
1073        if (p == NULL) {
1074                /* Leave because rcu_torture_writer is not yet underway */
1075                cur_ops->readunlock(idx);
1076                return;
1077        }
1078        if (p->rtort_mbtest == 0)
1079                atomic_inc(&n_rcu_torture_mberror);
1080        spin_lock(&rand_lock);
1081        cur_ops->read_delay(&rand);
1082        n_rcu_torture_timers++;
1083        spin_unlock(&rand_lock);
1084        preempt_disable();
1085        pipe_count = p->rtort_pipe_count;
1086        if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1087                /* Should not happen, but... */
1088                pipe_count = RCU_TORTURE_PIPE_LEN;
1089        }
1090        completed_end = cur_ops->completed();
1091        if (pipe_count > 1) {
1092                do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
1093                                          completed, completed_end);
1094                rcutorture_trace_dump();
1095        }
1096        __this_cpu_inc(rcu_torture_count[pipe_count]);
1097        completed = completed_end - completed;
1098        if (completed > RCU_TORTURE_PIPE_LEN) {
1099                /* Should not happen, but... */
1100                completed = RCU_TORTURE_PIPE_LEN;
1101        }
1102        __this_cpu_inc(rcu_torture_batch[completed]);
1103        preempt_enable();
1104        cur_ops->readunlock(idx);
1105}
1106
1107/*
1108 * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
1109 * incrementing the corresponding element of the pipeline array.  The
1110 * counter in the element should never be greater than 1, otherwise, the
1111 * RCU implementation is broken.
1112 */
1113static int
1114rcu_torture_reader(void *arg)
1115{
1116        int completed;
1117        int completed_end;
1118        int idx;
1119        DEFINE_RCU_RANDOM(rand);
1120        struct rcu_torture *p;
1121        int pipe_count;
1122        struct timer_list t;
1123        unsigned long long ts;
1124
1125        VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
1126        set_user_nice(current, 19);
1127        if (irqreader && cur_ops->irq_capable)
1128                setup_timer_on_stack(&t, rcu_torture_timer, 0);
1129
1130        do {
1131                if (irqreader && cur_ops->irq_capable) {
1132                        if (!timer_pending(&t))
1133                                mod_timer(&t, jiffies + 1);
1134                }
1135                idx = cur_ops->readlock();
1136                completed = cur_ops->completed();
1137                ts = rcu_trace_clock_local();
1138                p = rcu_dereference_check(rcu_torture_current,
1139                                          rcu_read_lock_bh_held() ||
1140                                          rcu_read_lock_sched_held() ||
1141                                          srcu_read_lock_held(&srcu_ctl));
1142                if (p == NULL) {
1143                        /* Wait for rcu_torture_writer to get underway */
1144                        cur_ops->readunlock(idx);
1145                        schedule_timeout_interruptible(HZ);
1146                        continue;
1147                }
1148                if (p->rtort_mbtest == 0)
1149                        atomic_inc(&n_rcu_torture_mberror);
1150                cur_ops->read_delay(&rand);
1151                preempt_disable();
1152                pipe_count = p->rtort_pipe_count;
1153                if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1154                        /* Should not happen, but... */
1155                        pipe_count = RCU_TORTURE_PIPE_LEN;
1156                }
1157                completed_end = cur_ops->completed();
1158                if (pipe_count > 1) {
1159                        do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
1160                                                  ts, completed, completed_end);
1161                        rcutorture_trace_dump();
1162                }
1163                __this_cpu_inc(rcu_torture_count[pipe_count]);
1164                completed = completed_end - completed;
1165                if (completed > RCU_TORTURE_PIPE_LEN) {
1166                        /* Should not happen, but... */
1167                        completed = RCU_TORTURE_PIPE_LEN;
1168                }
1169                __this_cpu_inc(rcu_torture_batch[completed]);
1170                preempt_enable();
1171                cur_ops->readunlock(idx);
1172                schedule();
1173                rcu_stutter_wait("rcu_torture_reader");
1174        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
1175        VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
1176        rcutorture_shutdown_absorb("rcu_torture_reader");
1177        if (irqreader && cur_ops->irq_capable)
1178                del_timer_sync(&t);
1179        while (!kthread_should_stop())
1180                schedule_timeout_uninterruptible(1);
1181        return 0;
1182}
1183
1184/*
1185 * Create an RCU-torture statistics message in the specified buffer.
1186 */
1187static int
1188rcu_torture_printk(char *page)
1189{
1190        int cnt = 0;
1191        int cpu;
1192        int i;
1193        long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1194        long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1195
1196        for_each_possible_cpu(cpu) {
1197                for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1198                        pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
1199                        batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
1200                }
1201        }
1202        for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
1203                if (pipesummary[i] != 0)
1204                        break;
1205        }
1206        cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
1207        cnt += sprintf(&page[cnt],
1208                       "rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
1209                       rcu_torture_current,
1210                       rcu_torture_current_version,
1211                       list_empty(&rcu_torture_freelist),
1212                       atomic_read(&n_rcu_torture_alloc),
1213                       atomic_read(&n_rcu_torture_alloc_fail),
1214                       atomic_read(&n_rcu_torture_free));
1215        cnt += sprintf(&page[cnt], "rtmbe: %d rtbke: %ld rtbre: %ld ",
1216                       atomic_read(&n_rcu_torture_mberror),
1217                       n_rcu_torture_boost_ktrerror,
1218                       n_rcu_torture_boost_rterror);
1219        cnt += sprintf(&page[cnt], "rtbf: %ld rtb: %ld nt: %ld ",
1220                       n_rcu_torture_boost_failure,
1221                       n_rcu_torture_boosts,
1222                       n_rcu_torture_timers);
1223        cnt += sprintf(&page[cnt],
1224                       "onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
1225                       n_online_successes, n_online_attempts,
1226                       n_offline_successes, n_offline_attempts,
1227                       min_online, max_online,
1228                       min_offline, max_offline,
1229                       sum_online, sum_offline, HZ);
1230        cnt += sprintf(&page[cnt], "barrier: %ld/%ld:%ld",
1231                       n_barrier_successes,
1232                       n_barrier_attempts,
1233                       n_rcu_torture_barrier_error);
1234        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
1235        if (atomic_read(&n_rcu_torture_mberror) != 0 ||
1236            n_rcu_torture_barrier_error != 0 ||
1237            n_rcu_torture_boost_ktrerror != 0 ||
1238            n_rcu_torture_boost_rterror != 0 ||
1239            n_rcu_torture_boost_failure != 0 ||
1240            i > 1) {
1241                cnt += sprintf(&page[cnt], "!!! ");
1242                atomic_inc(&n_rcu_torture_error);
1243                WARN_ON_ONCE(1);
1244        }
1245        cnt += sprintf(&page[cnt], "Reader Pipe: ");
1246        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1247                cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
1248        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
1249        cnt += sprintf(&page[cnt], "Reader Batch: ");
1250        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1251                cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
1252        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
1253        cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
1254        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1255                cnt += sprintf(&page[cnt], " %d",
1256                               atomic_read(&rcu_torture_wcount[i]));
1257        }
1258        cnt += sprintf(&page[cnt], "\n");
1259        if (cur_ops->stats)
1260                cnt += cur_ops->stats(&page[cnt]);
1261        return cnt;
1262}
1263
1264/*
1265 * Print torture statistics.  Caller must ensure that there is only
1266 * one call to this function at a given time!!!  This is normally
1267 * accomplished by relying on the module system to only have one copy
1268 * of the module loaded, and then by giving the rcu_torture_stats
1269 * kthread full control (or the init/cleanup functions when rcu_torture_stats
1270 * thread is not running).
1271 */
1272static void
1273rcu_torture_stats_print(void)
1274{
1275        int cnt;
1276
1277        cnt = rcu_torture_printk(printk_buf);
1278        pr_alert("%s", printk_buf);
1279}
1280
1281/*
1282 * Periodically prints torture statistics, if periodic statistics printing
1283 * was specified via the stat_interval module parameter.
1284 *
1285 * No need to worry about fullstop here, since this one doesn't reference
1286 * volatile state or register callbacks.
1287 */
1288static int
1289rcu_torture_stats(void *arg)
1290{
1291        VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
1292        do {
1293                schedule_timeout_interruptible(stat_interval * HZ);
1294                rcu_torture_stats_print();
1295                rcutorture_shutdown_absorb("rcu_torture_stats");
1296        } while (!kthread_should_stop());
1297        VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
1298        return 0;
1299}
1300
1301static int rcu_idle_cpu;        /* Force all torture tasks off this CPU */
1302
1303/* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
1304 * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
1305 */
1306static void rcu_torture_shuffle_tasks(void)
1307{
1308        int i;
1309
1310        cpumask_setall(shuffle_tmp_mask);
1311        get_online_cpus();
1312
1313        /* No point in shuffling if there is only one online CPU (ex: UP) */
1314        if (num_online_cpus() == 1) {
1315                put_online_cpus();
1316                return;
1317        }
1318
1319        if (rcu_idle_cpu != -1)
1320                cpumask_clear_cpu(rcu_idle_cpu, shuffle_tmp_mask);
1321
1322        set_cpus_allowed_ptr(current, shuffle_tmp_mask);
1323
1324        if (reader_tasks) {
1325                for (i = 0; i < nrealreaders; i++)
1326                        if (reader_tasks[i])
1327                                set_cpus_allowed_ptr(reader_tasks[i],
1328                                                     shuffle_tmp_mask);
1329        }
1330        if (fakewriter_tasks) {
1331                for (i = 0; i < nfakewriters; i++)
1332                        if (fakewriter_tasks[i])
1333                                set_cpus_allowed_ptr(fakewriter_tasks[i],
1334                                                     shuffle_tmp_mask);
1335        }
1336        if (writer_task)
1337                set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask);
1338        if (stats_task)
1339                set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask);
1340        if (stutter_task)
1341                set_cpus_allowed_ptr(stutter_task, shuffle_tmp_mask);
1342        if (fqs_task)
1343                set_cpus_allowed_ptr(fqs_task, shuffle_tmp_mask);
1344        if (shutdown_task)
1345                set_cpus_allowed_ptr(shutdown_task, shuffle_tmp_mask);
1346#ifdef CONFIG_HOTPLUG_CPU
1347        if (onoff_task)
1348                set_cpus_allowed_ptr(onoff_task, shuffle_tmp_mask);
1349#endif /* #ifdef CONFIG_HOTPLUG_CPU */
1350        if (stall_task)
1351                set_cpus_allowed_ptr(stall_task, shuffle_tmp_mask);
1352        if (barrier_cbs_tasks)
1353                for (i = 0; i < n_barrier_cbs; i++)
1354                        if (barrier_cbs_tasks[i])
1355                                set_cpus_allowed_ptr(barrier_cbs_tasks[i],
1356                                                     shuffle_tmp_mask);
1357        if (barrier_task)
1358                set_cpus_allowed_ptr(barrier_task, shuffle_tmp_mask);
1359
1360        if (rcu_idle_cpu == -1)
1361                rcu_idle_cpu = num_online_cpus() - 1;
1362        else
1363                rcu_idle_cpu--;
1364
1365        put_online_cpus();
1366}
1367
1368/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
1369 * system to become idle at a time and cut off its timer ticks. This is meant
1370 * to test the support for such tickless idle CPU in RCU.
1371 */
1372static int
1373rcu_torture_shuffle(void *arg)
1374{
1375        VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
1376        do {
1377                schedule_timeout_interruptible(shuffle_interval * HZ);
1378                rcu_torture_shuffle_tasks();
1379                rcutorture_shutdown_absorb("rcu_torture_shuffle");
1380        } while (!kthread_should_stop());
1381        VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
1382        return 0;
1383}
1384
1385/* Cause the rcutorture test to "stutter", starting and stopping all
1386 * threads periodically.
1387 */
1388static int
1389rcu_torture_stutter(void *arg)
1390{
1391        VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
1392        do {
1393                schedule_timeout_interruptible(stutter * HZ);
1394                stutter_pause_test = 1;
1395                if (!kthread_should_stop())
1396                        schedule_timeout_interruptible(stutter * HZ);
1397                stutter_pause_test = 0;
1398                rcutorture_shutdown_absorb("rcu_torture_stutter");
1399        } while (!kthread_should_stop());
1400        VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
1401        return 0;
1402}
1403
1404static inline void
1405rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, char *tag)
1406{
1407        pr_alert("%s" TORTURE_FLAG
1408                 "--- %s: nreaders=%d nfakewriters=%d "
1409                 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
1410                 "shuffle_interval=%d stutter=%d irqreader=%d "
1411                 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
1412                 "test_boost=%d/%d test_boost_interval=%d "
1413                 "test_boost_duration=%d shutdown_secs=%d "
1414                 "stall_cpu=%d stall_cpu_holdoff=%d "
1415                 "n_barrier_cbs=%d "
1416                 "onoff_interval=%d onoff_holdoff=%d\n",
1417                 torture_type, tag, nrealreaders, nfakewriters,
1418                 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
1419                 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
1420                 test_boost, cur_ops->can_boost,
1421                 test_boost_interval, test_boost_duration, shutdown_secs,
1422                 stall_cpu, stall_cpu_holdoff,
1423                 n_barrier_cbs,
1424                 onoff_interval, onoff_holdoff);
1425}
1426
1427static struct notifier_block rcutorture_shutdown_nb = {
1428        .notifier_call = rcutorture_shutdown_notify,
1429};
1430
1431static void rcutorture_booster_cleanup(int cpu)
1432{
1433        struct task_struct *t;
1434
1435        if (boost_tasks[cpu] == NULL)
1436                return;
1437        mutex_lock(&boost_mutex);
1438        VERBOSE_PRINTK_STRING("Stopping rcu_torture_boost task");
1439        t = boost_tasks[cpu];
1440        boost_tasks[cpu] = NULL;
1441        mutex_unlock(&boost_mutex);
1442
1443        /* This must be outside of the mutex, otherwise deadlock! */
1444        kthread_stop(t);
1445        boost_tasks[cpu] = NULL;
1446}
1447
1448static int rcutorture_booster_init(int cpu)
1449{
1450        int retval;
1451
1452        if (boost_tasks[cpu] != NULL)
1453                return 0;  /* Already created, nothing more to do. */
1454
1455        /* Don't allow time recalculation while creating a new task. */
1456        mutex_lock(&boost_mutex);
1457        VERBOSE_PRINTK_STRING("Creating rcu_torture_boost task");
1458        boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL,
1459                                                  cpu_to_node(cpu),
1460                                                  "rcu_torture_boost");
1461        if (IS_ERR(boost_tasks[cpu])) {
1462                retval = PTR_ERR(boost_tasks[cpu]);
1463                VERBOSE_PRINTK_STRING("rcu_torture_boost task create failed");
1464                n_rcu_torture_boost_ktrerror++;
1465                boost_tasks[cpu] = NULL;
1466                mutex_unlock(&boost_mutex);
1467                return retval;
1468        }
1469        kthread_bind(boost_tasks[cpu], cpu);
1470        wake_up_process(boost_tasks[cpu]);
1471        mutex_unlock(&boost_mutex);
1472        return 0;
1473}
1474
1475/*
1476 * Cause the rcutorture test to shutdown the system after the test has
1477 * run for the time specified by the shutdown_secs module parameter.
1478 */
1479static int
1480rcu_torture_shutdown(void *arg)
1481{
1482        long delta;
1483        unsigned long jiffies_snap;
1484
1485        VERBOSE_PRINTK_STRING("rcu_torture_shutdown task started");
1486        jiffies_snap = ACCESS_ONCE(jiffies);
1487        while (ULONG_CMP_LT(jiffies_snap, shutdown_time) &&
1488               !kthread_should_stop()) {
1489                delta = shutdown_time - jiffies_snap;
1490                if (verbose)
1491                        pr_alert("%s" TORTURE_FLAG
1492                                 "rcu_torture_shutdown task: %lu jiffies remaining\n",
1493                                 torture_type, delta);
1494                schedule_timeout_interruptible(delta);
1495                jiffies_snap = ACCESS_ONCE(jiffies);
1496        }
1497        if (kthread_should_stop()) {
1498                VERBOSE_PRINTK_STRING("rcu_torture_shutdown task stopping");
1499                return 0;
1500        }
1501
1502        /* OK, shut down the system. */
1503
1504        VERBOSE_PRINTK_STRING("rcu_torture_shutdown task shutting down system");
1505        shutdown_task = NULL;   /* Avoid self-kill deadlock. */
1506        rcu_torture_cleanup();  /* Get the success/failure message. */
1507        kernel_power_off();     /* Shut down the system. */
1508        return 0;
1509}
1510
1511#ifdef CONFIG_HOTPLUG_CPU
1512
1513/*
1514 * Execute random CPU-hotplug operations at the interval specified
1515 * by the onoff_interval.
1516 */
1517static int __cpuinit
1518rcu_torture_onoff(void *arg)
1519{
1520        int cpu;
1521        unsigned long delta;
1522        int maxcpu = -1;
1523        DEFINE_RCU_RANDOM(rand);
1524        int ret;
1525        unsigned long starttime;
1526
1527        VERBOSE_PRINTK_STRING("rcu_torture_onoff task started");
1528        for_each_online_cpu(cpu)
1529                maxcpu = cpu;
1530        WARN_ON(maxcpu < 0);
1531        if (onoff_holdoff > 0) {
1532                VERBOSE_PRINTK_STRING("rcu_torture_onoff begin holdoff");
1533                schedule_timeout_interruptible(onoff_holdoff * HZ);
1534                VERBOSE_PRINTK_STRING("rcu_torture_onoff end holdoff");
1535        }
1536        while (!kthread_should_stop()) {
1537                cpu = (rcu_random(&rand) >> 4) % (maxcpu + 1);
1538                if (cpu_online(cpu) && cpu_is_hotpluggable(cpu)) {
1539                        if (verbose)
1540                                pr_alert("%s" TORTURE_FLAG
1541                                         "rcu_torture_onoff task: offlining %d\n",
1542                                         torture_type, cpu);
1543                        starttime = jiffies;
1544                        n_offline_attempts++;
1545                        ret = cpu_down(cpu);
1546                        if (ret) {
1547                                if (verbose)
1548                                        pr_alert("%s" TORTURE_FLAG
1549                                                 "rcu_torture_onoff task: offline %d failed: errno %d\n",
1550                                                 torture_type, cpu, ret);
1551                        } else {
1552                                if (verbose)
1553                                        pr_alert("%s" TORTURE_FLAG
1554                                                 "rcu_torture_onoff task: offlined %d\n",
1555                                                 torture_type, cpu);
1556                                n_offline_successes++;
1557                                delta = jiffies - starttime;
1558                                sum_offline += delta;
1559                                if (min_offline < 0) {
1560                                        min_offline = delta;
1561                                        max_offline = delta;
1562                                }
1563                                if (min_offline > delta)
1564                                        min_offline = delta;
1565                                if (max_offline < delta)
1566                                        max_offline = delta;
1567                        }
1568                } else if (cpu_is_hotpluggable(cpu)) {
1569                        if (verbose)
1570                                pr_alert("%s" TORTURE_FLAG
1571                                         "rcu_torture_onoff task: onlining %d\n",
1572                                         torture_type, cpu);
1573                        starttime = jiffies;
1574                        n_online_attempts++;
1575                        if (cpu_up(cpu) == 0) {
1576                                if (verbose)
1577                                        pr_alert("%s" TORTURE_FLAG
1578                                                 "rcu_torture_onoff task: onlined %d\n",
1579                                                 torture_type, cpu);
1580                                n_online_successes++;
1581                                delta = jiffies - starttime;
1582                                sum_online += delta;
1583                                if (min_online < 0) {
1584                                        min_online = delta;
1585                                        max_online = delta;
1586                                }
1587                                if (min_online > delta)
1588                                        min_online = delta;
1589                                if (max_online < delta)
1590                                        max_online = delta;
1591                        }
1592                }
1593                schedule_timeout_interruptible(onoff_interval * HZ);
1594        }
1595        VERBOSE_PRINTK_STRING("rcu_torture_onoff task stopping");
1596        return 0;
1597}
1598
1599static int __cpuinit
1600rcu_torture_onoff_init(void)
1601{
1602        int ret;
1603
1604        if (onoff_interval <= 0)
1605                return 0;
1606        onoff_task = kthread_run(rcu_torture_onoff, NULL, "rcu_torture_onoff");
1607        if (IS_ERR(onoff_task)) {
1608                ret = PTR_ERR(onoff_task);
1609                onoff_task = NULL;
1610                return ret;
1611        }
1612        return 0;
1613}
1614
1615static void rcu_torture_onoff_cleanup(void)
1616{
1617        if (onoff_task == NULL)
1618                return;
1619        VERBOSE_PRINTK_STRING("Stopping rcu_torture_onoff task");
1620        kthread_stop(onoff_task);
1621        onoff_task = NULL;
1622}
1623
1624#else /* #ifdef CONFIG_HOTPLUG_CPU */
1625
1626static int
1627rcu_torture_onoff_init(void)
1628{
1629        return 0;
1630}
1631
1632static void rcu_torture_onoff_cleanup(void)
1633{
1634}
1635
1636#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
1637
1638/*
1639 * CPU-stall kthread.  It waits as specified by stall_cpu_holdoff, then
1640 * induces a CPU stall for the time specified by stall_cpu.
1641 */
1642static int __cpuinit rcu_torture_stall(void *args)
1643{
1644        unsigned long stop_at;
1645
1646        VERBOSE_PRINTK_STRING("rcu_torture_stall task started");
1647        if (stall_cpu_holdoff > 0) {
1648                VERBOSE_PRINTK_STRING("rcu_torture_stall begin holdoff");
1649                schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
1650                VERBOSE_PRINTK_STRING("rcu_torture_stall end holdoff");
1651        }
1652        if (!kthread_should_stop()) {
1653                stop_at = get_seconds() + stall_cpu;
1654                /* RCU CPU stall is expected behavior in following code. */
1655                pr_alert("rcu_torture_stall start.\n");
1656                rcu_read_lock();
1657                preempt_disable();
1658                while (ULONG_CMP_LT(get_seconds(), stop_at))
1659                        continue;  /* Induce RCU CPU stall warning. */
1660                preempt_enable();
1661                rcu_read_unlock();
1662                pr_alert("rcu_torture_stall end.\n");
1663        }
1664        rcutorture_shutdown_absorb("rcu_torture_stall");
1665        while (!kthread_should_stop())
1666                schedule_timeout_interruptible(10 * HZ);
1667        return 0;
1668}
1669
1670/* Spawn CPU-stall kthread, if stall_cpu specified. */
1671static int __init rcu_torture_stall_init(void)
1672{
1673        int ret;
1674
1675        if (stall_cpu <= 0)
1676                return 0;
1677        stall_task = kthread_run(rcu_torture_stall, NULL, "rcu_torture_stall");
1678        if (IS_ERR(stall_task)) {
1679                ret = PTR_ERR(stall_task);
1680                stall_task = NULL;
1681                return ret;
1682        }
1683        return 0;
1684}
1685
1686/* Clean up after the CPU-stall kthread, if one was spawned. */
1687static void rcu_torture_stall_cleanup(void)
1688{
1689        if (stall_task == NULL)
1690                return;
1691        VERBOSE_PRINTK_STRING("Stopping rcu_torture_stall_task.");
1692        kthread_stop(stall_task);
1693        stall_task = NULL;
1694}
1695
1696/* Callback function for RCU barrier testing. */
1697void rcu_torture_barrier_cbf(struct rcu_head *rcu)
1698{
1699        atomic_inc(&barrier_cbs_invoked);
1700}
1701
1702/* kthread function to register callbacks used to test RCU barriers. */
1703static int rcu_torture_barrier_cbs(void *arg)
1704{
1705        long myid = (long)arg;
1706        bool lastphase = 0;
1707        struct rcu_head rcu;
1708
1709        init_rcu_head_on_stack(&rcu);
1710        VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task started");
1711        set_user_nice(current, 19);
1712        do {
1713                wait_event(barrier_cbs_wq[myid],
1714                           barrier_phase != lastphase ||
1715                           kthread_should_stop() ||
1716                           fullstop != FULLSTOP_DONTSTOP);
1717                lastphase = barrier_phase;
1718                smp_mb(); /* ensure barrier_phase load before ->call(). */
1719                if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
1720                        break;
1721                cur_ops->call(&rcu, rcu_torture_barrier_cbf);
1722                if (atomic_dec_and_test(&barrier_cbs_count))
1723                        wake_up(&barrier_wq);
1724        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
1725        VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task stopping");
1726        rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");
1727        while (!kthread_should_stop())
1728                schedule_timeout_interruptible(1);
1729        cur_ops->cb_barrier();
1730        destroy_rcu_head_on_stack(&rcu);
1731        return 0;
1732}
1733
1734/* kthread function to drive and coordinate RCU barrier testing. */
1735static int rcu_torture_barrier(void *arg)
1736{
1737        int i;
1738
1739        VERBOSE_PRINTK_STRING("rcu_torture_barrier task starting");
1740        do {
1741                atomic_set(&barrier_cbs_invoked, 0);
1742                atomic_set(&barrier_cbs_count, n_barrier_cbs);
1743                smp_mb(); /* Ensure barrier_phase after prior assignments. */
1744                barrier_phase = !barrier_phase;
1745                for (i = 0; i < n_barrier_cbs; i++)
1746                        wake_up(&barrier_cbs_wq[i]);
1747                wait_event(barrier_wq,
1748                           atomic_read(&barrier_cbs_count) == 0 ||
1749                           kthread_should_stop() ||
1750                           fullstop != FULLSTOP_DONTSTOP);
1751                if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
1752                        break;
1753                n_barrier_attempts++;
1754                cur_ops->cb_barrier();
1755                if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
1756                        n_rcu_torture_barrier_error++;
1757                        WARN_ON_ONCE(1);
1758                }
1759                n_barrier_successes++;
1760                schedule_timeout_interruptible(HZ / 10);
1761        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
1762        VERBOSE_PRINTK_STRING("rcu_torture_barrier task stopping");
1763        rcutorture_shutdown_absorb("rcu_torture_barrier");
1764        while (!kthread_should_stop())
1765                schedule_timeout_interruptible(1);
1766        return 0;
1767}
1768
1769/* Initialize RCU barrier testing. */
1770static int rcu_torture_barrier_init(void)
1771{
1772        int i;
1773        int ret;
1774
1775        if (n_barrier_cbs == 0)
1776                return 0;
1777        if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
1778                pr_alert("%s" TORTURE_FLAG
1779                         " Call or barrier ops missing for %s,\n",
1780                         torture_type, cur_ops->name);
1781                pr_alert("%s" TORTURE_FLAG
1782                         " RCU barrier testing omitted from run.\n",
1783                         torture_type);
1784                return 0;
1785        }
1786        atomic_set(&barrier_cbs_count, 0);
1787        atomic_set(&barrier_cbs_invoked, 0);
1788        barrier_cbs_tasks =
1789                kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
1790                        GFP_KERNEL);
1791        barrier_cbs_wq =
1792                kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
1793                        GFP_KERNEL);
1794        if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
1795                return -ENOMEM;
1796        for (i = 0; i < n_barrier_cbs; i++) {
1797                init_waitqueue_head(&barrier_cbs_wq[i]);
1798                barrier_cbs_tasks[i] = kthread_run(rcu_torture_barrier_cbs,
1799                                                   (void *)(long)i,
1800                                                   "rcu_torture_barrier_cbs");
1801                if (IS_ERR(barrier_cbs_tasks[i])) {
1802                        ret = PTR_ERR(barrier_cbs_tasks[i]);
1803                        VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier_cbs");
1804                        barrier_cbs_tasks[i] = NULL;
1805                        return ret;
1806                }
1807        }
1808        barrier_task = kthread_run(rcu_torture_barrier, NULL,
1809                                   "rcu_torture_barrier");
1810        if (IS_ERR(barrier_task)) {
1811                ret = PTR_ERR(barrier_task);
1812                VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier");
1813                barrier_task = NULL;
1814        }
1815        return 0;
1816}
1817
1818/* Clean up after RCU barrier testing. */
1819static void rcu_torture_barrier_cleanup(void)
1820{
1821        int i;
1822
1823        if (barrier_task != NULL) {
1824                VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier task");
1825                kthread_stop(barrier_task);
1826                barrier_task = NULL;
1827        }
1828        if (barrier_cbs_tasks != NULL) {
1829                for (i = 0; i < n_barrier_cbs; i++) {
1830                        if (barrier_cbs_tasks[i] != NULL) {
1831                                VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier_cbs task");
1832                                kthread_stop(barrier_cbs_tasks[i]);
1833                                barrier_cbs_tasks[i] = NULL;
1834                        }
1835                }
1836                kfree(barrier_cbs_tasks);
1837                barrier_cbs_tasks = NULL;
1838        }
1839        if (barrier_cbs_wq != NULL) {
1840                kfree(barrier_cbs_wq);
1841                barrier_cbs_wq = NULL;
1842        }
1843}
1844
1845static int rcutorture_cpu_notify(struct notifier_block *self,
1846                                 unsigned long action, void *hcpu)
1847{
1848        long cpu = (long)hcpu;
1849
1850        switch (action) {
1851        case CPU_ONLINE:
1852        case CPU_DOWN_FAILED:
1853                (void)rcutorture_booster_init(cpu);
1854                break;
1855        case CPU_DOWN_PREPARE:
1856                rcutorture_booster_cleanup(cpu);
1857                break;
1858        default:
1859                break;
1860        }
1861        return NOTIFY_OK;
1862}
1863
1864static struct notifier_block rcutorture_cpu_nb = {
1865        .notifier_call = rcutorture_cpu_notify,
1866};
1867
1868static void
1869rcu_torture_cleanup(void)
1870{
1871        int i;
1872
1873        mutex_lock(&fullstop_mutex);
1874        rcutorture_record_test_transition();
1875        if (fullstop == FULLSTOP_SHUTDOWN) {
1876                pr_warn(/* but going down anyway, so... */
1877                       "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
1878                mutex_unlock(&fullstop_mutex);
1879                schedule_timeout_uninterruptible(10);
1880                if (cur_ops->cb_barrier != NULL)
1881                        cur_ops->cb_barrier();
1882                return;
1883        }
1884        fullstop = FULLSTOP_RMMOD;
1885        mutex_unlock(&fullstop_mutex);
1886        unregister_reboot_notifier(&rcutorture_shutdown_nb);
1887        rcu_torture_barrier_cleanup();
1888        rcu_torture_stall_cleanup();
1889        if (stutter_task) {
1890                VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
1891                kthread_stop(stutter_task);
1892        }
1893        stutter_task = NULL;
1894        if (shuffler_task) {
1895                VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
1896                kthread_stop(shuffler_task);
1897                free_cpumask_var(shuffle_tmp_mask);
1898        }
1899        shuffler_task = NULL;
1900
1901        if (writer_task) {
1902                VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
1903                kthread_stop(writer_task);
1904        }
1905        writer_task = NULL;
1906
1907        if (reader_tasks) {
1908                for (i = 0; i < nrealreaders; i++) {
1909                        if (reader_tasks[i]) {
1910                                VERBOSE_PRINTK_STRING(
1911                                        "Stopping rcu_torture_reader task");
1912                                kthread_stop(reader_tasks[i]);
1913                        }
1914                        reader_tasks[i] = NULL;
1915                }
1916                kfree(reader_tasks);
1917                reader_tasks = NULL;
1918        }
1919        rcu_torture_current = NULL;
1920
1921        if (fakewriter_tasks) {
1922                for (i = 0; i < nfakewriters; i++) {
1923                        if (fakewriter_tasks[i]) {
1924                                VERBOSE_PRINTK_STRING(
1925                                        "Stopping rcu_torture_fakewriter task");
1926                                kthread_stop(fakewriter_tasks[i]);
1927                        }
1928                        fakewriter_tasks[i] = NULL;
1929                }
1930                kfree(fakewriter_tasks);
1931                fakewriter_tasks = NULL;
1932        }
1933
1934        if (stats_task) {
1935                VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
1936                kthread_stop(stats_task);
1937        }
1938        stats_task = NULL;
1939
1940        if (fqs_task) {
1941                VERBOSE_PRINTK_STRING("Stopping rcu_torture_fqs task");
1942                kthread_stop(fqs_task);
1943        }
1944        fqs_task = NULL;
1945        if ((test_boost == 1 && cur_ops->can_boost) ||
1946            test_boost == 2) {
1947                unregister_cpu_notifier(&rcutorture_cpu_nb);
1948                for_each_possible_cpu(i)
1949                        rcutorture_booster_cleanup(i);
1950        }
1951        if (shutdown_task != NULL) {
1952                VERBOSE_PRINTK_STRING("Stopping rcu_torture_shutdown task");
1953                kthread_stop(shutdown_task);
1954        }
1955        shutdown_task = NULL;
1956        rcu_torture_onoff_cleanup();
1957
1958        /* Wait for all RCU callbacks to fire.  */
1959
1960        if (cur_ops->cb_barrier != NULL)
1961                cur_ops->cb_barrier();
1962
1963        rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
1964
1965        if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
1966                rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
1967        else if (n_online_successes != n_online_attempts ||
1968                 n_offline_successes != n_offline_attempts)
1969                rcu_torture_print_module_parms(cur_ops,
1970                                               "End of test: RCU_HOTPLUG");
1971        else
1972                rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
1973}
1974
1975static int __init
1976rcu_torture_init(void)
1977{
1978        int i;
1979        int cpu;
1980        int firsterr = 0;
1981        int retval;
1982        static struct rcu_torture_ops *torture_ops[] =
1983                { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
1984                  &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
1985                  &srcu_ops, &srcu_sync_ops, &srcu_expedited_ops,
1986                  &srcu_raw_ops, &srcu_raw_sync_ops,
1987                  &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
1988
1989        mutex_lock(&fullstop_mutex);
1990
1991        /* Process args and tell the world that the torturer is on the job. */
1992        for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1993                cur_ops = torture_ops[i];
1994                if (strcmp(torture_type, cur_ops->name) == 0)
1995                        break;
1996        }
1997        if (i == ARRAY_SIZE(torture_ops)) {
1998                pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
1999                         torture_type);
2000                pr_alert("rcu-torture types:");
2001                for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
2002                        pr_alert(" %s", torture_ops[i]->name);
2003                pr_alert("\n");
2004                mutex_unlock(&fullstop_mutex);
2005                return -EINVAL;
2006        }
2007        if (cur_ops->fqs == NULL && fqs_duration != 0) {
2008                pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
2009                fqs_duration = 0;
2010        }
2011        if (cur_ops->init)
2012                cur_ops->init(); /* no "goto unwind" prior to this point!!! */
2013
2014        if (nreaders >= 0)
2015                nrealreaders = nreaders;
2016        else
2017                nrealreaders = 2 * num_online_cpus();
2018        rcu_torture_print_module_parms(cur_ops, "Start of test");
2019        fullstop = FULLSTOP_DONTSTOP;
2020
2021        /* Set up the freelist. */
2022
2023        INIT_LIST_HEAD(&rcu_torture_freelist);
2024        for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
2025                rcu_tortures[i].rtort_mbtest = 0;
2026                list_add_tail(&rcu_tortures[i].rtort_free,
2027                              &rcu_torture_freelist);
2028        }
2029
2030        /* Initialize the statistics so that each run gets its own numbers. */
2031
2032        rcu_torture_current = NULL;
2033        rcu_torture_current_version = 0;
2034        atomic_set(&n_rcu_torture_alloc, 0);
2035        atomic_set(&n_rcu_torture_alloc_fail, 0);
2036        atomic_set(&n_rcu_torture_free, 0);
2037        atomic_set(&n_rcu_torture_mberror, 0);
2038        atomic_set(&n_rcu_torture_error, 0);
2039        n_rcu_torture_barrier_error = 0;
2040        n_rcu_torture_boost_ktrerror = 0;
2041        n_rcu_torture_boost_rterror = 0;
2042        n_rcu_torture_boost_failure = 0;
2043        n_rcu_torture_boosts = 0;
2044        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
2045                atomic_set(&rcu_torture_wcount[i], 0);
2046        for_each_possible_cpu(cpu) {
2047                for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
2048                        per_cpu(rcu_torture_count, cpu)[i] = 0;
2049                        per_cpu(rcu_torture_batch, cpu)[i] = 0;
2050                }
2051        }
2052
2053        /* Start up the kthreads. */
2054
2055        VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
2056        writer_task = kthread_create(rcu_torture_writer, NULL,
2057                                     "rcu_torture_writer");
2058        if (IS_ERR(writer_task)) {
2059                firsterr = PTR_ERR(writer_task);
2060                VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
2061                writer_task = NULL;
2062                goto unwind;
2063        }
2064        wake_up_process(writer_task);
2065        fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
2066                                   GFP_KERNEL);
2067        if (fakewriter_tasks == NULL) {
2068                VERBOSE_PRINTK_ERRSTRING("out of memory");
2069                firsterr = -ENOMEM;
2070                goto unwind;
2071        }
2072        for (i = 0; i < nfakewriters; i++) {
2073                VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
2074                fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
2075                                                  "rcu_torture_fakewriter");
2076                if (IS_ERR(fakewriter_tasks[i])) {
2077                        firsterr = PTR_ERR(fakewriter_tasks[i]);
2078                        VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
2079                        fakewriter_tasks[i] = NULL;
2080                        goto unwind;
2081                }
2082        }
2083        reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
2084                               GFP_KERNEL);
2085        if (reader_tasks == NULL) {
2086                VERBOSE_PRINTK_ERRSTRING("out of memory");
2087                firsterr = -ENOMEM;
2088                goto unwind;
2089        }
2090        for (i = 0; i < nrealreaders; i++) {
2091                VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
2092                reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
2093                                              "rcu_torture_reader");
2094                if (IS_ERR(reader_tasks[i])) {
2095                        firsterr = PTR_ERR(reader_tasks[i]);
2096                        VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
2097                        reader_tasks[i] = NULL;
2098                        goto unwind;
2099                }
2100        }
2101        if (stat_interval > 0) {
2102                VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
2103                stats_task = kthread_run(rcu_torture_stats, NULL,
2104                                        "rcu_torture_stats");
2105                if (IS_ERR(stats_task)) {
2106                        firsterr = PTR_ERR(stats_task);
2107                        VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
2108                        stats_task = NULL;
2109                        goto unwind;
2110                }
2111        }
2112        if (test_no_idle_hz) {
2113                rcu_idle_cpu = num_online_cpus() - 1;
2114
2115                if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
2116                        firsterr = -ENOMEM;
2117                        VERBOSE_PRINTK_ERRSTRING("Failed to alloc mask");
2118                        goto unwind;
2119                }
2120
2121                /* Create the shuffler thread */
2122                shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
2123                                          "rcu_torture_shuffle");
2124                if (IS_ERR(shuffler_task)) {
2125                        free_cpumask_var(shuffle_tmp_mask);
2126                        firsterr = PTR_ERR(shuffler_task);
2127                        VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
2128                        shuffler_task = NULL;
2129                        goto unwind;
2130                }
2131        }
2132        if (stutter < 0)
2133                stutter = 0;
2134        if (stutter) {
2135                /* Create the stutter thread */
2136                stutter_task = kthread_run(rcu_torture_stutter, NULL,
2137                                          "rcu_torture_stutter");
2138                if (IS_ERR(stutter_task)) {
2139                        firsterr = PTR_ERR(stutter_task);
2140                        VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
2141                        stutter_task = NULL;
2142                        goto unwind;
2143                }
2144        }
2145        if (fqs_duration < 0)
2146                fqs_duration = 0;
2147        if (fqs_duration) {
2148                /* Create the stutter thread */
2149                fqs_task = kthread_run(rcu_torture_fqs, NULL,
2150                                       "rcu_torture_fqs");
2151                if (IS_ERR(fqs_task)) {
2152                        firsterr = PTR_ERR(fqs_task);
2153                        VERBOSE_PRINTK_ERRSTRING("Failed to create fqs");
2154                        fqs_task = NULL;
2155                        goto unwind;
2156                }
2157        }
2158        if (test_boost_interval < 1)
2159                test_boost_interval = 1;
2160        if (test_boost_duration < 2)
2161                test_boost_duration = 2;
2162        if ((test_boost == 1 && cur_ops->can_boost) ||
2163            test_boost == 2) {
2164
2165                boost_starttime = jiffies + test_boost_interval * HZ;
2166                register_cpu_notifier(&rcutorture_cpu_nb);
2167                for_each_possible_cpu(i) {
2168                        if (cpu_is_offline(i))
2169                                continue;  /* Heuristic: CPU can go offline. */
2170                        retval = rcutorture_booster_init(i);
2171                        if (retval < 0) {
2172                                firsterr = retval;
2173                                goto unwind;
2174                        }
2175                }
2176        }
2177        if (shutdown_secs > 0) {
2178                shutdown_time = jiffies + shutdown_secs * HZ;
2179                shutdown_task = kthread_create(rcu_torture_shutdown, NULL,
2180                                               "rcu_torture_shutdown");
2181                if (IS_ERR(shutdown_task)) {
2182                        firsterr = PTR_ERR(shutdown_task);
2183                        VERBOSE_PRINTK_ERRSTRING("Failed to create shutdown");
2184                        shutdown_task = NULL;
2185                        goto unwind;
2186                }
2187                wake_up_process(shutdown_task);
2188        }
2189        i = rcu_torture_onoff_init();
2190        if (i != 0) {
2191                firsterr = i;
2192                goto unwind;
2193        }
2194        register_reboot_notifier(&rcutorture_shutdown_nb);
2195        i = rcu_torture_stall_init();
2196        if (i != 0) {
2197                firsterr = i;
2198                goto unwind;
2199        }
2200        retval = rcu_torture_barrier_init();
2201        if (retval != 0) {
2202                firsterr = retval;
2203                goto unwind;
2204        }
2205        rcutorture_record_test_transition();
2206        mutex_unlock(&fullstop_mutex);
2207        return 0;
2208
2209unwind:
2210        mutex_unlock(&fullstop_mutex);
2211        rcu_torture_cleanup();
2212        return firsterr;
2213}
2214
2215module_init(rcu_torture_init);
2216module_exit(rcu_torture_cleanup);
2217
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.