linux/kernel/rcutorture.c
<<
>>
Prefs
   1/*
   2 * Read-Copy Update module-based torture test facility
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License as published by
   6 * the Free Software Foundation; either version 2 of the License, or
   7 * (at your option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write to the Free Software
  16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17 *
  18 * Copyright (C) IBM Corporation, 2005, 2006
  19 *
  20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21 *        Josh Triplett <josh@freedesktop.org>
  22 *
  23 * See also:  Documentation/RCU/torture.txt
  24 */
  25#include <linux/types.h>
  26#include <linux/kernel.h>
  27#include <linux/init.h>
  28#include <linux/module.h>
  29#include <linux/kthread.h>
  30#include <linux/err.h>
  31#include <linux/spinlock.h>
  32#include <linux/smp.h>
  33#include <linux/rcupdate.h>
  34#include <linux/interrupt.h>
  35#include <linux/sched.h>
  36#include <linux/atomic.h>
  37#include <linux/bitops.h>
  38#include <linux/completion.h>
  39#include <linux/moduleparam.h>
  40#include <linux/percpu.h>
  41#include <linux/notifier.h>
  42#include <linux/reboot.h>
  43#include <linux/freezer.h>
  44#include <linux/cpu.h>
  45#include <linux/delay.h>
  46#include <linux/stat.h>
  47#include <linux/srcu.h>
  48#include <linux/slab.h>
  49#include <linux/trace_clock.h>
  50#include <asm/byteorder.h>
  51
  52MODULE_LICENSE("GPL");
  53MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@freedesktop.org>");
  54
  55static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
  56static int nfakewriters = 4;    /* # fake writer threads */
  57static int stat_interval = 60;  /* Interval between stats, in seconds. */
  58                                /*  Zero means "only at end of test". */
  59static bool verbose;            /* Print more debug info. */
  60static bool test_no_idle_hz = true;
  61                                /* Test RCU support for tickless idle CPUs. */
  62static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
  63static int stutter = 5;         /* Start/stop testing interval (in sec) */
  64static int irqreader = 1;       /* RCU readers from irq (timers). */
  65static int fqs_duration;        /* Duration of bursts (us), 0 to disable. */
  66static int fqs_holdoff;         /* Hold time within burst (us). */
  67static int fqs_stutter = 3;     /* Wait time between bursts (s). */
  68static int n_barrier_cbs;       /* Number of callbacks to test RCU barriers. */
  69static int onoff_interval;      /* Wait time between CPU hotplugs, 0=disable. */
  70static int onoff_holdoff;       /* Seconds after boot before CPU hotplugs. */
  71static int shutdown_secs;       /* Shutdown time (s).  <=0 for no shutdown. */
  72static int stall_cpu;           /* CPU-stall duration (s).  0 for no stall. */
  73static int stall_cpu_holdoff = 10; /* Time to wait until stall (s).  */
  74static int test_boost = 1;      /* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */
  75static int test_boost_interval = 7; /* Interval between boost tests, seconds. */
  76static int test_boost_duration = 4; /* Duration of each boost test, seconds. */
  77static char *torture_type = "rcu"; /* What RCU implementation to torture. */
  78
  79module_param(nreaders, int, 0444);
  80MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
  81module_param(nfakewriters, int, 0444);
  82MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
  83module_param(stat_interval, int, 0644);
  84MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
  85module_param(verbose, bool, 0444);
  86MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
  87module_param(test_no_idle_hz, bool, 0444);
  88MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
  89module_param(shuffle_interval, int, 0444);
  90MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
  91module_param(stutter, int, 0444);
  92MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
  93module_param(irqreader, int, 0444);
  94MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
  95module_param(fqs_duration, int, 0444);
  96MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");
  97module_param(fqs_holdoff, int, 0444);
  98MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
  99module_param(fqs_stutter, int, 0444);
 100MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
 101module_param(n_barrier_cbs, int, 0444);
 102MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");
 103module_param(onoff_interval, int, 0444);
 104MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
 105module_param(onoff_holdoff, int, 0444);
 106MODULE_PARM_DESC(onoff_holdoff, "Time after boot before CPU hotplugs (s)");
 107module_param(shutdown_secs, int, 0444);
 108MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), zero to disable.");
 109module_param(stall_cpu, int, 0444);
 110MODULE_PARM_DESC(stall_cpu, "Stall duration (s), zero to disable.");
 111module_param(stall_cpu_holdoff, int, 0444);
 112MODULE_PARM_DESC(stall_cpu_holdoff, "Time to wait before starting stall (s).");
 113module_param(test_boost, int, 0444);
 114MODULE_PARM_DESC(test_boost, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
 115module_param(test_boost_interval, int, 0444);
 116MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds.");
 117module_param(test_boost_duration, int, 0444);
 118MODULE_PARM_DESC(test_boost_duration, "Duration of each boost test, seconds.");
 119module_param(torture_type, charp, 0444);
 120MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
 121
 122#define TORTURE_FLAG "-torture:"
 123#define PRINTK_STRING(s) \
 124        do { pr_alert("%s" TORTURE_FLAG s "\n", torture_type); } while (0)
 125#define VERBOSE_PRINTK_STRING(s) \
 126        do { if (verbose) pr_alert("%s" TORTURE_FLAG s "\n", torture_type); } while (0)
 127#define VERBOSE_PRINTK_ERRSTRING(s) \
 128        do { if (verbose) pr_alert("%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
 129
 130static char printk_buf[4096];
 131
 132static int nrealreaders;
 133static struct task_struct *writer_task;
 134static struct task_struct **fakewriter_tasks;
 135static struct task_struct **reader_tasks;
 136static struct task_struct *stats_task;
 137static struct task_struct *shuffler_task;
 138static struct task_struct *stutter_task;
 139static struct task_struct *fqs_task;
 140static struct task_struct *boost_tasks[NR_CPUS];
 141static struct task_struct *shutdown_task;
 142#ifdef CONFIG_HOTPLUG_CPU
 143static struct task_struct *onoff_task;
 144#endif /* #ifdef CONFIG_HOTPLUG_CPU */
 145static struct task_struct *stall_task;
 146static struct task_struct **barrier_cbs_tasks;
 147static struct task_struct *barrier_task;
 148
 149#define RCU_TORTURE_PIPE_LEN 10
 150
 151struct rcu_torture {
 152        struct rcu_head rtort_rcu;
 153        int rtort_pipe_count;
 154        struct list_head rtort_free;
 155        int rtort_mbtest;
 156};
 157
 158static LIST_HEAD(rcu_torture_freelist);
 159static struct rcu_torture __rcu *rcu_torture_current;
 160static unsigned long rcu_torture_current_version;
 161static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 162static DEFINE_SPINLOCK(rcu_torture_lock);
 163static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
 164        { 0 };
 165static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
 166        { 0 };
 167static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
 168static atomic_t n_rcu_torture_alloc;
 169static atomic_t n_rcu_torture_alloc_fail;
 170static atomic_t n_rcu_torture_free;
 171static atomic_t n_rcu_torture_mberror;
 172static atomic_t n_rcu_torture_error;
 173static long n_rcu_torture_barrier_error;
 174static long n_rcu_torture_boost_ktrerror;
 175static long n_rcu_torture_boost_rterror;
 176static long n_rcu_torture_boost_failure;
 177static long n_rcu_torture_boosts;
 178static long n_rcu_torture_timers;
 179static long n_offline_attempts;
 180static long n_offline_successes;
 181static unsigned long sum_offline;
 182static int min_offline = -1;
 183static int max_offline;
 184static long n_online_attempts;
 185static long n_online_successes;
 186static unsigned long sum_online;
 187static int min_online = -1;
 188static int max_online;
 189static long n_barrier_attempts;
 190static long n_barrier_successes;
 191static struct list_head rcu_torture_removed;
 192static cpumask_var_t shuffle_tmp_mask;
 193
 194static int stutter_pause_test;
 195
 196#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
 197#define RCUTORTURE_RUNNABLE_INIT 1
 198#else
 199#define RCUTORTURE_RUNNABLE_INIT 0
 200#endif
 201int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
 202module_param(rcutorture_runnable, int, 0444);
 203MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot");
 204
 205#if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
 206#define rcu_can_boost() 1
 207#else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
 208#define rcu_can_boost() 0
 209#endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
 210
 211#ifdef CONFIG_RCU_TRACE
 212static u64 notrace rcu_trace_clock_local(void)
 213{
 214        u64 ts = trace_clock_local();
 215        unsigned long __maybe_unused ts_rem = do_div(ts, NSEC_PER_USEC);
 216        return ts;
 217}
 218#else /* #ifdef CONFIG_RCU_TRACE */
 219static u64 notrace rcu_trace_clock_local(void)
 220{
 221        return 0ULL;
 222}
 223#endif /* #else #ifdef CONFIG_RCU_TRACE */
 224
 225static unsigned long shutdown_time;     /* jiffies to system shutdown. */
 226static unsigned long boost_starttime;   /* jiffies of next boost test start. */
 227DEFINE_MUTEX(boost_mutex);              /* protect setting boost_starttime */
 228                                        /*  and boost task create/destroy. */
 229static atomic_t barrier_cbs_count;      /* Barrier callbacks registered. */
 230static bool barrier_phase;              /* Test phase. */
 231static atomic_t barrier_cbs_invoked;    /* Barrier callbacks invoked. */
 232static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
 233static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
 234
 235/* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */
 236
 237#define FULLSTOP_DONTSTOP 0     /* Normal operation. */
 238#define FULLSTOP_SHUTDOWN 1     /* System shutdown with rcutorture running. */
 239#define FULLSTOP_RMMOD    2     /* Normal rmmod of rcutorture. */
 240static int fullstop = FULLSTOP_RMMOD;
 241/*
 242 * Protect fullstop transitions and spawning of kthreads.
 243 */
 244static DEFINE_MUTEX(fullstop_mutex);
 245
 246/* Forward reference. */
 247static void rcu_torture_cleanup(void);
 248
 249/*
 250 * Detect and respond to a system shutdown.
 251 */
 252static int
 253rcutorture_shutdown_notify(struct notifier_block *unused1,
 254                           unsigned long unused2, void *unused3)
 255{
 256        mutex_lock(&fullstop_mutex);
 257        if (fullstop == FULLSTOP_DONTSTOP)
 258                fullstop = FULLSTOP_SHUTDOWN;
 259        else
 260                pr_warn(/* but going down anyway, so... */
 261                       "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
 262        mutex_unlock(&fullstop_mutex);
 263        return NOTIFY_DONE;
 264}
 265
 266/*
 267 * Absorb kthreads into a kernel function that won't return, so that
 268 * they won't ever access module text or data again.
 269 */
 270static void rcutorture_shutdown_absorb(char *title)
 271{
 272        if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
 273                pr_notice(
 274                       "rcutorture thread %s parking due to system shutdown\n",
 275                       title);
 276                schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
 277        }
 278}
 279
 280/*
 281 * Allocate an element from the rcu_tortures pool.
 282 */
 283static struct rcu_torture *
 284rcu_torture_alloc(void)
 285{
 286        struct list_head *p;
 287
 288        spin_lock_bh(&rcu_torture_lock);
 289        if (list_empty(&rcu_torture_freelist)) {
 290                atomic_inc(&n_rcu_torture_alloc_fail);
 291                spin_unlock_bh(&rcu_torture_lock);
 292                return NULL;
 293        }
 294        atomic_inc(&n_rcu_torture_alloc);
 295        p = rcu_torture_freelist.next;
 296        list_del_init(p);
 297        spin_unlock_bh(&rcu_torture_lock);
 298        return container_of(p, struct rcu_torture, rtort_free);
 299}
 300
 301/*
 302 * Free an element to the rcu_tortures pool.
 303 */
 304static void
 305rcu_torture_free(struct rcu_torture *p)
 306{
 307        atomic_inc(&n_rcu_torture_free);
 308        spin_lock_bh(&rcu_torture_lock);
 309        list_add_tail(&p->rtort_free, &rcu_torture_freelist);
 310        spin_unlock_bh(&rcu_torture_lock);
 311}
 312
 313struct rcu_random_state {
 314        unsigned long rrs_state;
 315        long rrs_count;
 316};
 317
 318#define RCU_RANDOM_MULT 39916801  /* prime */
 319#define RCU_RANDOM_ADD  479001701 /* prime */
 320#define RCU_RANDOM_REFRESH 10000
 321
 322#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
 323
 324/*
 325 * Crude but fast random-number generator.  Uses a linear congruential
 326 * generator, with occasional help from cpu_clock().
 327 */
 328static unsigned long
 329rcu_random(struct rcu_random_state *rrsp)
 330{
 331        if (--rrsp->rrs_count < 0) {
 332                rrsp->rrs_state += (unsigned long)local_clock();
 333                rrsp->rrs_count = RCU_RANDOM_REFRESH;
 334        }
 335        rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
 336        return swahw32(rrsp->rrs_state);
 337}
 338
 339static void
 340rcu_stutter_wait(char *title)
 341{
 342        while (stutter_pause_test || !rcutorture_runnable) {
 343                if (rcutorture_runnable)
 344                        schedule_timeout_interruptible(1);
 345                else
 346                        schedule_timeout_interruptible(round_jiffies_relative(HZ));
 347                rcutorture_shutdown_absorb(title);
 348        }
 349}
 350
 351/*
 352 * Operations vector for selecting different types of tests.
 353 */
 354
 355struct rcu_torture_ops {
 356        void (*init)(void);
 357        int (*readlock)(void);
 358        void (*read_delay)(struct rcu_random_state *rrsp);
 359        void (*readunlock)(int idx);
 360        int (*completed)(void);
 361        void (*deferred_free)(struct rcu_torture *p);
 362        void (*sync)(void);
 363        void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
 364        void (*cb_barrier)(void);
 365        void (*fqs)(void);
 366        int (*stats)(char *page);
 367        int irq_capable;
 368        int can_boost;
 369        char *name;
 370};
 371
 372static struct rcu_torture_ops *cur_ops;
 373
 374/*
 375 * Definitions for rcu torture testing.
 376 */
 377
 378static int rcu_torture_read_lock(void) __acquires(RCU)
 379{
 380        rcu_read_lock();
 381        return 0;
 382}
 383
 384static void rcu_read_delay(struct rcu_random_state *rrsp)
 385{
 386        const unsigned long shortdelay_us = 200;
 387        const unsigned long longdelay_ms = 50;
 388
 389        /* We want a short delay sometimes to make a reader delay the grace
 390         * period, and we want a long delay occasionally to trigger
 391         * force_quiescent_state. */
 392
 393        if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))
 394                mdelay(longdelay_ms);
 395        if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
 396                udelay(shortdelay_us);
 397#ifdef CONFIG_PREEMPT
 398        if (!preempt_count() && !(rcu_random(rrsp) % (nrealreaders * 20000)))
 399                preempt_schedule();  /* No QS if preempt_disable() in effect */
 400#endif
 401}
 402
 403static void rcu_torture_read_unlock(int idx) __releases(RCU)
 404{
 405        rcu_read_unlock();
 406}
 407
 408static int rcu_torture_completed(void)
 409{
 410        return rcu_batches_completed();
 411}
 412
 413static void
 414rcu_torture_cb(struct rcu_head *p)
 415{
 416        int i;
 417        struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
 418
 419        if (fullstop != FULLSTOP_DONTSTOP) {
 420                /* Test is ending, just drop callbacks on the floor. */
 421                /* The next initialization will pick up the pieces. */
 422                return;
 423        }
 424        i = rp->rtort_pipe_count;
 425        if (i > RCU_TORTURE_PIPE_LEN)
 426                i = RCU_TORTURE_PIPE_LEN;
 427        atomic_inc(&rcu_torture_wcount[i]);
 428        if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 429                rp->rtort_mbtest = 0;
 430                rcu_torture_free(rp);
 431        } else {
 432                cur_ops->deferred_free(rp);
 433        }
 434}
 435
 436static int rcu_no_completed(void)
 437{
 438        return 0;
 439}
 440
 441static void rcu_torture_deferred_free(struct rcu_torture *p)
 442{
 443        call_rcu(&p->rtort_rcu, rcu_torture_cb);
 444}
 445
 446static struct rcu_torture_ops rcu_ops = {
 447        .init           = NULL,
 448        .readlock       = rcu_torture_read_lock,
 449        .read_delay     = rcu_read_delay,
 450        .readunlock     = rcu_torture_read_unlock,
 451        .completed      = rcu_torture_completed,
 452        .deferred_free  = rcu_torture_deferred_free,
 453        .sync           = synchronize_rcu,
 454        .call           = call_rcu,
 455        .cb_barrier     = rcu_barrier,
 456        .fqs            = rcu_force_quiescent_state,
 457        .stats          = NULL,
 458        .irq_capable    = 1,
 459        .can_boost      = rcu_can_boost(),
 460        .name           = "rcu"
 461};
 462
 463static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
 464{
 465        int i;
 466        struct rcu_torture *rp;
 467        struct rcu_torture *rp1;
 468
 469        cur_ops->sync();
 470        list_add(&p->rtort_free, &rcu_torture_removed);
 471        list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
 472                i = rp->rtort_pipe_count;
 473                if (i > RCU_TORTURE_PIPE_LEN)
 474                        i = RCU_TORTURE_PIPE_LEN;
 475                atomic_inc(&rcu_torture_wcount[i]);
 476                if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 477                        rp->rtort_mbtest = 0;
 478                        list_del(&rp->rtort_free);
 479                        rcu_torture_free(rp);
 480                }
 481        }
 482}
 483
 484static void rcu_sync_torture_init(void)
 485{
 486        INIT_LIST_HEAD(&rcu_torture_removed);
 487}
 488
 489static struct rcu_torture_ops rcu_sync_ops = {
 490        .init           = rcu_sync_torture_init,
 491        .readlock       = rcu_torture_read_lock,
 492        .read_delay     = rcu_read_delay,
 493        .readunlock     = rcu_torture_read_unlock,
 494        .completed      = rcu_torture_completed,
 495        .deferred_free  = rcu_sync_torture_deferred_free,
 496        .sync           = synchronize_rcu,
 497        .call           = NULL,
 498        .cb_barrier     = NULL,
 499        .fqs            = rcu_force_quiescent_state,
 500        .stats          = NULL,
 501        .irq_capable    = 1,
 502        .can_boost      = rcu_can_boost(),
 503        .name           = "rcu_sync"
 504};
 505
 506static struct rcu_torture_ops rcu_expedited_ops = {
 507        .init           = rcu_sync_torture_init,
 508        .readlock       = rcu_torture_read_lock,
 509        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 510        .readunlock     = rcu_torture_read_unlock,
 511        .completed      = rcu_no_completed,
 512        .deferred_free  = rcu_sync_torture_deferred_free,
 513        .sync           = synchronize_rcu_expedited,
 514        .call           = NULL,
 515        .cb_barrier     = NULL,
 516        .fqs            = rcu_force_quiescent_state,
 517        .stats          = NULL,
 518        .irq_capable    = 1,
 519        .can_boost      = rcu_can_boost(),
 520        .name           = "rcu_expedited"
 521};
 522
 523/*
 524 * Definitions for rcu_bh torture testing.
 525 */
 526
 527static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
 528{
 529        rcu_read_lock_bh();
 530        return 0;
 531}
 532
 533static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
 534{
 535        rcu_read_unlock_bh();
 536}
 537
 538static int rcu_bh_torture_completed(void)
 539{
 540        return rcu_batches_completed_bh();
 541}
 542
 543static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 544{
 545        call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
 546}
 547
 548static struct rcu_torture_ops rcu_bh_ops = {
 549        .init           = NULL,
 550        .readlock       = rcu_bh_torture_read_lock,
 551        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 552        .readunlock     = rcu_bh_torture_read_unlock,
 553        .completed      = rcu_bh_torture_completed,
 554        .deferred_free  = rcu_bh_torture_deferred_free,
 555        .sync           = synchronize_rcu_bh,
 556        .call           = call_rcu_bh,
 557        .cb_barrier     = rcu_barrier_bh,
 558        .fqs            = rcu_bh_force_quiescent_state,
 559        .stats          = NULL,
 560        .irq_capable    = 1,
 561        .name           = "rcu_bh"
 562};
 563
 564static struct rcu_torture_ops rcu_bh_sync_ops = {
 565        .init           = rcu_sync_torture_init,
 566        .readlock       = rcu_bh_torture_read_lock,
 567        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 568        .readunlock     = rcu_bh_torture_read_unlock,
 569        .completed      = rcu_bh_torture_completed,
 570        .deferred_free  = rcu_sync_torture_deferred_free,
 571        .sync           = synchronize_rcu_bh,
 572        .call           = NULL,
 573        .cb_barrier     = NULL,
 574        .fqs            = rcu_bh_force_quiescent_state,
 575        .stats          = NULL,
 576        .irq_capable    = 1,
 577        .name           = "rcu_bh_sync"
 578};
 579
 580static struct rcu_torture_ops rcu_bh_expedited_ops = {
 581        .init           = rcu_sync_torture_init,
 582        .readlock       = rcu_bh_torture_read_lock,
 583        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 584        .readunlock     = rcu_bh_torture_read_unlock,
 585        .completed      = rcu_bh_torture_completed,
 586        .deferred_free  = rcu_sync_torture_deferred_free,
 587        .sync           = synchronize_rcu_bh_expedited,
 588        .call           = NULL,
 589        .cb_barrier     = NULL,
 590        .fqs            = rcu_bh_force_quiescent_state,
 591        .stats          = NULL,
 592        .irq_capable    = 1,
 593        .name           = "rcu_bh_expedited"
 594};
 595
 596/*
 597 * Definitions for srcu torture testing.
 598 */
 599
 600DEFINE_STATIC_SRCU(srcu_ctl);
 601
 602static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
 603{
 604        return srcu_read_lock(&srcu_ctl);
 605}
 606
 607static void srcu_read_delay(struct rcu_random_state *rrsp)
 608{
 609        long delay;
 610        const long uspertick = 1000000 / HZ;
 611        const long longdelay = 10;
 612
 613        /* We want there to be long-running readers, but not all the time. */
 614
 615        delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
 616        if (!delay)
 617                schedule_timeout_interruptible(longdelay);
 618        else
 619                rcu_read_delay(rrsp);
 620}
 621
 622static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
 623{
 624        srcu_read_unlock(&srcu_ctl, idx);
 625}
 626
 627static int srcu_torture_completed(void)
 628{
 629        return srcu_batches_completed(&srcu_ctl);
 630}
 631
 632static void srcu_torture_deferred_free(struct rcu_torture *rp)
 633{
 634        call_srcu(&srcu_ctl, &rp->rtort_rcu, rcu_torture_cb);
 635}
 636
 637static void srcu_torture_synchronize(void)
 638{
 639        synchronize_srcu(&srcu_ctl);
 640}
 641
 642static void srcu_torture_call(struct rcu_head *head,
 643                              void (*func)(struct rcu_head *head))
 644{
 645        call_srcu(&srcu_ctl, head, func);
 646}
 647
 648static void srcu_torture_barrier(void)
 649{
 650        srcu_barrier(&srcu_ctl);
 651}
 652
 653static int srcu_torture_stats(char *page)
 654{
 655        int cnt = 0;
 656        int cpu;
 657        int idx = srcu_ctl.completed & 0x1;
 658
 659        cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
 660                       torture_type, TORTURE_FLAG, idx);
 661        for_each_possible_cpu(cpu) {
 662                cnt += sprintf(&page[cnt], " %d(%lu,%lu)", cpu,
 663                               per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
 664                               per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
 665        }
 666        cnt += sprintf(&page[cnt], "\n");
 667        return cnt;
 668}
 669
 670static struct rcu_torture_ops srcu_ops = {
 671        .init           = rcu_sync_torture_init,
 672        .readlock       = srcu_torture_read_lock,
 673        .read_delay     = srcu_read_delay,
 674        .readunlock     = srcu_torture_read_unlock,
 675        .completed      = srcu_torture_completed,
 676        .deferred_free  = srcu_torture_deferred_free,
 677        .sync           = srcu_torture_synchronize,
 678        .call           = srcu_torture_call,
 679        .cb_barrier     = srcu_torture_barrier,
 680        .stats          = srcu_torture_stats,
 681        .name           = "srcu"
 682};
 683
 684static struct rcu_torture_ops srcu_sync_ops = {
 685        .init           = rcu_sync_torture_init,
 686        .readlock       = srcu_torture_read_lock,
 687        .read_delay     = srcu_read_delay,
 688        .readunlock     = srcu_torture_read_unlock,
 689        .completed      = srcu_torture_completed,
 690        .deferred_free  = rcu_sync_torture_deferred_free,
 691        .sync           = srcu_torture_synchronize,
 692        .call           = NULL,
 693        .cb_barrier     = NULL,
 694        .stats          = srcu_torture_stats,
 695        .name           = "srcu_sync"
 696};
 697
 698static void srcu_torture_synchronize_expedited(void)
 699{
 700        synchronize_srcu_expedited(&srcu_ctl);
 701}
 702
 703static struct rcu_torture_ops srcu_expedited_ops = {
 704        .init           = rcu_sync_torture_init,
 705        .readlock       = srcu_torture_read_lock,
 706        .read_delay     = srcu_read_delay,
 707        .readunlock     = srcu_torture_read_unlock,
 708        .completed      = srcu_torture_completed,
 709        .deferred_free  = rcu_sync_torture_deferred_free,
 710        .sync           = srcu_torture_synchronize_expedited,
 711        .call           = NULL,
 712        .cb_barrier     = NULL,
 713        .stats          = srcu_torture_stats,
 714        .name           = "srcu_expedited"
 715};
 716
 717/*
 718 * Definitions for sched torture testing.
 719 */
 720
 721static int sched_torture_read_lock(void)
 722{
 723        preempt_disable();
 724        return 0;
 725}
 726
 727static void sched_torture_read_unlock(int idx)
 728{
 729        preempt_enable();
 730}
 731
 732static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
 733{
 734        call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
 735}
 736
 737static struct rcu_torture_ops sched_ops = {
 738        .init           = rcu_sync_torture_init,
 739        .readlock       = sched_torture_read_lock,
 740        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 741        .readunlock     = sched_torture_read_unlock,
 742        .completed      = rcu_no_completed,
 743        .deferred_free  = rcu_sched_torture_deferred_free,
 744        .sync           = synchronize_sched,
 745        .cb_barrier     = rcu_barrier_sched,
 746        .fqs            = rcu_sched_force_quiescent_state,
 747        .stats          = NULL,
 748        .irq_capable    = 1,
 749        .name           = "sched"
 750};
 751
 752static struct rcu_torture_ops sched_sync_ops = {
 753        .init           = rcu_sync_torture_init,
 754        .readlock       = sched_torture_read_lock,
 755        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 756        .readunlock     = sched_torture_read_unlock,
 757        .completed      = rcu_no_completed,
 758        .deferred_free  = rcu_sync_torture_deferred_free,
 759        .sync           = synchronize_sched,
 760        .cb_barrier     = NULL,
 761        .fqs            = rcu_sched_force_quiescent_state,
 762        .stats          = NULL,
 763        .name           = "sched_sync"
 764};
 765
 766static struct rcu_torture_ops sched_expedited_ops = {
 767        .init           = rcu_sync_torture_init,
 768        .readlock       = sched_torture_read_lock,
 769        .read_delay     = rcu_read_delay,  /* just reuse rcu's version. */
 770        .readunlock     = sched_torture_read_unlock,
 771        .completed      = rcu_no_completed,
 772        .deferred_free  = rcu_sync_torture_deferred_free,
 773        .sync           = synchronize_sched_expedited,
 774        .cb_barrier     = NULL,
 775        .fqs            = rcu_sched_force_quiescent_state,
 776        .stats          = NULL,
 777        .irq_capable    = 1,
 778        .name           = "sched_expedited"
 779};
 780
 781/*
 782 * RCU torture priority-boost testing.  Runs one real-time thread per
 783 * CPU for moderate bursts, repeatedly registering RCU callbacks and
 784 * spinning waiting for them to be invoked.  If a given callback takes
 785 * too long to be invoked, we assume that priority inversion has occurred.
 786 */
 787
 788struct rcu_boost_inflight {
 789        struct rcu_head rcu;
 790        int inflight;
 791};
 792
 793static void rcu_torture_boost_cb(struct rcu_head *head)
 794{
 795        struct rcu_boost_inflight *rbip =
 796                container_of(head, struct rcu_boost_inflight, rcu);
 797
 798        smp_mb(); /* Ensure RCU-core accesses precede clearing ->inflight */
 799        rbip->inflight = 0;
 800}
 801
 802static int rcu_torture_boost(void *arg)
 803{
 804        unsigned long call_rcu_time;
 805        unsigned long endtime;
 806        unsigned long oldstarttime;
 807        struct rcu_boost_inflight rbi = { .inflight = 0 };
 808        struct sched_param sp;
 809
 810        VERBOSE_PRINTK_STRING("rcu_torture_boost started");
 811
 812        /* Set real-time priority. */
 813        sp.sched_priority = 1;
 814        if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) {
 815                VERBOSE_PRINTK_STRING("rcu_torture_boost RT prio failed!");
 816                n_rcu_torture_boost_rterror++;
 817        }
 818
 819        init_rcu_head_on_stack(&rbi.rcu);
 820        /* Each pass through the following loop does one boost-test cycle. */
 821        do {
 822                /* Wait for the next test interval. */
 823                oldstarttime = boost_starttime;
 824                while (ULONG_CMP_LT(jiffies, oldstarttime)) {
 825                        schedule_timeout_interruptible(oldstarttime - jiffies);
 826                        rcu_stutter_wait("rcu_torture_boost");
 827                        if (kthread_should_stop() ||
 828                            fullstop != FULLSTOP_DONTSTOP)
 829                                goto checkwait;
 830                }
 831
 832                /* Do one boost-test interval. */
 833                endtime = oldstarttime + test_boost_duration * HZ;
 834                call_rcu_time = jiffies;
 835                while (ULONG_CMP_LT(jiffies, endtime)) {
 836                        /* If we don't have a callback in flight, post one. */
 837                        if (!rbi.inflight) {
 838                                smp_mb(); /* RCU core before ->inflight = 1. */
 839                                rbi.inflight = 1;
 840                                call_rcu(&rbi.rcu, rcu_torture_boost_cb);
 841                                if (jiffies - call_rcu_time >
 842                                         test_boost_duration * HZ - HZ / 2) {
 843                                        VERBOSE_PRINTK_STRING("rcu_torture_boost boosting failed");
 844                                        n_rcu_torture_boost_failure++;
 845                                }
 846                                call_rcu_time = jiffies;
 847                        }
 848                        cond_resched();
 849                        rcu_stutter_wait("rcu_torture_boost");
 850                        if (kthread_should_stop() ||
 851                            fullstop != FULLSTOP_DONTSTOP)
 852                                goto checkwait;
 853                }
 854
 855                /*
 856                 * Set the start time of the next test interval.
 857                 * Yes, this is vulnerable to long delays, but such
 858                 * delays simply cause a false negative for the next
 859                 * interval.  Besides, we are running at RT priority,
 860                 * so delays should be relatively rare.
 861                 */
 862                while (oldstarttime == boost_starttime &&
 863                       !kthread_should_stop()) {
 864                        if (mutex_trylock(&boost_mutex)) {
 865                                boost_starttime = jiffies +
 866                                                  test_boost_interval * HZ;
 867                                n_rcu_torture_boosts++;
 868                                mutex_unlock(&boost_mutex);
 869                                break;
 870                        }
 871                        schedule_timeout_uninterruptible(1);
 872                }
 873
 874                /* Go do the stutter. */
 875checkwait:      rcu_stutter_wait("rcu_torture_boost");
 876        } while (!kthread_should_stop() && fullstop  == FULLSTOP_DONTSTOP);
 877
 878        /* Clean up and exit. */
 879        VERBOSE_PRINTK_STRING("rcu_torture_boost task stopping");
 880        rcutorture_shutdown_absorb("rcu_torture_boost");
 881        while (!kthread_should_stop() || rbi.inflight)
 882                schedule_timeout_uninterruptible(1);
 883        smp_mb(); /* order accesses to ->inflight before stack-frame death. */
 884        destroy_rcu_head_on_stack(&rbi.rcu);
 885        return 0;
 886}
 887
 888/*
 889 * RCU torture force-quiescent-state kthread.  Repeatedly induces
 890 * bursts of calls to force_quiescent_state(), increasing the probability
 891 * of occurrence of some important types of race conditions.
 892 */
 893static int
 894rcu_torture_fqs(void *arg)
 895{
 896        unsigned long fqs_resume_time;
 897        int fqs_burst_remaining;
 898
 899        VERBOSE_PRINTK_STRING("rcu_torture_fqs task started");
 900        do {
 901                fqs_resume_time = jiffies + fqs_stutter * HZ;
 902                while (ULONG_CMP_LT(jiffies, fqs_resume_time) &&
 903                       !kthread_should_stop()) {
 904                        schedule_timeout_interruptible(1);
 905                }
 906                fqs_burst_remaining = fqs_duration;
 907                while (fqs_burst_remaining > 0 &&
 908                       !kthread_should_stop()) {
 909                        cur_ops->fqs();
 910                        udelay(fqs_holdoff);
 911                        fqs_burst_remaining -= fqs_holdoff;
 912                }
 913                rcu_stutter_wait("rcu_torture_fqs");
 914        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
 915        VERBOSE_PRINTK_STRING("rcu_torture_fqs task stopping");
 916        rcutorture_shutdown_absorb("rcu_torture_fqs");
 917        while (!kthread_should_stop())
 918                schedule_timeout_uninterruptible(1);
 919        return 0;
 920}
 921
 922/*
 923 * RCU torture writer kthread.  Repeatedly substitutes a new structure
 924 * for that pointed to by rcu_torture_current, freeing the old structure
 925 * after a series of grace periods (the "pipeline").
 926 */
 927static int
 928rcu_torture_writer(void *arg)
 929{
 930        int i;
 931        long oldbatch = rcu_batches_completed();
 932        struct rcu_torture *rp;
 933        struct rcu_torture *old_rp;
 934        static DEFINE_RCU_RANDOM(rand);
 935
 936        VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
 937        set_user_nice(current, 19);
 938
 939        do {
 940                schedule_timeout_uninterruptible(1);
 941                rp = rcu_torture_alloc();
 942                if (rp == NULL)
 943                        continue;
 944                rp->rtort_pipe_count = 0;
 945                udelay(rcu_random(&rand) & 0x3ff);
 946                old_rp = rcu_dereference_check(rcu_torture_current,
 947                                               current == writer_task);
 948                rp->rtort_mbtest = 1;
 949                rcu_assign_pointer(rcu_torture_current, rp);
 950                smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
 951                if (old_rp) {
 952                        i = old_rp->rtort_pipe_count;
 953                        if (i > RCU_TORTURE_PIPE_LEN)
 954                                i = RCU_TORTURE_PIPE_LEN;
 955                        atomic_inc(&rcu_torture_wcount[i]);
 956                        old_rp->rtort_pipe_count++;
 957                        cur_ops->deferred_free(old_rp);
 958                }
 959                rcutorture_record_progress(++rcu_torture_current_version);
 960                oldbatch = cur_ops->completed();
 961                rcu_stutter_wait("rcu_torture_writer");
 962        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
 963        VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
 964        rcutorture_shutdown_absorb("rcu_torture_writer");
 965        while (!kthread_should_stop())
 966                schedule_timeout_uninterruptible(1);
 967        return 0;
 968}
 969
 970/*
 971 * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
 972 * delay between calls.
 973 */
 974static int
 975rcu_torture_fakewriter(void *arg)
 976{
 977        DEFINE_RCU_RANDOM(rand);
 978
 979        VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
 980        set_user_nice(current, 19);
 981
 982        do {
 983                schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
 984                udelay(rcu_random(&rand) & 0x3ff);
 985                if (cur_ops->cb_barrier != NULL &&
 986                    rcu_random(&rand) % (nfakewriters * 8) == 0)
 987                        cur_ops->cb_barrier();
 988                else
 989                        cur_ops->sync();
 990                rcu_stutter_wait("rcu_torture_fakewriter");
 991        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
 992
 993        VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
 994        rcutorture_shutdown_absorb("rcu_torture_fakewriter");
 995        while (!kthread_should_stop())
 996                schedule_timeout_uninterruptible(1);
 997        return 0;
 998}
 999
1000void rcutorture_trace_dump(void)
1001{
1002        static atomic_t beenhere = ATOMIC_INIT(0);
1003
1004        if (atomic_read(&beenhere))
1005                return;
1006        if (atomic_xchg(&beenhere, 1) != 0)
1007                return;
1008        ftrace_dump(DUMP_ALL);
1009}
1010
1011/*
1012 * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
1013 * incrementing the corresponding element of the pipeline array.  The
1014 * counter in the element should never be greater than 1, otherwise, the
1015 * RCU implementation is broken.
1016 */
1017static void rcu_torture_timer(unsigned long unused)
1018{
1019        int idx;
1020        int completed;
1021        int completed_end;
1022        static DEFINE_RCU_RANDOM(rand);
1023        static DEFINE_SPINLOCK(rand_lock);
1024        struct rcu_torture *p;
1025        int pipe_count;
1026        unsigned long long ts;
1027
1028        idx = cur_ops->readlock();
1029        completed = cur_ops->completed();
1030        ts = rcu_trace_clock_local();
1031        p = rcu_dereference_check(rcu_torture_current,
1032                                  rcu_read_lock_bh_held() ||
1033                                  rcu_read_lock_sched_held() ||
1034                                  srcu_read_lock_held(&srcu_ctl));
1035        if (p == NULL) {
1036                /* Leave because rcu_torture_writer is not yet underway */
1037                cur_ops->readunlock(idx);
1038                return;
1039        }
1040        if (p->rtort_mbtest == 0)
1041                atomic_inc(&n_rcu_torture_mberror);
1042        spin_lock(&rand_lock);
1043        cur_ops->read_delay(&rand);
1044        n_rcu_torture_timers++;
1045        spin_unlock(&rand_lock);
1046        preempt_disable();
1047        pipe_count = p->rtort_pipe_count;
1048        if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1049                /* Should not happen, but... */
1050                pipe_count = RCU_TORTURE_PIPE_LEN;
1051        }
1052        completed_end = cur_ops->completed();
1053        if (pipe_count > 1) {
1054                do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, ts,
1055                                          completed, completed_end);
1056                rcutorture_trace_dump();
1057        }
1058        __this_cpu_inc(rcu_torture_count[pipe_count]);
1059        completed = completed_end - completed;
1060        if (completed > RCU_TORTURE_PIPE_LEN) {
1061                /* Should not happen, but... */
1062                completed = RCU_TORTURE_PIPE_LEN;
1063        }
1064        __this_cpu_inc(rcu_torture_batch[completed]);
1065        preempt_enable();
1066        cur_ops->readunlock(idx);
1067}
1068
1069/*
1070 * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
1071 * incrementing the corresponding element of the pipeline array.  The
1072 * counter in the element should never be greater than 1, otherwise, the
1073 * RCU implementation is broken.
1074 */
1075static int
1076rcu_torture_reader(void *arg)
1077{
1078        int completed;
1079        int completed_end;
1080        int idx;
1081        DEFINE_RCU_RANDOM(rand);
1082        struct rcu_torture *p;
1083        int pipe_count;
1084        struct timer_list t;
1085        unsigned long long ts;
1086
1087        VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
1088        set_user_nice(current, 19);
1089        if (irqreader && cur_ops->irq_capable)
1090                setup_timer_on_stack(&t, rcu_torture_timer, 0);
1091
1092        do {
1093                if (irqreader && cur_ops->irq_capable) {
1094                        if (!timer_pending(&t))
1095                                mod_timer(&t, jiffies + 1);
1096                }
1097                idx = cur_ops->readlock();
1098                completed = cur_ops->completed();
1099                ts = rcu_trace_clock_local();
1100                p = rcu_dereference_check(rcu_torture_current,
1101                                          rcu_read_lock_bh_held() ||
1102                                          rcu_read_lock_sched_held() ||
1103                                          srcu_read_lock_held(&srcu_ctl));
1104                if (p == NULL) {
1105                        /* Wait for rcu_torture_writer to get underway */
1106                        cur_ops->readunlock(idx);
1107                        schedule_timeout_interruptible(HZ);
1108                        continue;
1109                }
1110                if (p->rtort_mbtest == 0)
1111                        atomic_inc(&n_rcu_torture_mberror);
1112                cur_ops->read_delay(&rand);
1113                preempt_disable();
1114                pipe_count = p->rtort_pipe_count;
1115                if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1116                        /* Should not happen, but... */
1117                        pipe_count = RCU_TORTURE_PIPE_LEN;
1118                }
1119                completed_end = cur_ops->completed();
1120                if (pipe_count > 1) {
1121                        do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu,
1122                                                  ts, completed, completed_end);
1123                        rcutorture_trace_dump();
1124                }
1125                __this_cpu_inc(rcu_torture_count[pipe_count]);
1126                completed = completed_end - completed;
1127                if (completed > RCU_TORTURE_PIPE_LEN) {
1128                        /* Should not happen, but... */
1129                        completed = RCU_TORTURE_PIPE_LEN;
1130                }
1131                __this_cpu_inc(rcu_torture_batch[completed]);
1132                preempt_enable();
1133                cur_ops->readunlock(idx);
1134                schedule();
1135                rcu_stutter_wait("rcu_torture_reader");
1136        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
1137        VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
1138        rcutorture_shutdown_absorb("rcu_torture_reader");
1139        if (irqreader && cur_ops->irq_capable)
1140                del_timer_sync(&t);
1141        while (!kthread_should_stop())
1142                schedule_timeout_uninterruptible(1);
1143        return 0;
1144}
1145
1146/*
1147 * Create an RCU-torture statistics message in the specified buffer.
1148 */
1149static int
1150rcu_torture_printk(char *page)
1151{
1152        int cnt = 0;
1153        int cpu;
1154        int i;
1155        long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1156        long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1157
1158        for_each_possible_cpu(cpu) {
1159                for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1160                        pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
1161                        batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
1162                }
1163        }
1164        for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
1165                if (pipesummary[i] != 0)
1166                        break;
1167        }
1168        cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
1169        cnt += sprintf(&page[cnt],
1170                       "rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
1171                       rcu_torture_current,
1172                       rcu_torture_current_version,
1173                       list_empty(&rcu_torture_freelist),
1174                       atomic_read(&n_rcu_torture_alloc),
1175                       atomic_read(&n_rcu_torture_alloc_fail),
1176                       atomic_read(&n_rcu_torture_free));
1177        cnt += sprintf(&page[cnt], "rtmbe: %d rtbke: %ld rtbre: %ld ",
1178                       atomic_read(&n_rcu_torture_mberror),
1179                       n_rcu_torture_boost_ktrerror,
1180                       n_rcu_torture_boost_rterror);
1181        cnt += sprintf(&page[cnt], "rtbf: %ld rtb: %ld nt: %ld ",
1182                       n_rcu_torture_boost_failure,
1183                       n_rcu_torture_boosts,
1184                       n_rcu_torture_timers);
1185        cnt += sprintf(&page[cnt],
1186                       "onoff: %ld/%ld:%ld/%ld %d,%d:%d,%d %lu:%lu (HZ=%d) ",
1187                       n_online_successes, n_online_attempts,
1188                       n_offline_successes, n_offline_attempts,
1189                       min_online, max_online,
1190                       min_offline, max_offline,
1191                       sum_online, sum_offline, HZ);
1192        cnt += sprintf(&page[cnt], "barrier: %ld/%ld:%ld",
1193                       n_barrier_successes,
1194                       n_barrier_attempts,
1195                       n_rcu_torture_barrier_error);
1196        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
1197        if (atomic_read(&n_rcu_torture_mberror) != 0 ||
1198            n_rcu_torture_barrier_error != 0 ||
1199            n_rcu_torture_boost_ktrerror != 0 ||
1200            n_rcu_torture_boost_rterror != 0 ||
1201            n_rcu_torture_boost_failure != 0 ||
1202            i > 1) {
1203                cnt += sprintf(&page[cnt], "!!! ");
1204                atomic_inc(&n_rcu_torture_error);
1205                WARN_ON_ONCE(1);
1206        }
1207        cnt += sprintf(&page[cnt], "Reader Pipe: ");
1208        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1209                cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
1210        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
1211        cnt += sprintf(&page[cnt], "Reader Batch: ");
1212        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1213                cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
1214        cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
1215        cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
1216        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1217                cnt += sprintf(&page[cnt], " %d",
1218                               atomic_read(&rcu_torture_wcount[i]));
1219        }
1220        cnt += sprintf(&page[cnt], "\n");
1221        if (cur_ops->stats)
1222                cnt += cur_ops->stats(&page[cnt]);
1223        return cnt;
1224}
1225
1226/*
1227 * Print torture statistics.  Caller must ensure that there is only
1228 * one call to this function at a given time!!!  This is normally
1229 * accomplished by relying on the module system to only have one copy
1230 * of the module loaded, and then by giving the rcu_torture_stats
1231 * kthread full control (or the init/cleanup functions when rcu_torture_stats
1232 * thread is not running).
1233 */
1234static void
1235rcu_torture_stats_print(void)
1236{
1237        int cnt;
1238
1239        cnt = rcu_torture_printk(printk_buf);
1240        pr_alert("%s", printk_buf);
1241}
1242
1243/*
1244 * Periodically prints torture statistics, if periodic statistics printing
1245 * was specified via the stat_interval module parameter.
1246 *
1247 * No need to worry about fullstop here, since this one doesn't reference
1248 * volatile state or register callbacks.
1249 */
1250static int
1251rcu_torture_stats(void *arg)
1252{
1253        VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
1254        do {
1255                schedule_timeout_interruptible(stat_interval * HZ);
1256                rcu_torture_stats_print();
1257                rcutorture_shutdown_absorb("rcu_torture_stats");
1258        } while (!kthread_should_stop());
1259        VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
1260        return 0;
1261}
1262
1263static int rcu_idle_cpu;        /* Force all torture tasks off this CPU */
1264
1265/* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
1266 * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
1267 */
1268static void rcu_torture_shuffle_tasks(void)
1269{
1270        int i;
1271
1272        cpumask_setall(shuffle_tmp_mask);
1273        get_online_cpus();
1274
1275        /* No point in shuffling if there is only one online CPU (ex: UP) */
1276        if (num_online_cpus() == 1) {
1277                put_online_cpus();
1278                return;
1279        }
1280
1281        if (rcu_idle_cpu != -1)
1282                cpumask_clear_cpu(rcu_idle_cpu, shuffle_tmp_mask);
1283
1284        set_cpus_allowed_ptr(current, shuffle_tmp_mask);
1285
1286        if (reader_tasks) {
1287                for (i = 0; i < nrealreaders; i++)
1288                        if (reader_tasks[i])
1289                                set_cpus_allowed_ptr(reader_tasks[i],
1290                                                     shuffle_tmp_mask);
1291        }
1292        if (fakewriter_tasks) {
1293                for (i = 0; i < nfakewriters; i++)
1294                        if (fakewriter_tasks[i])
1295                                set_cpus_allowed_ptr(fakewriter_tasks[i],
1296                                                     shuffle_tmp_mask);
1297        }
1298        if (writer_task)
1299                set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask);
1300        if (stats_task)
1301                set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask);
1302        if (stutter_task)
1303                set_cpus_allowed_ptr(stutter_task, shuffle_tmp_mask);
1304        if (fqs_task)
1305                set_cpus_allowed_ptr(fqs_task, shuffle_tmp_mask);
1306        if (shutdown_task)
1307                set_cpus_allowed_ptr(shutdown_task, shuffle_tmp_mask);
1308#ifdef CONFIG_HOTPLUG_CPU
1309        if (onoff_task)
1310                set_cpus_allowed_ptr(onoff_task, shuffle_tmp_mask);
1311#endif /* #ifdef CONFIG_HOTPLUG_CPU */
1312        if (stall_task)
1313                set_cpus_allowed_ptr(stall_task, shuffle_tmp_mask);
1314        if (barrier_cbs_tasks)
1315                for (i = 0; i < n_barrier_cbs; i++)
1316                        if (barrier_cbs_tasks[i])
1317                                set_cpus_allowed_ptr(barrier_cbs_tasks[i],
1318                                                     shuffle_tmp_mask);
1319        if (barrier_task)
1320                set_cpus_allowed_ptr(barrier_task, shuffle_tmp_mask);
1321
1322        if (rcu_idle_cpu == -1)
1323                rcu_idle_cpu = num_online_cpus() - 1;
1324        else
1325                rcu_idle_cpu--;
1326
1327        put_online_cpus();
1328}
1329
1330/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
1331 * system to become idle at a time and cut off its timer ticks. This is meant
1332 * to test the support for such tickless idle CPU in RCU.
1333 */
1334static int
1335rcu_torture_shuffle(void *arg)
1336{
1337        VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
1338        do {
1339                schedule_timeout_interruptible(shuffle_interval * HZ);
1340                rcu_torture_shuffle_tasks();
1341                rcutorture_shutdown_absorb("rcu_torture_shuffle");
1342        } while (!kthread_should_stop());
1343        VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
1344        return 0;
1345}
1346
1347/* Cause the rcutorture test to "stutter", starting and stopping all
1348 * threads periodically.
1349 */
1350static int
1351rcu_torture_stutter(void *arg)
1352{
1353        VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
1354        do {
1355                schedule_timeout_interruptible(stutter * HZ);
1356                stutter_pause_test = 1;
1357                if (!kthread_should_stop())
1358                        schedule_timeout_interruptible(stutter * HZ);
1359                stutter_pause_test = 0;
1360                rcutorture_shutdown_absorb("rcu_torture_stutter");
1361        } while (!kthread_should_stop());
1362        VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
1363        return 0;
1364}
1365
1366static inline void
1367rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, char *tag)
1368{
1369        pr_alert("%s" TORTURE_FLAG
1370                 "--- %s: nreaders=%d nfakewriters=%d "
1371                 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
1372                 "shuffle_interval=%d stutter=%d irqreader=%d "
1373                 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
1374                 "test_boost=%d/%d test_boost_interval=%d "
1375                 "test_boost_duration=%d shutdown_secs=%d "
1376                 "stall_cpu=%d stall_cpu_holdoff=%d "
1377                 "n_barrier_cbs=%d "
1378                 "onoff_interval=%d onoff_holdoff=%d\n",
1379                 torture_type, tag, nrealreaders, nfakewriters,
1380                 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
1381                 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
1382                 test_boost, cur_ops->can_boost,
1383                 test_boost_interval, test_boost_duration, shutdown_secs,
1384                 stall_cpu, stall_cpu_holdoff,
1385                 n_barrier_cbs,
1386                 onoff_interval, onoff_holdoff);
1387}
1388
1389static struct notifier_block rcutorture_shutdown_nb = {
1390        .notifier_call = rcutorture_shutdown_notify,
1391};
1392
1393static void rcutorture_booster_cleanup(int cpu)
1394{
1395        struct task_struct *t;
1396
1397        if (boost_tasks[cpu] == NULL)
1398                return;
1399        mutex_lock(&boost_mutex);
1400        VERBOSE_PRINTK_STRING("Stopping rcu_torture_boost task");
1401        t = boost_tasks[cpu];
1402        boost_tasks[cpu] = NULL;
1403        mutex_unlock(&boost_mutex);
1404
1405        /* This must be outside of the mutex, otherwise deadlock! */
1406        kthread_stop(t);
1407        boost_tasks[cpu] = NULL;
1408}
1409
1410static int rcutorture_booster_init(int cpu)
1411{
1412        int retval;
1413
1414        if (boost_tasks[cpu] != NULL)
1415                return 0;  /* Already created, nothing more to do. */
1416
1417        /* Don't allow time recalculation while creating a new task. */
1418        mutex_lock(&boost_mutex);
1419        VERBOSE_PRINTK_STRING("Creating rcu_torture_boost task");
1420        boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL,
1421                                                  cpu_to_node(cpu),
1422                                                  "rcu_torture_boost");
1423        if (IS_ERR(boost_tasks[cpu])) {
1424                retval = PTR_ERR(boost_tasks[cpu]);
1425                VERBOSE_PRINTK_STRING("rcu_torture_boost task create failed");
1426                n_rcu_torture_boost_ktrerror++;
1427                boost_tasks[cpu] = NULL;
1428                mutex_unlock(&boost_mutex);
1429                return retval;
1430        }
1431        kthread_bind(boost_tasks[cpu], cpu);
1432        wake_up_process(boost_tasks[cpu]);
1433        mutex_unlock(&boost_mutex);
1434        return 0;
1435}
1436
1437/*
1438 * Cause the rcutorture test to shutdown the system after the test has
1439 * run for the time specified by the shutdown_secs module parameter.
1440 */
1441static int
1442rcu_torture_shutdown(void *arg)
1443{
1444        long delta;
1445        unsigned long jiffies_snap;
1446
1447        VERBOSE_PRINTK_STRING("rcu_torture_shutdown task started");
1448        jiffies_snap = ACCESS_ONCE(jiffies);
1449        while (ULONG_CMP_LT(jiffies_snap, shutdown_time) &&
1450               !kthread_should_stop()) {
1451                delta = shutdown_time - jiffies_snap;
1452                if (verbose)
1453                        pr_alert("%s" TORTURE_FLAG
1454                                 "rcu_torture_shutdown task: %lu jiffies remaining\n",
1455                                 torture_type, delta);
1456                schedule_timeout_interruptible(delta);
1457                jiffies_snap = ACCESS_ONCE(jiffies);
1458        }
1459        if (kthread_should_stop()) {
1460                VERBOSE_PRINTK_STRING("rcu_torture_shutdown task stopping");
1461                return 0;
1462        }
1463
1464        /* OK, shut down the system. */
1465
1466        VERBOSE_PRINTK_STRING("rcu_torture_shutdown task shutting down system");
1467        shutdown_task = NULL;   /* Avoid self-kill deadlock. */
1468        rcu_torture_cleanup();  /* Get the success/failure message. */
1469        kernel_power_off();     /* Shut down the system. */
1470        return 0;
1471}
1472
1473#ifdef CONFIG_HOTPLUG_CPU
1474
1475/*
1476 * Execute random CPU-hotplug operations at the interval specified
1477 * by the onoff_interval.
1478 */
1479static int
1480rcu_torture_onoff(void *arg)
1481{
1482        int cpu;
1483        unsigned long delta;
1484        int maxcpu = -1;
1485        DEFINE_RCU_RANDOM(rand);
1486        int ret;
1487        unsigned long starttime;
1488
1489        VERBOSE_PRINTK_STRING("rcu_torture_onoff task started");
1490        for_each_online_cpu(cpu)
1491                maxcpu = cpu;
1492        WARN_ON(maxcpu < 0);
1493        if (onoff_holdoff > 0) {
1494                VERBOSE_PRINTK_STRING("rcu_torture_onoff begin holdoff");
1495                schedule_timeout_interruptible(onoff_holdoff * HZ);
1496                VERBOSE_PRINTK_STRING("rcu_torture_onoff end holdoff");
1497        }
1498        while (!kthread_should_stop()) {
1499                cpu = (rcu_random(&rand) >> 4) % (maxcpu + 1);
1500                if (cpu_online(cpu) && cpu_is_hotpluggable(cpu)) {
1501                        if (verbose)
1502                                pr_alert("%s" TORTURE_FLAG
1503                                         "rcu_torture_onoff task: offlining %d\n",
1504                                         torture_type, cpu);
1505                        starttime = jiffies;
1506                        n_offline_attempts++;
1507                        ret = cpu_down(cpu);
1508                        if (ret) {
1509                                if (verbose)
1510                                        pr_alert("%s" TORTURE_FLAG
1511                                                 "rcu_torture_onoff task: offline %d failed: errno %d\n",
1512                                                 torture_type, cpu, ret);
1513                        } else {
1514                                if (verbose)
1515                                        pr_alert("%s" TORTURE_FLAG
1516                                                 "rcu_torture_onoff task: offlined %d\n",
1517                                                 torture_type, cpu);
1518                                n_offline_successes++;
1519                                delta = jiffies - starttime;
1520                                sum_offline += delta;
1521                                if (min_offline < 0) {
1522                                        min_offline = delta;
1523                                        max_offline = delta;
1524                                }
1525                                if (min_offline > delta)
1526                                        min_offline = delta;
1527                                if (max_offline < delta)
1528                                        max_offline = delta;
1529                        }
1530                } else if (cpu_is_hotpluggable(cpu)) {
1531                        if (verbose)
1532                                pr_alert("%s" TORTURE_FLAG
1533                                         "rcu_torture_onoff task: onlining %d\n",
1534                                         torture_type, cpu);
1535                        starttime = jiffies;
1536                        n_online_attempts++;
1537                        if (cpu_up(cpu) == 0) {
1538                                if (verbose)
1539                                        pr_alert("%s" TORTURE_FLAG
1540                                                 "rcu_torture_onoff task: onlined %d\n",
1541                                                 torture_type, cpu);
1542                                n_online_successes++;
1543                                delta = jiffies - starttime;
1544                                sum_online += delta;
1545                                if (min_online < 0) {
1546                                        min_online = delta;
1547                                        max_online = delta;
1548                                }
1549                                if (min_online > delta)
1550                                        min_online = delta;
1551                                if (max_online < delta)
1552                                        max_online = delta;
1553                        }
1554                }
1555                schedule_timeout_interruptible(onoff_interval * HZ);
1556        }
1557        VERBOSE_PRINTK_STRING("rcu_torture_onoff task stopping");
1558        return 0;
1559}
1560
1561static int
1562rcu_torture_onoff_init(void)
1563{
1564        int ret;
1565
1566        if (onoff_interval <= 0)
1567                return 0;
1568        onoff_task = kthread_run(rcu_torture_onoff, NULL, "rcu_torture_onoff");
1569        if (IS_ERR(onoff_task)) {
1570                ret = PTR_ERR(onoff_task);
1571                onoff_task = NULL;
1572                return ret;
1573        }
1574        return 0;
1575}
1576
1577static void rcu_torture_onoff_cleanup(void)
1578{
1579        if (onoff_task == NULL)
1580                return;
1581        VERBOSE_PRINTK_STRING("Stopping rcu_torture_onoff task");
1582        kthread_stop(onoff_task);
1583        onoff_task = NULL;
1584}
1585
1586#else /* #ifdef CONFIG_HOTPLUG_CPU */
1587
1588static int
1589rcu_torture_onoff_init(void)
1590{
1591        return 0;
1592}
1593
1594static void rcu_torture_onoff_cleanup(void)
1595{
1596}
1597
1598#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
1599
1600/*
1601 * CPU-stall kthread.  It waits as specified by stall_cpu_holdoff, then
1602 * induces a CPU stall for the time specified by stall_cpu.
1603 */
1604static int rcu_torture_stall(void *args)
1605{
1606        unsigned long stop_at;
1607
1608        VERBOSE_PRINTK_STRING("rcu_torture_stall task started");
1609        if (stall_cpu_holdoff > 0) {
1610                VERBOSE_PRINTK_STRING("rcu_torture_stall begin holdoff");
1611                schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
1612                VERBOSE_PRINTK_STRING("rcu_torture_stall end holdoff");
1613        }
1614        if (!kthread_should_stop()) {
1615                stop_at = get_seconds() + stall_cpu;
1616                /* RCU CPU stall is expected behavior in following code. */
1617                pr_alert("rcu_torture_stall start.\n");
1618                rcu_read_lock();
1619                preempt_disable();
1620                while (ULONG_CMP_LT(get_seconds(), stop_at))
1621                        continue;  /* Induce RCU CPU stall warning. */
1622                preempt_enable();
1623                rcu_read_unlock();
1624                pr_alert("rcu_torture_stall end.\n");
1625        }
1626        rcutorture_shutdown_absorb("rcu_torture_stall");
1627        while (!kthread_should_stop())
1628                schedule_timeout_interruptible(10 * HZ);
1629        return 0;
1630}
1631
1632/* Spawn CPU-stall kthread, if stall_cpu specified. */
1633static int __init rcu_torture_stall_init(void)
1634{
1635        int ret;
1636
1637        if (stall_cpu <= 0)
1638                return 0;
1639        stall_task = kthread_run(rcu_torture_stall, NULL, "rcu_torture_stall");
1640        if (IS_ERR(stall_task)) {
1641                ret = PTR_ERR(stall_task);
1642                stall_task = NULL;
1643                return ret;
1644        }
1645        return 0;
1646}
1647
1648/* Clean up after the CPU-stall kthread, if one was spawned. */
1649static void rcu_torture_stall_cleanup(void)
1650{
1651        if (stall_task == NULL)
1652                return;
1653        VERBOSE_PRINTK_STRING("Stopping rcu_torture_stall_task.");
1654        kthread_stop(stall_task);
1655        stall_task = NULL;
1656}
1657
1658/* Callback function for RCU barrier testing. */
1659void rcu_torture_barrier_cbf(struct rcu_head *rcu)
1660{
1661        atomic_inc(&barrier_cbs_invoked);
1662}
1663
1664/* kthread function to register callbacks used to test RCU barriers. */
1665static int rcu_torture_barrier_cbs(void *arg)
1666{
1667        long myid = (long)arg;
1668        bool lastphase = 0;
1669        struct rcu_head rcu;
1670
1671        init_rcu_head_on_stack(&rcu);
1672        VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task started");
1673        set_user_nice(current, 19);
1674        do {
1675                wait_event(barrier_cbs_wq[myid],
1676                           barrier_phase != lastphase ||
1677                           kthread_should_stop() ||
1678                           fullstop != FULLSTOP_DONTSTOP);
1679                lastphase = barrier_phase;
1680                smp_mb(); /* ensure barrier_phase load before ->call(). */
1681                if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
1682                        break;
1683                cur_ops->call(&rcu, rcu_torture_barrier_cbf);
1684                if (atomic_dec_and_test(&barrier_cbs_count))
1685                        wake_up(&barrier_wq);
1686        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
1687        VERBOSE_PRINTK_STRING("rcu_torture_barrier_cbs task stopping");
1688        rcutorture_shutdown_absorb("rcu_torture_barrier_cbs");
1689        while (!kthread_should_stop())
1690                schedule_timeout_interruptible(1);
1691        cur_ops->cb_barrier();
1692        destroy_rcu_head_on_stack(&rcu);
1693        return 0;
1694}
1695
1696/* kthread function to drive and coordinate RCU barrier testing. */
1697static int rcu_torture_barrier(void *arg)
1698{
1699        int i;
1700
1701        VERBOSE_PRINTK_STRING("rcu_torture_barrier task starting");
1702        do {
1703                atomic_set(&barrier_cbs_invoked, 0);
1704                atomic_set(&barrier_cbs_count, n_barrier_cbs);
1705                smp_mb(); /* Ensure barrier_phase after prior assignments. */
1706                barrier_phase = !barrier_phase;
1707                for (i = 0; i < n_barrier_cbs; i++)
1708                        wake_up(&barrier_cbs_wq[i]);
1709                wait_event(barrier_wq,
1710                           atomic_read(&barrier_cbs_count) == 0 ||
1711                           kthread_should_stop() ||
1712                           fullstop != FULLSTOP_DONTSTOP);
1713                if (kthread_should_stop() || fullstop != FULLSTOP_DONTSTOP)
1714                        break;
1715                n_barrier_attempts++;
1716                cur_ops->cb_barrier();
1717                if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
1718                        n_rcu_torture_barrier_error++;
1719                        WARN_ON_ONCE(1);
1720                }
1721                n_barrier_successes++;
1722                schedule_timeout_interruptible(HZ / 10);
1723        } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
1724        VERBOSE_PRINTK_STRING("rcu_torture_barrier task stopping");
1725        rcutorture_shutdown_absorb("rcu_torture_barrier");
1726        while (!kthread_should_stop())
1727                schedule_timeout_interruptible(1);
1728        return 0;
1729}
1730
1731/* Initialize RCU barrier testing. */
1732static int rcu_torture_barrier_init(void)
1733{
1734        int i;
1735        int ret;
1736
1737        if (n_barrier_cbs == 0)
1738                return 0;
1739        if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
1740                pr_alert("%s" TORTURE_FLAG
1741                         " Call or barrier ops missing for %s,\n",
1742                         torture_type, cur_ops->name);
1743                pr_alert("%s" TORTURE_FLAG
1744                         " RCU barrier testing omitted from run.\n",
1745                         torture_type);
1746                return 0;
1747        }
1748        atomic_set(&barrier_cbs_count, 0);
1749        atomic_set(&barrier_cbs_invoked, 0);
1750        barrier_cbs_tasks =
1751                kzalloc(n_barrier_cbs * sizeof(barrier_cbs_tasks[0]),
1752                        GFP_KERNEL);
1753        barrier_cbs_wq =
1754                kzalloc(n_barrier_cbs * sizeof(barrier_cbs_wq[0]),
1755                        GFP_KERNEL);
1756        if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
1757                return -ENOMEM;
1758        for (i = 0; i < n_barrier_cbs; i++) {
1759                init_waitqueue_head(&barrier_cbs_wq[i]);
1760                barrier_cbs_tasks[i] = kthread_run(rcu_torture_barrier_cbs,
1761                                                   (void *)(long)i,
1762                                                   "rcu_torture_barrier_cbs");
1763                if (IS_ERR(barrier_cbs_tasks[i])) {
1764                        ret = PTR_ERR(barrier_cbs_tasks[i]);
1765                        VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier_cbs");
1766                        barrier_cbs_tasks[i] = NULL;
1767                        return ret;
1768                }
1769        }
1770        barrier_task = kthread_run(rcu_torture_barrier, NULL,
1771                                   "rcu_torture_barrier");
1772        if (IS_ERR(barrier_task)) {
1773                ret = PTR_ERR(barrier_task);
1774                VERBOSE_PRINTK_ERRSTRING("Failed to create rcu_torture_barrier");
1775                barrier_task = NULL;
1776        }
1777        return 0;
1778}
1779
1780/* Clean up after RCU barrier testing. */
1781static void rcu_torture_barrier_cleanup(void)
1782{
1783        int i;
1784
1785        if (barrier_task != NULL) {
1786                VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier task");
1787                kthread_stop(barrier_task);
1788                barrier_task = NULL;
1789        }
1790        if (barrier_cbs_tasks != NULL) {
1791                for (i = 0; i < n_barrier_cbs; i++) {
1792                        if (barrier_cbs_tasks[i] != NULL) {
1793                                VERBOSE_PRINTK_STRING("Stopping rcu_torture_barrier_cbs task");
1794                                kthread_stop(barrier_cbs_tasks[i]);
1795                                barrier_cbs_tasks[i] = NULL;
1796                        }
1797                }
1798                kfree(barrier_cbs_tasks);
1799                barrier_cbs_tasks = NULL;
1800        }
1801        if (barrier_cbs_wq != NULL) {
1802                kfree(barrier_cbs_wq);
1803                barrier_cbs_wq = NULL;
1804        }
1805}
1806
1807static int rcutorture_cpu_notify(struct notifier_block *self,
1808                                 unsigned long action, void *hcpu)
1809{
1810        long cpu = (long)hcpu;
1811
1812        switch (action) {
1813        case CPU_ONLINE:
1814        case CPU_DOWN_FAILED:
1815                (void)rcutorture_booster_init(cpu);
1816                break;
1817        case CPU_DOWN_PREPARE:
1818                rcutorture_booster_cleanup(cpu);
1819                break;
1820        default:
1821                break;
1822        }
1823        return NOTIFY_OK;
1824}
1825
1826static struct notifier_block rcutorture_cpu_nb = {
1827        .notifier_call = rcutorture_cpu_notify,
1828};
1829
1830static void
1831rcu_torture_cleanup(void)
1832{
1833        int i;
1834
1835        mutex_lock(&fullstop_mutex);
1836        rcutorture_record_test_transition();
1837        if (fullstop == FULLSTOP_SHUTDOWN) {
1838                pr_warn(/* but going down anyway, so... */
1839                       "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
1840                mutex_unlock(&fullstop_mutex);
1841                schedule_timeout_uninterruptible(10);
1842                if (cur_ops->cb_barrier != NULL)
1843                        cur_ops->cb_barrier();
1844                return;
1845        }
1846        fullstop = FULLSTOP_RMMOD;
1847        mutex_unlock(&fullstop_mutex);
1848        unregister_reboot_notifier(&rcutorture_shutdown_nb);
1849        rcu_torture_barrier_cleanup();
1850        rcu_torture_stall_cleanup();
1851        if (stutter_task) {
1852                VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
1853                kthread_stop(stutter_task);
1854        }
1855        stutter_task = NULL;
1856        if (shuffler_task) {
1857                VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
1858                kthread_stop(shuffler_task);
1859                free_cpumask_var(shuffle_tmp_mask);
1860        }
1861        shuffler_task = NULL;
1862
1863        if (writer_task) {
1864                VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
1865                kthread_stop(writer_task);
1866        }
1867        writer_task = NULL;
1868
1869        if (reader_tasks) {
1870                for (i = 0; i < nrealreaders; i++) {
1871                        if (reader_tasks[i]) {
1872                                VERBOSE_PRINTK_STRING(
1873                                        "Stopping rcu_torture_reader task");
1874                                kthread_stop(reader_tasks[i]);
1875                        }
1876                        reader_tasks[i] = NULL;
1877                }
1878                kfree(reader_tasks);
1879                reader_tasks = NULL;
1880        }
1881        rcu_torture_current = NULL;
1882
1883        if (fakewriter_tasks) {
1884                for (i = 0; i < nfakewriters; i++) {
1885                        if (fakewriter_tasks[i]) {
1886                                VERBOSE_PRINTK_STRING(
1887                                        "Stopping rcu_torture_fakewriter task");
1888                                kthread_stop(fakewriter_tasks[i]);
1889                        }
1890                        fakewriter_tasks[i] = NULL;
1891                }
1892                kfree(fakewriter_tasks);
1893                fakewriter_tasks = NULL;
1894        }
1895
1896        if (stats_task) {
1897                VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
1898                kthread_stop(stats_task);
1899        }
1900        stats_task = NULL;
1901
1902        if (fqs_task) {
1903                VERBOSE_PRINTK_STRING("Stopping rcu_torture_fqs task");
1904                kthread_stop(fqs_task);
1905        }
1906        fqs_task = NULL;
1907        if ((test_boost == 1 && cur_ops->can_boost) ||
1908            test_boost == 2) {
1909                unregister_cpu_notifier(&rcutorture_cpu_nb);
1910                for_each_possible_cpu(i)
1911                        rcutorture_booster_cleanup(i);
1912        }
1913        if (shutdown_task != NULL) {
1914                VERBOSE_PRINTK_STRING("Stopping rcu_torture_shutdown task");
1915                kthread_stop(shutdown_task);
1916        }
1917        shutdown_task = NULL;
1918        rcu_torture_onoff_cleanup();
1919
1920        /* Wait for all RCU callbacks to fire.  */
1921
1922        if (cur_ops->cb_barrier != NULL)
1923                cur_ops->cb_barrier();
1924
1925        rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
1926
1927        if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
1928                rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
1929        else if (n_online_successes != n_online_attempts ||
1930                 n_offline_successes != n_offline_attempts)
1931                rcu_torture_print_module_parms(cur_ops,
1932                                               "End of test: RCU_HOTPLUG");
1933        else
1934                rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
1935}
1936
1937static int __init
1938rcu_torture_init(void)
1939{
1940        int i;
1941        int cpu;
1942        int firsterr = 0;
1943        int retval;
1944        static struct rcu_torture_ops *torture_ops[] =
1945                { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
1946                  &rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
1947                  &srcu_ops, &srcu_sync_ops, &srcu_expedited_ops,
1948                  &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
1949
1950        mutex_lock(&fullstop_mutex);
1951
1952        /* Process args and tell the world that the torturer is on the job. */
1953        for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1954                cur_ops = torture_ops[i];
1955                if (strcmp(torture_type, cur_ops->name) == 0)
1956                        break;
1957        }
1958        if (i == ARRAY_SIZE(torture_ops)) {
1959                pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
1960                         torture_type);
1961                pr_alert("rcu-torture types:");
1962                for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1963                        pr_alert(" %s", torture_ops[i]->name);
1964                pr_alert("\n");
1965                mutex_unlock(&fullstop_mutex);
1966                return -EINVAL;
1967        }
1968        if (cur_ops->fqs == NULL && fqs_duration != 0) {
1969                pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
1970                fqs_duration = 0;
1971        }
1972        if (cur_ops->init)
1973                cur_ops->init(); /* no "goto unwind" prior to this point!!! */
1974
1975        if (nreaders >= 0)
1976                nrealreaders = nreaders;
1977        else
1978                nrealreaders = 2 * num_online_cpus();
1979        rcu_torture_print_module_parms(cur_ops, "Start of test");
1980        fullstop = FULLSTOP_DONTSTOP;
1981
1982        /* Set up the freelist. */
1983
1984        INIT_LIST_HEAD(&rcu_torture_freelist);
1985        for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
1986                rcu_tortures[i].rtort_mbtest = 0;
1987                list_add_tail(&rcu_tortures[i].rtort_free,
1988                              &rcu_torture_freelist);
1989        }
1990
1991        /* Initialize the statistics so that each run gets its own numbers. */
1992
1993        rcu_torture_current = NULL;
1994        rcu_torture_current_version = 0;
1995        atomic_set(&n_rcu_torture_alloc, 0);
1996        atomic_set(&n_rcu_torture_alloc_fail, 0);
1997        atomic_set(&n_rcu_torture_free, 0);
1998        atomic_set(&n_rcu_torture_mberror, 0);
1999        atomic_set(&n_rcu_torture_error, 0);
2000        n_rcu_torture_barrier_error = 0;
2001        n_rcu_torture_boost_ktrerror = 0;
2002        n_rcu_torture_boost_rterror = 0;
2003        n_rcu_torture_boost_failure = 0;
2004        n_rcu_torture_boosts = 0;
2005        for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
2006                atomic_set(&rcu_torture_wcount[i], 0);
2007        for_each_possible_cpu(cpu) {
2008                for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
2009                        per_cpu(rcu_torture_count, cpu)[i] = 0;
2010                        per_cpu(rcu_torture_batch, cpu)[i] = 0;
2011                }
2012        }
2013
2014        /* Start up the kthreads. */
2015
2016        VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
2017        writer_task = kthread_create(rcu_torture_writer, NULL,
2018                                     "rcu_torture_writer");
2019        if (IS_ERR(writer_task)) {
2020                firsterr = PTR_ERR(writer_task);
2021                VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
2022                writer_task = NULL;
2023                goto unwind;
2024        }
2025        wake_up_process(writer_task);
2026        fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
2027                                   GFP_KERNEL);
2028        if (fakewriter_tasks == NULL) {
2029                VERBOSE_PRINTK_ERRSTRING("out of memory");
2030                firsterr = -ENOMEM;
2031                goto unwind;
2032        }
2033        for (i = 0; i < nfakewriters; i++) {
2034                VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
2035                fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
2036                                                  "rcu_torture_fakewriter");
2037                if (IS_ERR(fakewriter_tasks[i])) {
2038                        firsterr = PTR_ERR(fakewriter_tasks[i]);
2039                        VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
2040                        fakewriter_tasks[i] = NULL;
2041                        goto unwind;
2042                }
2043        }
2044        reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
2045                               GFP_KERNEL);
2046        if (reader_tasks == NULL) {
2047                VERBOSE_PRINTK_ERRSTRING("out of memory");
2048                firsterr = -ENOMEM;
2049                goto unwind;
2050        }
2051        for (i = 0; i < nrealreaders; i++) {
2052                VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
2053                reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
2054                                              "rcu_torture_reader");
2055                if (IS_ERR(reader_tasks[i])) {
2056                        firsterr = PTR_ERR(reader_tasks[i]);
2057                        VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
2058                        reader_tasks[i] = NULL;
2059                        goto unwind;
2060                }
2061        }
2062        if (stat_interval > 0) {
2063                VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
2064                stats_task = kthread_run(rcu_torture_stats, NULL,
2065                                        "rcu_torture_stats");
2066                if (IS_ERR(stats_task)) {
2067                        firsterr = PTR_ERR(stats_task);
2068                        VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
2069                        stats_task = NULL;
2070                        goto unwind;
2071                }
2072        }
2073        if (test_no_idle_hz) {
2074                rcu_idle_cpu = num_online_cpus() - 1;
2075
2076                if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
2077                        firsterr = -ENOMEM;
2078                        VERBOSE_PRINTK_ERRSTRING("Failed to alloc mask");
2079                        goto unwind;
2080                }
2081
2082                /* Create the shuffler thread */
2083                shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
2084                                          "rcu_torture_shuffle");
2085                if (IS_ERR(shuffler_task)) {
2086                        free_cpumask_var(shuffle_tmp_mask);
2087                        firsterr = PTR_ERR(shuffler_task);
2088                        VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
2089                        shuffler_task = NULL;
2090                        goto unwind;
2091                }
2092        }
2093        if (stutter < 0)
2094                stutter = 0;
2095        if (stutter) {
2096                /* Create the stutter thread */
2097                stutter_task = kthread_run(rcu_torture_stutter, NULL,
2098                                          "rcu_torture_stutter");
2099                if (IS_ERR(stutter_task)) {
2100                        firsterr = PTR_ERR(stutter_task);
2101                        VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
2102                        stutter_task = NULL;
2103                        goto unwind;
2104                }
2105        }
2106        if (fqs_duration < 0)
2107                fqs_duration = 0;
2108        if (fqs_duration) {
2109                /* Create the stutter thread */
2110                fqs_task = kthread_run(rcu_torture_fqs, NULL,
2111                                       "rcu_torture_fqs");
2112                if (IS_ERR(fqs_task)) {
2113                        firsterr = PTR_ERR(fqs_task);
2114                        VERBOSE_PRINTK_ERRSTRING("Failed to create fqs");
2115                        fqs_task = NULL;
2116                        goto unwind;
2117                }
2118        }
2119        if (test_boost_interval < 1)
2120                test_boost_interval = 1;
2121        if (test_boost_duration < 2)
2122                test_boost_duration = 2;
2123        if ((test_boost == 1 && cur_ops->can_boost) ||
2124            test_boost == 2) {
2125
2126                boost_starttime = jiffies + test_boost_interval * HZ;
2127                register_cpu_notifier(&rcutorture_cpu_nb);
2128                for_each_possible_cpu(i) {
2129                        if (cpu_is_offline(i))
2130                                continue;  /* Heuristic: CPU can go offline. */
2131                        retval = rcutorture_booster_init(i);
2132                        if (retval < 0) {
2133                                firsterr = retval;
2134                                goto unwind;
2135                        }
2136                }
2137        }
2138        if (shutdown_secs > 0) {
2139                shutdown_time = jiffies + shutdown_secs * HZ;
2140                shutdown_task = kthread_create(rcu_torture_shutdown, NULL,
2141                                               "rcu_torture_shutdown");
2142                if (IS_ERR(shutdown_task)) {
2143                        firsterr = PTR_ERR(shutdown_task);
2144                        VERBOSE_PRINTK_ERRSTRING("Failed to create shutdown");
2145                        shutdown_task = NULL;
2146                        goto unwind;
2147                }
2148                wake_up_process(shutdown_task);
2149        }
2150        i = rcu_torture_onoff_init();
2151        if (i != 0) {
2152                firsterr = i;
2153                goto unwind;
2154        }
2155        register_reboot_notifier(&rcutorture_shutdown_nb);
2156        i = rcu_torture_stall_init();
2157        if (i != 0) {
2158                firsterr = i;
2159                goto unwind;
2160        }
2161        retval = rcu_torture_barrier_init();
2162        if (retval != 0) {
2163                firsterr = retval;
2164                goto unwind;
2165        }
2166        rcutorture_record_test_transition();
2167        mutex_unlock(&fullstop_mutex);
2168        return 0;
2169
2170unwind:
2171        mutex_unlock(&fullstop_mutex);
2172        rcu_torture_cleanup();
2173        return firsterr;
2174}
2175
2176module_init(rcu_torture_init);
2177module_exit(rcu_torture_cleanup);
2178
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.