linux/kernel/workqueue.c
<<
>>
Prefs
   1/*
   2 * kernel/workqueue.c - generic async execution with shared worker pool
   3 *
   4 * Copyright (C) 2002           Ingo Molnar
   5 *
   6 *   Derived from the taskqueue/keventd code by:
   7 *     David Woodhouse <dwmw2@infradead.org>
   8 *     Andrew Morton
   9 *     Kai Petzke <wpp@marie.physik.tu-berlin.de>
  10 *     Theodore Ts'o <tytso@mit.edu>
  11 *
  12 * Made to use alloc_percpu by Christoph Lameter.
  13 *
  14 * Copyright (C) 2010           SUSE Linux Products GmbH
  15 * Copyright (C) 2010           Tejun Heo <tj@kernel.org>
  16 *
  17 * This is the generic async execution mechanism.  Work items as are
  18 * executed in process context.  The worker pool is shared and
  19 * automatically managed.  There are two worker pools for each CPU (one for
  20 * normal work items and the other for high priority ones) and some extra
  21 * pools for workqueues which are not bound to any specific CPU - the
  22 * number of these backing pools is dynamic.
  23 *
  24 * Please read Documentation/workqueue.txt for details.
  25 */
  26
  27#include <linux/export.h>
  28#include <linux/kernel.h>
  29#include <linux/sched.h>
  30#include <linux/init.h>
  31#include <linux/signal.h>
  32#include <linux/completion.h>
  33#include <linux/workqueue.h>
  34#include <linux/slab.h>
  35#include <linux/cpu.h>
  36#include <linux/notifier.h>
  37#include <linux/kthread.h>
  38#include <linux/hardirq.h>
  39#include <linux/mempolicy.h>
  40#include <linux/freezer.h>
  41#include <linux/kallsyms.h>
  42#include <linux/debug_locks.h>
  43#include <linux/lockdep.h>
  44#include <linux/idr.h>
  45#include <linux/jhash.h>
  46#include <linux/hashtable.h>
  47#include <linux/rculist.h>
  48#include <linux/nodemask.h>
  49#include <linux/moduleparam.h>
  50#include <linux/uaccess.h>
  51
  52#include "workqueue_internal.h"
  53
  54enum {
  55        /*
  56         * worker_pool flags
  57         *
  58         * A bound pool is either associated or disassociated with its CPU.
  59         * While associated (!DISASSOCIATED), all workers are bound to the
  60         * CPU and none has %WORKER_UNBOUND set and concurrency management
  61         * is in effect.
  62         *
  63         * While DISASSOCIATED, the cpu may be offline and all workers have
  64         * %WORKER_UNBOUND set and concurrency management disabled, and may
  65         * be executing on any CPU.  The pool behaves as an unbound one.
  66         *
  67         * Note that DISASSOCIATED should be flipped only while holding
  68         * manager_mutex to avoid changing binding state while
  69         * create_worker() is in progress.
  70         */
  71        POOL_MANAGE_WORKERS     = 1 << 0,       /* need to manage workers */
  72        POOL_DISASSOCIATED      = 1 << 2,       /* cpu can't serve workers */
  73        POOL_FREEZING           = 1 << 3,       /* freeze in progress */
  74
  75        /* worker flags */
  76        WORKER_STARTED          = 1 << 0,       /* started */
  77        WORKER_DIE              = 1 << 1,       /* die die die */
  78        WORKER_IDLE             = 1 << 2,       /* is idle */
  79        WORKER_PREP             = 1 << 3,       /* preparing to run works */
  80        WORKER_CPU_INTENSIVE    = 1 << 6,       /* cpu intensive */
  81        WORKER_UNBOUND          = 1 << 7,       /* worker is unbound */
  82        WORKER_REBOUND          = 1 << 8,       /* worker was rebound */
  83
  84        WORKER_NOT_RUNNING      = WORKER_PREP | WORKER_CPU_INTENSIVE |
  85                                  WORKER_UNBOUND | WORKER_REBOUND,
  86
  87        NR_STD_WORKER_POOLS     = 2,            /* # standard pools per cpu */
  88
  89        UNBOUND_POOL_HASH_ORDER = 6,            /* hashed by pool->attrs */
  90        BUSY_WORKER_HASH_ORDER  = 6,            /* 64 pointers */
  91
  92        MAX_IDLE_WORKERS_RATIO  = 4,            /* 1/4 of busy can be idle */
  93        IDLE_WORKER_TIMEOUT     = 300 * HZ,     /* keep idle ones for 5 mins */
  94
  95        MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,
  96                                                /* call for help after 10ms
  97                                                   (min two ticks) */
  98        MAYDAY_INTERVAL         = HZ / 10,      /* and then every 100ms */
  99        CREATE_COOLDOWN         = HZ,           /* time to breath after fail */
 100
 101        /*
 102         * Rescue workers are used only on emergencies and shared by
 103         * all cpus.  Give -20.
 104         */
 105        RESCUER_NICE_LEVEL      = -20,
 106        HIGHPRI_NICE_LEVEL      = -20,
 107
 108        WQ_NAME_LEN             = 24,
 109};
 110
 111/*
 112 * Structure fields follow one of the following exclusion rules.
 113 *
 114 * I: Modifiable by initialization/destruction paths and read-only for
 115 *    everyone else.
 116 *
 117 * P: Preemption protected.  Disabling preemption is enough and should
 118 *    only be modified and accessed from the local cpu.
 119 *
 120 * L: pool->lock protected.  Access with pool->lock held.
 121 *
 122 * X: During normal operation, modification requires pool->lock and should
 123 *    be done only from local cpu.  Either disabling preemption on local
 124 *    cpu or grabbing pool->lock is enough for read access.  If
 125 *    POOL_DISASSOCIATED is set, it's identical to L.
 126 *
 127 * MG: pool->manager_mutex and pool->lock protected.  Writes require both
 128 *     locks.  Reads can happen under either lock.
 129 *
 130 * PL: wq_pool_mutex protected.
 131 *
 132 * PR: wq_pool_mutex protected for writes.  Sched-RCU protected for reads.
 133 *
 134 * WQ: wq->mutex protected.
 135 *
 136 * WR: wq->mutex protected for writes.  Sched-RCU protected for reads.
 137 *
 138 * MD: wq_mayday_lock protected.
 139 */
 140
 141/* struct worker is defined in workqueue_internal.h */
 142
 143struct worker_pool {
 144        spinlock_t              lock;           /* the pool lock */
 145        int                     cpu;            /* I: the associated cpu */
 146        int                     node;           /* I: the associated node ID */
 147        int                     id;             /* I: pool ID */
 148        unsigned int            flags;          /* X: flags */
 149
 150        struct list_head        worklist;       /* L: list of pending works */
 151        int                     nr_workers;     /* L: total number of workers */
 152
 153        /* nr_idle includes the ones off idle_list for rebinding */
 154        int                     nr_idle;        /* L: currently idle ones */
 155
 156        struct list_head        idle_list;      /* X: list of idle workers */
 157        struct timer_list       idle_timer;     /* L: worker idle timeout */
 158        struct timer_list       mayday_timer;   /* L: SOS timer for workers */
 159
 160        /* a workers is either on busy_hash or idle_list, or the manager */
 161        DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
 162                                                /* L: hash of busy workers */
 163
 164        /* see manage_workers() for details on the two manager mutexes */
 165        struct mutex            manager_arb;    /* manager arbitration */
 166        struct mutex            manager_mutex;  /* manager exclusion */
 167        struct idr              worker_idr;     /* MG: worker IDs and iteration */
 168
 169        struct workqueue_attrs  *attrs;         /* I: worker attributes */
 170        struct hlist_node       hash_node;      /* PL: unbound_pool_hash node */
 171        int                     refcnt;         /* PL: refcnt for unbound pools */
 172
 173        /*
 174         * The current concurrency level.  As it's likely to be accessed
 175         * from other CPUs during try_to_wake_up(), put it in a separate
 176         * cacheline.
 177         */
 178        atomic_t                nr_running ____cacheline_aligned_in_smp;
 179
 180        /*
 181         * Destruction of pool is sched-RCU protected to allow dereferences
 182         * from get_work_pool().
 183         */
 184        struct rcu_head         rcu;
 185} ____cacheline_aligned_in_smp;
 186
 187/*
 188 * The per-pool workqueue.  While queued, the lower WORK_STRUCT_FLAG_BITS
 189 * of work_struct->data are used for flags and the remaining high bits
 190 * point to the pwq; thus, pwqs need to be aligned at two's power of the
 191 * number of flag bits.
 192 */
 193struct pool_workqueue {
 194        struct worker_pool      *pool;          /* I: the associated pool */
 195        struct workqueue_struct *wq;            /* I: the owning workqueue */
 196        int                     work_color;     /* L: current color */
 197        int                     flush_color;    /* L: flushing color */
 198        int                     refcnt;         /* L: reference count */
 199        int                     nr_in_flight[WORK_NR_COLORS];
 200                                                /* L: nr of in_flight works */
 201        int                     nr_active;      /* L: nr of active works */
 202        int                     max_active;     /* L: max active works */
 203        struct list_head        delayed_works;  /* L: delayed works */
 204        struct list_head        pwqs_node;      /* WR: node on wq->pwqs */
 205        struct list_head        mayday_node;    /* MD: node on wq->maydays */
 206
 207        /*
 208         * Release of unbound pwq is punted to system_wq.  See put_pwq()
 209         * and pwq_unbound_release_workfn() for details.  pool_workqueue
 210         * itself is also sched-RCU protected so that the first pwq can be
 211         * determined without grabbing wq->mutex.
 212         */
 213        struct work_struct      unbound_release_work;
 214        struct rcu_head         rcu;
 215} __aligned(1 << WORK_STRUCT_FLAG_BITS);
 216
 217/*
 218 * Structure used to wait for workqueue flush.
 219 */
 220struct wq_flusher {
 221        struct list_head        list;           /* WQ: list of flushers */
 222        int                     flush_color;    /* WQ: flush color waiting for */
 223        struct completion       done;           /* flush completion */
 224};
 225
 226struct wq_device;
 227
 228/*
 229 * The externally visible workqueue.  It relays the issued work items to
 230 * the appropriate worker_pool through its pool_workqueues.
 231 */
 232struct workqueue_struct {
 233        struct list_head        pwqs;           /* WR: all pwqs of this wq */
 234        struct list_head        list;           /* PL: list of all workqueues */
 235
 236        struct mutex            mutex;          /* protects this wq */
 237        int                     work_color;     /* WQ: current work color */
 238        int                     flush_color;    /* WQ: current flush color */
 239        atomic_t                nr_pwqs_to_flush; /* flush in progress */
 240        struct wq_flusher       *first_flusher; /* WQ: first flusher */
 241        struct list_head        flusher_queue;  /* WQ: flush waiters */
 242        struct list_head        flusher_overflow; /* WQ: flush overflow list */
 243
 244        struct list_head        maydays;        /* MD: pwqs requesting rescue */
 245        struct worker           *rescuer;       /* I: rescue worker */
 246
 247        int                     nr_drainers;    /* WQ: drain in progress */
 248        int                     saved_max_active; /* WQ: saved pwq max_active */
 249
 250        struct workqueue_attrs  *unbound_attrs; /* WQ: only for unbound wqs */
 251        struct pool_workqueue   *dfl_pwq;       /* WQ: only for unbound wqs */
 252
 253#ifdef CONFIG_SYSFS
 254        struct wq_device        *wq_dev;        /* I: for sysfs interface */
 255#endif
 256#ifdef CONFIG_LOCKDEP
 257        struct lockdep_map      lockdep_map;
 258#endif
 259        char                    name[WQ_NAME_LEN]; /* I: workqueue name */
 260
 261        /* hot fields used during command issue, aligned to cacheline */
 262        unsigned int            flags ____cacheline_aligned; /* WQ: WQ_* flags */
 263        struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
 264        struct pool_workqueue __rcu *numa_pwq_tbl[]; /* FR: unbound pwqs indexed by node */
 265};
 266
 267static struct kmem_cache *pwq_cache;
 268
 269static int wq_numa_tbl_len;             /* highest possible NUMA node id + 1 */
 270static cpumask_var_t *wq_numa_possible_cpumask;
 271                                        /* possible CPUs of each node */
 272
 273static bool wq_disable_numa;
 274module_param_named(disable_numa, wq_disable_numa, bool, 0444);
 275
 276/* see the comment above the definition of WQ_POWER_EFFICIENT */
 277#ifdef CONFIG_WQ_POWER_EFFICIENT_DEFAULT
 278static bool wq_power_efficient = true;
 279#else
 280static bool wq_power_efficient;
 281#endif
 282
 283module_param_named(power_efficient, wq_power_efficient, bool, 0444);
 284
 285static bool wq_numa_enabled;            /* unbound NUMA affinity enabled */
 286
 287/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
 288static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
 289
 290static DEFINE_MUTEX(wq_pool_mutex);     /* protects pools and workqueues list */
 291static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
 292
 293static LIST_HEAD(workqueues);           /* PL: list of all workqueues */
 294static bool workqueue_freezing;         /* PL: have wqs started freezing? */
 295
 296/* the per-cpu worker pools */
 297static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
 298                                     cpu_worker_pools);
 299
 300static DEFINE_IDR(worker_pool_idr);     /* PR: idr of all pools */
 301
 302/* PL: hash of all unbound pools keyed by pool->attrs */
 303static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
 304
 305/* I: attributes used when instantiating standard unbound pools on demand */
 306static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
 307
 308struct workqueue_struct *system_wq __read_mostly;
 309EXPORT_SYMBOL(system_wq);
 310struct workqueue_struct *system_highpri_wq __read_mostly;
 311EXPORT_SYMBOL_GPL(system_highpri_wq);
 312struct workqueue_struct *system_long_wq __read_mostly;
 313EXPORT_SYMBOL_GPL(system_long_wq);
 314struct workqueue_struct *system_unbound_wq __read_mostly;
 315EXPORT_SYMBOL_GPL(system_unbound_wq);
 316struct workqueue_struct *system_freezable_wq __read_mostly;
 317EXPORT_SYMBOL_GPL(system_freezable_wq);
 318struct workqueue_struct *system_power_efficient_wq __read_mostly;
 319EXPORT_SYMBOL_GPL(system_power_efficient_wq);
 320struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
 321EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
 322
 323static int worker_thread(void *__worker);
 324static void copy_workqueue_attrs(struct workqueue_attrs *to,
 325                                 const struct workqueue_attrs *from);
 326
 327#define CREATE_TRACE_POINTS
 328#include <trace/events/workqueue.h>
 329
 330#define assert_rcu_or_pool_mutex()                                      \
 331        rcu_lockdep_assert(rcu_read_lock_sched_held() ||                \
 332                           lockdep_is_held(&wq_pool_mutex),             \
 333                           "sched RCU or wq_pool_mutex should be held")
 334
 335#define assert_rcu_or_wq_mutex(wq)                                      \
 336        rcu_lockdep_assert(rcu_read_lock_sched_held() ||                \
 337                           lockdep_is_held(&wq->mutex),                 \
 338                           "sched RCU or wq->mutex should be held")
 339
 340#ifdef CONFIG_LOCKDEP
 341#define assert_manager_or_pool_lock(pool)                               \
 342        WARN_ONCE(debug_locks &&                                        \
 343                  !lockdep_is_held(&(pool)->manager_mutex) &&           \
 344                  !lockdep_is_held(&(pool)->lock),                      \
 345                  "pool->manager_mutex or ->lock should be held")
 346#else
 347#define assert_manager_or_pool_lock(pool)       do { } while (0)
 348#endif
 349
 350#define for_each_cpu_worker_pool(pool, cpu)                             \
 351        for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];               \
 352             (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
 353             (pool)++)
 354
 355/**
 356 * for_each_pool - iterate through all worker_pools in the system
 357 * @pool: iteration cursor
 358 * @pi: integer used for iteration
 359 *
 360 * This must be called either with wq_pool_mutex held or sched RCU read
 361 * locked.  If the pool needs to be used beyond the locking in effect, the
 362 * caller is responsible for guaranteeing that the pool stays online.
 363 *
 364 * The if/else clause exists only for the lockdep assertion and can be
 365 * ignored.
 366 */
 367#define for_each_pool(pool, pi)                                         \
 368        idr_for_each_entry(&worker_pool_idr, pool, pi)                  \
 369                if (({ assert_rcu_or_pool_mutex(); false; })) { }       \
 370                else
 371
 372/**
 373 * for_each_pool_worker - iterate through all workers of a worker_pool
 374 * @worker: iteration cursor
 375 * @wi: integer used for iteration
 376 * @pool: worker_pool to iterate workers of
 377 *
 378 * This must be called with either @pool->manager_mutex or ->lock held.
 379 *
 380 * The if/else clause exists only for the lockdep assertion and can be
 381 * ignored.
 382 */
 383#define for_each_pool_worker(worker, wi, pool)                          \
 384        idr_for_each_entry(&(pool)->worker_idr, (worker), (wi))         \
 385                if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
 386                else
 387
 388/**
 389 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
 390 * @pwq: iteration cursor
 391 * @wq: the target workqueue
 392 *
 393 * This must be called either with wq->mutex held or sched RCU read locked.
 394 * If the pwq needs to be used beyond the locking in effect, the caller is
 395 * responsible for guaranteeing that the pwq stays online.
 396 *
 397 * The if/else clause exists only for the lockdep assertion and can be
 398 * ignored.
 399 */
 400#define for_each_pwq(pwq, wq)                                           \
 401        list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node)          \
 402                if (({ assert_rcu_or_wq_mutex(wq); false; })) { }       \
 403                else
 404
 405#ifdef CONFIG_DEBUG_OBJECTS_WORK
 406
 407static struct debug_obj_descr work_debug_descr;
 408
 409static void *work_debug_hint(void *addr)
 410{
 411        return ((struct work_struct *) addr)->func;
 412}
 413
 414/*
 415 * fixup_init is called when:
 416 * - an active object is initialized
 417 */
 418static int work_fixup_init(void *addr, enum debug_obj_state state)
 419{
 420        struct work_struct *work = addr;
 421
 422        switch (state) {
 423        case ODEBUG_STATE_ACTIVE:
 424                cancel_work_sync(work);
 425                debug_object_init(work, &work_debug_descr);
 426                return 1;
 427        default:
 428                return 0;
 429        }
 430}
 431
 432/*
 433 * fixup_activate is called when:
 434 * - an active object is activated
 435 * - an unknown object is activated (might be a statically initialized object)
 436 */
 437static int work_fixup_activate(void *addr, enum debug_obj_state state)
 438{
 439        struct work_struct *work = addr;
 440
 441        switch (state) {
 442
 443        case ODEBUG_STATE_NOTAVAILABLE:
 444                /*
 445                 * This is not really a fixup. The work struct was
 446                 * statically initialized. We just make sure that it
 447                 * is tracked in the object tracker.
 448                 */
 449                if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
 450                        debug_object_init(work, &work_debug_descr);
 451                        debug_object_activate(work, &work_debug_descr);
 452                        return 0;
 453                }
 454                WARN_ON_ONCE(1);
 455                return 0;
 456
 457        case ODEBUG_STATE_ACTIVE:
 458                WARN_ON(1);
 459
 460        default:
 461                return 0;
 462        }
 463}
 464
 465/*
 466 * fixup_free is called when:
 467 * - an active object is freed
 468 */
 469static int work_fixup_free(void *addr, enum debug_obj_state state)
 470{
 471        struct work_struct *work = addr;
 472
 473        switch (state) {
 474        case ODEBUG_STATE_ACTIVE:
 475                cancel_work_sync(work);
 476                debug_object_free(work, &work_debug_descr);
 477                return 1;
 478        default:
 479                return 0;
 480        }
 481}
 482
 483static struct debug_obj_descr work_debug_descr = {
 484        .name           = "work_struct",
 485        .debug_hint     = work_debug_hint,
 486        .fixup_init     = work_fixup_init,
 487        .fixup_activate = work_fixup_activate,
 488        .fixup_free     = work_fixup_free,
 489};
 490
 491static inline void debug_work_activate(struct work_struct *work)
 492{
 493        debug_object_activate(work, &work_debug_descr);
 494}
 495
 496static inline void debug_work_deactivate(struct work_struct *work)
 497{
 498        debug_object_deactivate(work, &work_debug_descr);
 499}
 500
 501void __init_work(struct work_struct *work, int onstack)
 502{
 503        if (onstack)
 504                debug_object_init_on_stack(work, &work_debug_descr);
 505        else
 506                debug_object_init(work, &work_debug_descr);
 507}
 508EXPORT_SYMBOL_GPL(__init_work);
 509
 510void destroy_work_on_stack(struct work_struct *work)
 511{
 512        debug_object_free(work, &work_debug_descr);
 513}
 514EXPORT_SYMBOL_GPL(destroy_work_on_stack);
 515
 516#else
 517static inline void debug_work_activate(struct work_struct *work) { }
 518static inline void debug_work_deactivate(struct work_struct *work) { }
 519#endif
 520
 521/* allocate ID and assign it to @pool */
 522static int worker_pool_assign_id(struct worker_pool *pool)
 523{
 524        int ret;
 525
 526        lockdep_assert_held(&wq_pool_mutex);
 527
 528        ret = idr_alloc(&worker_pool_idr, pool, 0, 0, GFP_KERNEL);
 529        if (ret >= 0) {
 530                pool->id = ret;
 531                return 0;
 532        }
 533        return ret;
 534}
 535
 536/**
 537 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
 538 * @wq: the target workqueue
 539 * @node: the node ID
 540 *
 541 * This must be called either with pwq_lock held or sched RCU read locked.
 542 * If the pwq needs to be used beyond the locking in effect, the caller is
 543 * responsible for guaranteeing that the pwq stays online.
 544 *
 545 * Return: The unbound pool_workqueue for @node.
 546 */
 547static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
 548                                                  int node)
 549{
 550        assert_rcu_or_wq_mutex(wq);
 551        return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
 552}
 553
 554static unsigned int work_color_to_flags(int color)
 555{
 556        return color << WORK_STRUCT_COLOR_SHIFT;
 557}
 558
 559static int get_work_color(struct work_struct *work)
 560{
 561        return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
 562                ((1 << WORK_STRUCT_COLOR_BITS) - 1);
 563}
 564
 565static int work_next_color(int color)
 566{
 567        return (color + 1) % WORK_NR_COLORS;
 568}
 569
 570/*
 571 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
 572 * contain the pointer to the queued pwq.  Once execution starts, the flag
 573 * is cleared and the high bits contain OFFQ flags and pool ID.
 574 *
 575 * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
 576 * and clear_work_data() can be used to set the pwq, pool or clear
 577 * work->data.  These functions should only be called while the work is
 578 * owned - ie. while the PENDING bit is set.
 579 *
 580 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
 581 * corresponding to a work.  Pool is available once the work has been
 582 * queued anywhere after initialization until it is sync canceled.  pwq is
 583 * available only while the work item is queued.
 584 *
 585 * %WORK_OFFQ_CANCELING is used to mark a work item which is being
 586 * canceled.  While being canceled, a work item may have its PENDING set
 587 * but stay off timer and worklist for arbitrarily long and nobody should
 588 * try to steal the PENDING bit.
 589 */
 590static inline void set_work_data(struct work_struct *work, unsigned long data,
 591                                 unsigned long flags)
 592{
 593        WARN_ON_ONCE(!work_pending(work));
 594        atomic_long_set(&work->data, data | flags | work_static(work));
 595}
 596
 597static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
 598                         unsigned long extra_flags)
 599{
 600        set_work_data(work, (unsigned long)pwq,
 601                      WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
 602}
 603
 604static void set_work_pool_and_keep_pending(struct work_struct *work,
 605                                           int pool_id)
 606{
 607        set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
 608                      WORK_STRUCT_PENDING);
 609}
 610
 611static void set_work_pool_and_clear_pending(struct work_struct *work,
 612                                            int pool_id)
 613{
 614        /*
 615         * The following wmb is paired with the implied mb in
 616         * test_and_set_bit(PENDING) and ensures all updates to @work made
 617         * here are visible to and precede any updates by the next PENDING
 618         * owner.
 619         */
 620        smp_wmb();
 621        set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
 622}
 623
 624static void clear_work_data(struct work_struct *work)
 625{
 626        smp_wmb();      /* see set_work_pool_and_clear_pending() */
 627        set_work_data(work, WORK_STRUCT_NO_POOL, 0);
 628}
 629
 630static struct pool_workqueue *get_work_pwq(struct work_struct *work)
 631{
 632        unsigned long data = atomic_long_read(&work->data);
 633
 634        if (data & WORK_STRUCT_PWQ)
 635                return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
 636        else
 637                return NULL;
 638}
 639
 640/**
 641 * get_work_pool - return the worker_pool a given work was associated with
 642 * @work: the work item of interest
 643 *
 644 * Pools are created and destroyed under wq_pool_mutex, and allows read
 645 * access under sched-RCU read lock.  As such, this function should be
 646 * called under wq_pool_mutex or with preemption disabled.
 647 *
 648 * All fields of the returned pool are accessible as long as the above
 649 * mentioned locking is in effect.  If the returned pool needs to be used
 650 * beyond the critical section, the caller is responsible for ensuring the
 651 * returned pool is and stays online.
 652 *
 653 * Return: The worker_pool @work was last associated with.  %NULL if none.
 654 */
 655static struct worker_pool *get_work_pool(struct work_struct *work)
 656{
 657        unsigned long data = atomic_long_read(&work->data);
 658        int pool_id;
 659
 660        assert_rcu_or_pool_mutex();
 661
 662        if (data & WORK_STRUCT_PWQ)
 663                return ((struct pool_workqueue *)
 664                        (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
 665
 666        pool_id = data >> WORK_OFFQ_POOL_SHIFT;
 667        if (pool_id == WORK_OFFQ_POOL_NONE)
 668                return NULL;
 669
 670        return idr_find(&worker_pool_idr, pool_id);
 671}
 672
 673/**
 674 * get_work_pool_id - return the worker pool ID a given work is associated with
 675 * @work: the work item of interest
 676 *
 677 * Return: The worker_pool ID @work was last associated with.
 678 * %WORK_OFFQ_POOL_NONE if none.
 679 */
 680static int get_work_pool_id(struct work_struct *work)
 681{
 682        unsigned long data = atomic_long_read(&work->data);
 683
 684        if (data & WORK_STRUCT_PWQ)
 685                return ((struct pool_workqueue *)
 686                        (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
 687
 688        return data >> WORK_OFFQ_POOL_SHIFT;
 689}
 690
 691static void mark_work_canceling(struct work_struct *work)
 692{
 693        unsigned long pool_id = get_work_pool_id(work);
 694
 695        pool_id <<= WORK_OFFQ_POOL_SHIFT;
 696        set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
 697}
 698
 699static bool work_is_canceling(struct work_struct *work)
 700{
 701        unsigned long data = atomic_long_read(&work->data);
 702
 703        return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
 704}
 705
 706/*
 707 * Policy functions.  These define the policies on how the global worker
 708 * pools are managed.  Unless noted otherwise, these functions assume that
 709 * they're being called with pool->lock held.
 710 */
 711
 712static bool __need_more_worker(struct worker_pool *pool)
 713{
 714        return !atomic_read(&pool->nr_running);
 715}
 716
 717/*
 718 * Need to wake up a worker?  Called from anything but currently
 719 * running workers.
 720 *
 721 * Note that, because unbound workers never contribute to nr_running, this
 722 * function will always return %true for unbound pools as long as the
 723 * worklist isn't empty.
 724 */
 725static bool need_more_worker(struct worker_pool *pool)
 726{
 727        return !list_empty(&pool->worklist) && __need_more_worker(pool);
 728}
 729
 730/* Can I start working?  Called from busy but !running workers. */
 731static bool may_start_working(struct worker_pool *pool)
 732{
 733        return pool->nr_idle;
 734}
 735
 736/* Do I need to keep working?  Called from currently running workers. */
 737static bool keep_working(struct worker_pool *pool)
 738{
 739        return !list_empty(&pool->worklist) &&
 740                atomic_read(&pool->nr_running) <= 1;
 741}
 742
 743/* Do we need a new worker?  Called from manager. */
 744static bool need_to_create_worker(struct worker_pool *pool)
 745{
 746        return need_more_worker(pool) && !may_start_working(pool);
 747}
 748
 749/* Do I need to be the manager? */
 750static bool need_to_manage_workers(struct worker_pool *pool)
 751{
 752        return need_to_create_worker(pool) ||
 753                (pool->flags & POOL_MANAGE_WORKERS);
 754}
 755
 756/* Do we have too many workers and should some go away? */
 757static bool too_many_workers(struct worker_pool *pool)
 758{
 759        bool managing = mutex_is_locked(&pool->manager_arb);
 760        int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
 761        int nr_busy = pool->nr_workers - nr_idle;
 762
 763        /*
 764         * nr_idle and idle_list may disagree if idle rebinding is in
 765         * progress.  Never return %true if idle_list is empty.
 766         */
 767        if (list_empty(&pool->idle_list))
 768                return false;
 769
 770        return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
 771}
 772
 773/*
 774 * Wake up functions.
 775 */
 776
 777/* Return the first worker.  Safe with preemption disabled */
 778static struct worker *first_worker(struct worker_pool *pool)
 779{
 780        if (unlikely(list_empty(&pool->idle_list)))
 781                return NULL;
 782
 783        return list_first_entry(&pool->idle_list, struct worker, entry);
 784}
 785
 786/**
 787 * wake_up_worker - wake up an idle worker
 788 * @pool: worker pool to wake worker from
 789 *
 790 * Wake up the first idle worker of @pool.
 791 *
 792 * CONTEXT:
 793 * spin_lock_irq(pool->lock).
 794 */
 795static void wake_up_worker(struct worker_pool *pool)
 796{
 797        struct worker *worker = first_worker(pool);
 798
 799        if (likely(worker))
 800                wake_up_process(worker->task);
 801}
 802
 803/**
 804 * wq_worker_waking_up - a worker is waking up
 805 * @task: task waking up
 806 * @cpu: CPU @task is waking up to
 807 *
 808 * This function is called during try_to_wake_up() when a worker is
 809 * being awoken.
 810 *
 811 * CONTEXT:
 812 * spin_lock_irq(rq->lock)
 813 */
 814void wq_worker_waking_up(struct task_struct *task, int cpu)
 815{
 816        struct worker *worker = kthread_data(task);
 817
 818        if (!(worker->flags & WORKER_NOT_RUNNING)) {
 819                WARN_ON_ONCE(worker->pool->cpu != cpu);
 820                atomic_inc(&worker->pool->nr_running);
 821        }
 822}
 823
 824/**
 825 * wq_worker_sleeping - a worker is going to sleep
 826 * @task: task going to sleep
 827 * @cpu: CPU in question, must be the current CPU number
 828 *
 829 * This function is called during schedule() when a busy worker is
 830 * going to sleep.  Worker on the same cpu can be woken up by
 831 * returning pointer to its task.
 832 *
 833 * CONTEXT:
 834 * spin_lock_irq(rq->lock)
 835 *
 836 * Return:
 837 * Worker task on @cpu to wake up, %NULL if none.
 838 */
 839struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
 840{
 841        struct worker *worker = kthread_data(task), *to_wakeup = NULL;
 842        struct worker_pool *pool;
 843
 844        /*
 845         * Rescuers, which may not have all the fields set up like normal
 846         * workers, also reach here, let's not access anything before
 847         * checking NOT_RUNNING.
 848         */
 849        if (worker->flags & WORKER_NOT_RUNNING)
 850                return NULL;
 851
 852        pool = worker->pool;
 853
 854        /* this can only happen on the local cpu */
 855        if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
 856                return NULL;
 857
 858        /*
 859         * The counterpart of the following dec_and_test, implied mb,
 860         * worklist not empty test sequence is in insert_work().
 861         * Please read comment there.
 862         *
 863         * NOT_RUNNING is clear.  This means that we're bound to and
 864         * running on the local cpu w/ rq lock held and preemption
 865         * disabled, which in turn means that none else could be
 866         * manipulating idle_list, so dereferencing idle_list without pool
 867         * lock is safe.
 868         */
 869        if (atomic_dec_and_test(&pool->nr_running) &&
 870            !list_empty(&pool->worklist))
 871                to_wakeup = first_worker(pool);
 872        return to_wakeup ? to_wakeup->task : NULL;
 873}
 874
 875/**
 876 * worker_set_flags - set worker flags and adjust nr_running accordingly
 877 * @worker: self
 878 * @flags: flags to set
 879 * @wakeup: wakeup an idle worker if necessary
 880 *
 881 * Set @flags in @worker->flags and adjust nr_running accordingly.  If
 882 * nr_running becomes zero and @wakeup is %true, an idle worker is
 883 * woken up.
 884 *
 885 * CONTEXT:
 886 * spin_lock_irq(pool->lock)
 887 */
 888static inline void worker_set_flags(struct worker *worker, unsigned int flags,
 889                                    bool wakeup)
 890{
 891        struct worker_pool *pool = worker->pool;
 892
 893        WARN_ON_ONCE(worker->task != current);
 894
 895        /*
 896         * If transitioning into NOT_RUNNING, adjust nr_running and
 897         * wake up an idle worker as necessary if requested by
 898         * @wakeup.
 899         */
 900        if ((flags & WORKER_NOT_RUNNING) &&
 901            !(worker->flags & WORKER_NOT_RUNNING)) {
 902                if (wakeup) {
 903                        if (atomic_dec_and_test(&pool->nr_running) &&
 904                            !list_empty(&pool->worklist))
 905                                wake_up_worker(pool);
 906                } else
 907                        atomic_dec(&pool->nr_running);
 908        }
 909
 910        worker->flags |= flags;
 911}
 912
 913/**
 914 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
 915 * @worker: self
 916 * @flags: flags to clear
 917 *
 918 * Clear @flags in @worker->flags and adjust nr_running accordingly.
 919 *
 920 * CONTEXT:
 921 * spin_lock_irq(pool->lock)
 922 */
 923static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
 924{
 925        struct worker_pool *pool = worker->pool;
 926        unsigned int oflags = worker->flags;
 927
 928        WARN_ON_ONCE(worker->task != current);
 929
 930        worker->flags &= ~flags;
 931
 932        /*
 933         * If transitioning out of NOT_RUNNING, increment nr_running.  Note
 934         * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask
 935         * of multiple flags, not a single flag.
 936         */
 937        if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
 938                if (!(worker->flags & WORKER_NOT_RUNNING))
 939                        atomic_inc(&pool->nr_running);
 940}
 941
 942/**
 943 * find_worker_executing_work - find worker which is executing a work
 944 * @pool: pool of interest
 945 * @work: work to find worker for
 946 *
 947 * Find a worker which is executing @work on @pool by searching
 948 * @pool->busy_hash which is keyed by the address of @work.  For a worker
 949 * to match, its current execution should match the address of @work and
 950 * its work function.  This is to avoid unwanted dependency between
 951 * unrelated work executions through a work item being recycled while still
 952 * being executed.
 953 *
 954 * This is a bit tricky.  A work item may be freed once its execution
 955 * starts and nothing prevents the freed area from being recycled for
 956 * another work item.  If the same work item address ends up being reused
 957 * before the original execution finishes, workqueue will identify the
 958 * recycled work item as currently executing and make it wait until the
 959 * current execution finishes, introducing an unwanted dependency.
 960 *
 961 * This function checks the work item address and work function to avoid
 962 * false positives.  Note that this isn't complete as one may construct a
 963 * work function which can introduce dependency onto itself through a
 964 * recycled work item.  Well, if somebody wants to shoot oneself in the
 965 * foot that badly, there's only so much we can do, and if such deadlock
 966 * actually occurs, it should be easy to locate the culprit work function.
 967 *
 968 * CONTEXT:
 969 * spin_lock_irq(pool->lock).
 970 *
 971 * Return:
 972 * Pointer to worker which is executing @work if found, %NULL
 973 * otherwise.
 974 */
 975static struct worker *find_worker_executing_work(struct worker_pool *pool,
 976                                                 struct work_struct *work)
 977{
 978        struct worker *worker;
 979
 980        hash_for_each_possible(pool->busy_hash, worker, hentry,
 981                               (unsigned long)work)
 982                if (worker->current_work == work &&
 983                    worker->current_func == work->func)
 984                        return worker;
 985
 986        return NULL;
 987}
 988
 989/**
 990 * move_linked_works - move linked works to a list
 991 * @work: start of series of works to be scheduled
 992 * @head: target list to append @work to
 993 * @nextp: out paramter for nested worklist walking
 994 *
 995 * Schedule linked works starting from @work to @head.  Work series to
 996 * be scheduled starts at @work and includes any consecutive work with
 997 * WORK_STRUCT_LINKED set in its predecessor.
 998 *
 999 * If @nextp is not NULL, it's updated to point to the next work of
1000 * the last scheduled work.  This allows move_linked_works() to be
1001 * nested inside outer list_for_each_entry_safe().
1002 *
1003 * CONTEXT:
1004 * spin_lock_irq(pool->lock).
1005 */
1006static void move_linked_works(struct work_struct *work, struct list_head *head,
1007                              struct work_struct **nextp)
1008{
1009        struct work_struct *n;
1010
1011        /*
1012         * Linked worklist will always end before the end of the list,
1013         * use NULL for list head.
1014         */
1015        list_for_each_entry_safe_from(work, n, NULL, entry) {
1016                list_move_tail(&work->entry, head);
1017                if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1018                        break;
1019        }
1020
1021        /*
1022         * If we're already inside safe list traversal and have moved
1023         * multiple works to the scheduled queue, the next position
1024         * needs to be updated.
1025         */
1026        if (nextp)
1027                *nextp = n;
1028}
1029
1030/**
1031 * get_pwq - get an extra reference on the specified pool_workqueue
1032 * @pwq: pool_workqueue to get
1033 *
1034 * Obtain an extra reference on @pwq.  The caller should guarantee that
1035 * @pwq has positive refcnt and be holding the matching pool->lock.
1036 */
1037static void get_pwq(struct pool_workqueue *pwq)
1038{
1039        lockdep_assert_held(&pwq->pool->lock);
1040        WARN_ON_ONCE(pwq->refcnt <= 0);
1041        pwq->refcnt++;
1042}
1043
1044/**
1045 * put_pwq - put a pool_workqueue reference
1046 * @pwq: pool_workqueue to put
1047 *
1048 * Drop a reference of @pwq.  If its refcnt reaches zero, schedule its
1049 * destruction.  The caller should be holding the matching pool->lock.
1050 */
1051static void put_pwq(struct pool_workqueue *pwq)
1052{
1053        lockdep_assert_held(&pwq->pool->lock);
1054        if (likely(--pwq->refcnt))
1055                return;
1056        if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1057                return;
1058        /*
1059         * @pwq can't be released under pool->lock, bounce to
1060         * pwq_unbound_release_workfn().  This never recurses on the same
1061         * pool->lock as this path is taken only for unbound workqueues and
1062         * the release work item is scheduled on a per-cpu workqueue.  To
1063         * avoid lockdep warning, unbound pool->locks are given lockdep
1064         * subclass of 1 in get_unbound_pool().
1065         */
1066        schedule_work(&pwq->unbound_release_work);
1067}
1068
1069/**
1070 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1071 * @pwq: pool_workqueue to put (can be %NULL)
1072 *
1073 * put_pwq() with locking.  This function also allows %NULL @pwq.
1074 */
1075static void put_pwq_unlocked(struct pool_workqueue *pwq)
1076{
1077        if (pwq) {
1078                /*
1079                 * As both pwqs and pools are sched-RCU protected, the
1080                 * following lock operations are safe.
1081                 */
1082                spin_lock_irq(&pwq->pool->lock);
1083                put_pwq(pwq);
1084                spin_unlock_irq(&pwq->pool->lock);
1085        }
1086}
1087
1088static void pwq_activate_delayed_work(struct work_struct *work)
1089{
1090        struct pool_workqueue *pwq = get_work_pwq(work);
1091
1092        trace_workqueue_activate_work(work);
1093        move_linked_works(work, &pwq->pool->worklist, NULL);
1094        __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1095        pwq->nr_active++;
1096}
1097
1098static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
1099{
1100        struct work_struct *work = list_first_entry(&pwq->delayed_works,
1101                                                    struct work_struct, entry);
1102
1103        pwq_activate_delayed_work(work);
1104}
1105
1106/**
1107 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1108 * @pwq: pwq of interest
1109 * @color: color of work which left the queue
1110 *
1111 * A work either has completed or is removed from pending queue,
1112 * decrement nr_in_flight of its pwq and handle workqueue flushing.
1113 *
1114 * CONTEXT:
1115 * spin_lock_irq(pool->lock).
1116 */
1117static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1118{
1119        /* uncolored work items don't participate in flushing or nr_active */
1120        if (color == WORK_NO_COLOR)
1121                goto out_put;
1122
1123        pwq->nr_in_flight[color]--;
1124
1125        pwq->nr_active--;
1126        if (!list_empty(&pwq->delayed_works)) {
1127                /* one down, submit a delayed one */
1128                if (pwq->nr_active < pwq->max_active)
1129                        pwq_activate_first_delayed(pwq);
1130        }
1131
1132        /* is flush in progress and are we at the flushing tip? */
1133        if (likely(pwq->flush_color != color))
1134                goto out_put;
1135
1136        /* are there still in-flight works? */
1137        if (pwq->nr_in_flight[color])
1138                goto out_put;
1139
1140        /* this pwq is done, clear flush_color */
1141        pwq->flush_color = -1;
1142
1143        /*
1144         * If this was the last pwq, wake up the first flusher.  It
1145         * will handle the rest.
1146         */
1147        if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1148                complete(&pwq->wq->first_flusher->done);
1149out_put:
1150        put_pwq(pwq);
1151}
1152
1153/**
1154 * try_to_grab_pending - steal work item from worklist and disable irq
1155 * @work: work item to steal
1156 * @is_dwork: @work is a delayed_work
1157 * @flags: place to store irq state
1158 *
1159 * Try to grab PENDING bit of @work.  This function can handle @work in any
1160 * stable state - idle, on timer or on worklist.
1161 *
1162 * Return:
1163 *  1           if @work was pending and we successfully stole PENDING
1164 *  0           if @work was idle and we claimed PENDING
1165 *  -EAGAIN     if PENDING couldn't be grabbed at the moment, safe to busy-retry
1166 *  -ENOENT     if someone else is canceling @work, this state may persist
1167 *              for arbitrarily long
1168 *
1169 * Note:
1170 * On >= 0 return, the caller owns @work's PENDING bit.  To avoid getting
1171 * interrupted while holding PENDING and @work off queue, irq must be
1172 * disabled on entry.  This, combined with delayed_work->timer being
1173 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
1174 *
1175 * On successful return, >= 0, irq is disabled and the caller is
1176 * responsible for releasing it using local_irq_restore(*@flags).
1177 *
1178 * This function is safe to call from any context including IRQ handler.
1179 */
1180static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1181                               unsigned long *flags)
1182{
1183        struct worker_pool *pool;
1184        struct pool_workqueue *pwq;
1185
1186        local_irq_save(*flags);
1187
1188        /* try to steal the timer if it exists */
1189        if (is_dwork) {
1190                struct delayed_work *dwork = to_delayed_work(work);
1191
1192                /*
1193                 * dwork->timer is irqsafe.  If del_timer() fails, it's
1194                 * guaranteed that the timer is not queued anywhere and not
1195                 * running on the local CPU.
1196                 */
1197                if (likely(del_timer(&dwork->timer)))
1198                        return 1;
1199        }
1200
1201        /* try to claim PENDING the normal way */
1202        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1203                return 0;
1204
1205        /*
1206         * The queueing is in progress, or it is already queued. Try to
1207         * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1208         */
1209        pool = get_work_pool(work);
1210        if (!pool)
1211                goto fail;
1212
1213        spin_lock(&pool->lock);
1214        /*
1215         * work->data is guaranteed to point to pwq only while the work
1216         * item is queued on pwq->wq, and both updating work->data to point
1217         * to pwq on queueing and to pool on dequeueing are done under
1218         * pwq->pool->lock.  This in turn guarantees that, if work->data
1219         * points to pwq which is associated with a locked pool, the work
1220         * item is currently queued on that pool.
1221         */
1222        pwq = get_work_pwq(work);
1223        if (pwq && pwq->pool == pool) {
1224                debug_work_deactivate(work);
1225
1226                /*
1227                 * A delayed work item cannot be grabbed directly because
1228                 * it might have linked NO_COLOR work items which, if left
1229                 * on the delayed_list, will confuse pwq->nr_active
1230                 * management later on and cause stall.  Make sure the work
1231                 * item is activated before grabbing.
1232                 */
1233                if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
1234                        pwq_activate_delayed_work(work);
1235
1236                list_del_init(&work->entry);
1237                pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
1238
1239                /* work->data points to pwq iff queued, point to pool */
1240                set_work_pool_and_keep_pending(work, pool->id);
1241
1242                spin_unlock(&pool->lock);
1243                return 1;
1244        }
1245        spin_unlock(&pool->lock);
1246fail:
1247        local_irq_restore(*flags);
1248        if (work_is_canceling(work))
1249                return -ENOENT;
1250        cpu_relax();
1251        return -EAGAIN;
1252}
1253
1254/**
1255 * insert_work - insert a work into a pool
1256 * @pwq: pwq @work belongs to
1257 * @work: work to insert
1258 * @head: insertion point
1259 * @extra_flags: extra WORK_STRUCT_* flags to set
1260 *
1261 * Insert @work which belongs to @pwq after @head.  @extra_flags is or'd to
1262 * work_struct flags.
1263 *
1264 * CONTEXT:
1265 * spin_lock_irq(pool->lock).
1266 */
1267static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1268                        struct list_head *head, unsigned int extra_flags)
1269{
1270        struct worker_pool *pool = pwq->pool;
1271
1272        /* we own @work, set data and link */
1273        set_work_pwq(work, pwq, extra_flags);
1274        list_add_tail(&work->entry, head);
1275        get_pwq(pwq);
1276
1277        /*
1278         * Ensure either wq_worker_sleeping() sees the above
1279         * list_add_tail() or we see zero nr_running to avoid workers lying
1280         * around lazily while there are works to be processed.
1281         */
1282        smp_mb();
1283
1284        if (__need_more_worker(pool))
1285                wake_up_worker(pool);
1286}
1287
1288/*
1289 * Test whether @work is being queued from another work executing on the
1290 * same workqueue.
1291 */
1292static bool is_chained_work(struct workqueue_struct *wq)
1293{
1294        struct worker *worker;
1295
1296        worker = current_wq_worker();
1297        /*
1298         * Return %true iff I'm a worker execuing a work item on @wq.  If
1299         * I'm @worker, it's safe to dereference it without locking.
1300         */
1301        return worker && worker->current_pwq->wq == wq;
1302}
1303
1304static void __queue_work(int cpu, struct workqueue_struct *wq,
1305                         struct work_struct *work)
1306{
1307        struct pool_workqueue *pwq;
1308        struct worker_pool *last_pool;
1309        struct list_head *worklist;
1310        unsigned int work_flags;
1311        unsigned int req_cpu = cpu;
1312
1313        /*
1314         * While a work item is PENDING && off queue, a task trying to
1315         * steal the PENDING will busy-loop waiting for it to either get
1316         * queued or lose PENDING.  Grabbing PENDING and queueing should
1317         * happen with IRQ disabled.
1318         */
1319        WARN_ON_ONCE(!irqs_disabled());
1320
1321        debug_work_activate(work);
1322
1323        /* if dying, only works from the same workqueue are allowed */
1324        if (unlikely(wq->flags & __WQ_DRAINING) &&
1325            WARN_ON_ONCE(!is_chained_work(wq)))
1326                return;
1327retry:
1328        if (req_cpu == WORK_CPU_UNBOUND)
1329                cpu = raw_smp_processor_id();
1330
1331        /* pwq which will be used unless @work is executing elsewhere */
1332        if (!(wq->flags & WQ_UNBOUND))
1333                pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1334        else
1335                pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1336
1337        /*
1338         * If @work was previously on a different pool, it might still be
1339         * running there, in which case the work needs to be queued on that
1340         * pool to guarantee non-reentrancy.
1341         */
1342        last_pool = get_work_pool(work);
1343        if (last_pool && last_pool != pwq->pool) {
1344                struct worker *worker;
1345
1346                spin_lock(&last_pool->lock);
1347
1348                worker = find_worker_executing_work(last_pool, work);
1349
1350                if (worker && worker->current_pwq->wq == wq) {
1351                        pwq = worker->current_pwq;
1352                } else {
1353                        /* meh... not running there, queue here */
1354                        spin_unlock(&last_pool->lock);
1355                        spin_lock(&pwq->pool->lock);
1356                }
1357        } else {
1358                spin_lock(&pwq->pool->lock);
1359        }
1360
1361        /*
1362         * pwq is determined and locked.  For unbound pools, we could have
1363         * raced with pwq release and it could already be dead.  If its
1364         * refcnt is zero, repeat pwq selection.  Note that pwqs never die
1365         * without another pwq replacing it in the numa_pwq_tbl or while
1366         * work items are executing on it, so the retrying is guaranteed to
1367         * make forward-progress.
1368         */
1369        if (unlikely(!pwq->refcnt)) {
1370                if (wq->flags & WQ_UNBOUND) {
1371                        spin_unlock(&pwq->pool->lock);
1372                        cpu_relax();
1373                        goto retry;
1374                }
1375                /* oops */
1376                WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1377                          wq->name, cpu);
1378        }
1379
1380        /* pwq determined, queue */
1381        trace_workqueue_queue_work(req_cpu, pwq, work);
1382
1383        if (WARN_ON(!list_empty(&work->entry))) {
1384                spin_unlock(&pwq->pool->lock);
1385                return;
1386        }
1387
1388        pwq->nr_in_flight[pwq->work_color]++;
1389        work_flags = work_color_to_flags(pwq->work_color);
1390
1391        if (likely(pwq->nr_active < pwq->max_active)) {
1392                trace_workqueue_activate_work(work);
1393                pwq->nr_active++;
1394                worklist = &pwq->pool->worklist;
1395        } else {
1396                work_flags |= WORK_STRUCT_DELAYED;
1397                worklist = &pwq->delayed_works;
1398        }
1399
1400        insert_work(pwq, work, worklist, work_flags);
1401
1402        spin_unlock(&pwq->pool->lock);
1403}
1404
1405/**
1406 * queue_work_on - queue work on specific cpu
1407 * @cpu: CPU number to execute work on
1408 * @wq: workqueue to use
1409 * @work: work to queue
1410 *
1411 * We queue the work to a specific CPU, the caller must ensure it
1412 * can't go away.
1413 *
1414 * Return: %false if @work was already on a queue, %true otherwise.
1415 */
1416bool queue_work_on(int cpu, struct workqueue_struct *wq,
1417                   struct work_struct *work)
1418{
1419        bool ret = false;
1420        unsigned long flags;
1421
1422        local_irq_save(flags);
1423
1424        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1425                __queue_work(cpu, wq, work);
1426                ret = true;
1427        }
1428
1429        local_irq_restore(flags);
1430        return ret;
1431}
1432EXPORT_SYMBOL(queue_work_on);
1433
1434void delayed_work_timer_fn(unsigned long __data)
1435{
1436        struct delayed_work *dwork = (struct delayed_work *)__data;
1437
1438        /* should have been called from irqsafe timer with irq already off */
1439        __queue_work(dwork->cpu, dwork->wq, &dwork->work);
1440}
1441EXPORT_SYMBOL(delayed_work_timer_fn);
1442
1443static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1444                                struct delayed_work *dwork, unsigned long delay)
1445{
1446        struct timer_list *timer = &dwork->timer;
1447        struct work_struct *work = &dwork->work;
1448
1449        WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
1450                     timer->data != (unsigned long)dwork);
1451        WARN_ON_ONCE(timer_pending(timer));
1452        WARN_ON_ONCE(!list_empty(&work->entry));
1453
1454        /*
1455         * If @delay is 0, queue @dwork->work immediately.  This is for
1456         * both optimization and correctness.  The earliest @timer can
1457         * expire is on the closest next tick and delayed_work users depend
1458         * on that there's no such delay when @delay is 0.
1459         */
1460        if (!delay) {
1461                __queue_work(cpu, wq, &dwork->work);
1462                return;
1463        }
1464
1465        timer_stats_timer_set_start_info(&dwork->timer);
1466
1467        dwork->wq = wq;
1468        dwork->cpu = cpu;
1469        timer->expires = jiffies + delay;
1470
1471        if (unlikely(cpu != WORK_CPU_UNBOUND))
1472                add_timer_on(timer, cpu);
1473        else
1474                add_timer(timer);
1475}
1476
1477/**
1478 * queue_delayed_work_on - queue work on specific CPU after delay
1479 * @cpu: CPU number to execute work on
1480 * @wq: workqueue to use
1481 * @dwork: work to queue
1482 * @delay: number of jiffies to wait before queueing
1483 *
1484 * Return: %false if @work was already on a queue, %true otherwise.  If
1485 * @delay is zero and @dwork is idle, it will be scheduled for immediate
1486 * execution.
1487 */
1488bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1489                           struct delayed_work *dwork, unsigned long delay)
1490{
1491        struct work_struct *work = &dwork->work;
1492        bool ret = false;
1493        unsigned long flags;
1494
1495        /* read the comment in __queue_work() */
1496        local_irq_save(flags);
1497
1498        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1499                __queue_delayed_work(cpu, wq, dwork, delay);
1500                ret = true;
1501        }
1502
1503        local_irq_restore(flags);
1504        return ret;
1505}
1506EXPORT_SYMBOL(queue_delayed_work_on);
1507
1508/**
1509 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1510 * @cpu: CPU number to execute work on
1511 * @wq: workqueue to use
1512 * @dwork: work to queue
1513 * @delay: number of jiffies to wait before queueing
1514 *
1515 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1516 * modify @dwork's timer so that it expires after @delay.  If @delay is
1517 * zero, @work is guaranteed to be scheduled immediately regardless of its
1518 * current state.
1519 *
1520 * Return: %false if @dwork was idle and queued, %true if @dwork was
1521 * pending and its timer was modified.
1522 *
1523 * This function is safe to call from any context including IRQ handler.
1524 * See try_to_grab_pending() for details.
1525 */
1526bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1527                         struct delayed_work *dwork, unsigned long delay)
1528{
1529        unsigned long flags;
1530        int ret;
1531
1532        do {
1533                ret = try_to_grab_pending(&dwork->work, true, &flags);
1534        } while (unlikely(ret == -EAGAIN));
1535
1536        if (likely(ret >= 0)) {
1537                __queue_delayed_work(cpu, wq, dwork, delay);
1538                local_irq_restore(flags);
1539        }
1540
1541        /* -ENOENT from try_to_grab_pending() becomes %true */
1542        return ret;
1543}
1544EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1545
1546/**
1547 * worker_enter_idle - enter idle state
1548 * @worker: worker which is entering idle state
1549 *
1550 * @worker is entering idle state.  Update stats and idle timer if
1551 * necessary.
1552 *
1553 * LOCKING:
1554 * spin_lock_irq(pool->lock).
1555 */
1556static void worker_enter_idle(struct worker *worker)
1557{
1558        struct worker_pool *pool = worker->pool;
1559
1560        if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1561            WARN_ON_ONCE(!list_empty(&worker->entry) &&
1562                         (worker->hentry.next || worker->hentry.pprev)))
1563                return;
1564
1565        /* can't use worker_set_flags(), also called from start_worker() */
1566        worker->flags |= WORKER_IDLE;
1567        pool->nr_idle++;
1568        worker->last_active = jiffies;
1569
1570        /* idle_list is LIFO */
1571        list_add(&worker->entry, &pool->idle_list);
1572
1573        if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1574                mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1575
1576        /*
1577         * Sanity check nr_running.  Because wq_unbind_fn() releases
1578         * pool->lock between setting %WORKER_UNBOUND and zapping
1579         * nr_running, the warning may trigger spuriously.  Check iff
1580         * unbind is not in progress.
1581         */
1582        WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1583                     pool->nr_workers == pool->nr_idle &&
1584                     atomic_read(&pool->nr_running));
1585}
1586
1587/**
1588 * worker_leave_idle - leave idle state
1589 * @worker: worker which is leaving idle state
1590 *
1591 * @worker is leaving idle state.  Update stats.
1592 *
1593 * LOCKING:
1594 * spin_lock_irq(pool->lock).
1595 */
1596static void worker_leave_idle(struct worker *worker)
1597{
1598        struct worker_pool *pool = worker->pool;
1599
1600        if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1601                return;
1602        worker_clr_flags(worker, WORKER_IDLE);
1603        pool->nr_idle--;
1604        list_del_init(&worker->entry);
1605}
1606
1607/**
1608 * worker_maybe_bind_and_lock - try to bind %current to worker_pool and lock it
1609 * @pool: target worker_pool
1610 *
1611 * Bind %current to the cpu of @pool if it is associated and lock @pool.
1612 *
1613 * Works which are scheduled while the cpu is online must at least be
1614 * scheduled to a worker which is bound to the cpu so that if they are
1615 * flushed from cpu callbacks while cpu is going down, they are
1616 * guaranteed to execute on the cpu.
1617 *
1618 * This function is to be used by unbound workers and rescuers to bind
1619 * themselves to the target cpu and may race with cpu going down or
1620 * coming online.  kthread_bind() can't be used because it may put the
1621 * worker to already dead cpu and set_cpus_allowed_ptr() can't be used
1622 * verbatim as it's best effort and blocking and pool may be
1623 * [dis]associated in the meantime.
1624 *
1625 * This function tries set_cpus_allowed() and locks pool and verifies the
1626 * binding against %POOL_DISASSOCIATED which is set during
1627 * %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
1628 * enters idle state or fetches works without dropping lock, it can
1629 * guarantee the scheduling requirement described in the first paragraph.
1630 *
1631 * CONTEXT:
1632 * Might sleep.  Called without any lock but returns with pool->lock
1633 * held.
1634 *
1635 * Return:
1636 * %true if the associated pool is online (@worker is successfully
1637 * bound), %false if offline.
1638 */
1639static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
1640__acquires(&pool->lock)
1641{
1642        while (true) {
1643                /*
1644                 * The following call may fail, succeed or succeed
1645                 * without actually migrating the task to the cpu if
1646                 * it races with cpu hotunplug operation.  Verify
1647                 * against POOL_DISASSOCIATED.
1648                 */
1649                if (!(pool->flags & POOL_DISASSOCIATED))
1650                        set_cpus_allowed_ptr(current, pool->attrs->cpumask);
1651
1652                spin_lock_irq(&pool->lock);
1653                if (pool->flags & POOL_DISASSOCIATED)
1654                        return false;
1655                if (task_cpu(current) == pool->cpu &&
1656                    cpumask_equal(&current->cpus_allowed, pool->attrs->cpumask))
1657                        return true;
1658                spin_unlock_irq(&pool->lock);
1659
1660                /*
1661                 * We've raced with CPU hot[un]plug.  Give it a breather
1662                 * and retry migration.  cond_resched() is required here;
1663                 * otherwise, we might deadlock against cpu_stop trying to
1664                 * bring down the CPU on non-preemptive kernel.
1665                 */
1666                cpu_relax();
1667                cond_resched();
1668        }
1669}
1670
1671static struct worker *alloc_worker(void)
1672{
1673        struct worker *worker;
1674
1675        worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1676        if (worker) {
1677                INIT_LIST_HEAD(&worker->entry);
1678                INIT_LIST_HEAD(&worker->scheduled);
1679                /* on creation a worker is in !idle && prep state */
1680                worker->flags = WORKER_PREP;
1681        }
1682        return worker;
1683}
1684
1685/**
1686 * create_worker - create a new workqueue worker
1687 * @pool: pool the new worker will belong to
1688 *
1689 * Create a new worker which is bound to @pool.  The returned worker
1690 * can be started by calling start_worker() or destroyed using
1691 * destroy_worker().
1692 *
1693 * CONTEXT:
1694 * Might sleep.  Does GFP_KERNEL allocations.
1695 *
1696 * Return:
1697 * Pointer to the newly created worker.
1698 */
1699static struct worker *create_worker(struct worker_pool *pool)
1700{
1701        struct worker *worker = NULL;
1702        int id = -1;
1703        char id_buf[16];
1704
1705        lockdep_assert_held(&pool->manager_mutex);
1706
1707        /*
1708         * ID is needed to determine kthread name.  Allocate ID first
1709         * without installing the pointer.
1710         */
1711        idr_preload(GFP_KERNEL);
1712        spin_lock_irq(&pool->lock);
1713
1714        id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
1715
1716        spin_unlock_irq(&pool->lock);
1717        idr_preload_end();
1718        if (id < 0)
1719                goto fail;
1720
1721        worker = alloc_worker();
1722        if (!worker)
1723                goto fail;
1724
1725        worker->pool = pool;
1726        worker->id = id;
1727
1728        if (pool->cpu >= 0)
1729                snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1730                         pool->attrs->nice < 0  ? "H" : "");
1731        else
1732                snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1733
1734        worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1735                                              "kworker/%s", id_buf);
1736        if (IS_ERR(worker->task))
1737                goto fail;
1738
1739        /*
1740         * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1741         * online CPUs.  It'll be re-applied when any of the CPUs come up.
1742         */
1743        set_user_nice(worker->task, pool->attrs->nice);
1744        set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1745
1746        /* prevent userland from meddling with cpumask of workqueue workers */
1747        worker->task->flags |= PF_NO_SETAFFINITY;
1748
1749        /*
1750         * The caller is responsible for ensuring %POOL_DISASSOCIATED
1751         * remains stable across this function.  See the comments above the
1752         * flag definition for details.
1753         */
1754        if (pool->flags & POOL_DISASSOCIATED)
1755                worker->flags |= WORKER_UNBOUND;
1756
1757        /* successful, commit the pointer to idr */
1758        spin_lock_irq(&pool->lock);
1759        idr_replace(&pool->worker_idr, worker, worker->id);
1760        spin_unlock_irq(&pool->lock);
1761
1762        return worker;
1763
1764fail:
1765        if (id >= 0) {
1766                spin_lock_irq(&pool->lock);
1767                idr_remove(&pool->worker_idr, id);
1768                spin_unlock_irq(&pool->lock);
1769        }
1770        kfree(worker);
1771        return NULL;
1772}
1773
1774/**
1775 * start_worker - start a newly created worker
1776 * @worker: worker to start
1777 *
1778 * Make the pool aware of @worker and start it.
1779 *
1780 * CONTEXT:
1781 * spin_lock_irq(pool->lock).
1782 */
1783static void start_worker(struct worker *worker)
1784{
1785        worker->flags |= WORKER_STARTED;
1786        worker->pool->nr_workers++;
1787        worker_enter_idle(worker);
1788        wake_up_process(worker->task);
1789}
1790
1791/**
1792 * create_and_start_worker - create and start a worker for a pool
1793 * @pool: the target pool
1794 *
1795 * Grab the managership of @pool and create and start a new worker for it.
1796 *
1797 * Return: 0 on success. A negative error code otherwise.
1798 */
1799static int create_and_start_worker(struct worker_pool *pool)
1800{
1801        struct worker *worker;
1802
1803        mutex_lock(&pool->manager_mutex);
1804
1805        worker = create_worker(pool);
1806        if (worker) {
1807                spin_lock_irq(&pool->lock);
1808                start_worker(worker);
1809                spin_unlock_irq(&pool->lock);
1810        }
1811
1812        mutex_unlock(&pool->manager_mutex);
1813
1814        return worker ? 0 : -ENOMEM;
1815}
1816
1817/**
1818 * destroy_worker - destroy a workqueue worker
1819 * @worker: worker to be destroyed
1820 *
1821 * Destroy @worker and adjust @pool stats accordingly.
1822 *
1823 * CONTEXT:
1824 * spin_lock_irq(pool->lock) which is released and regrabbed.
1825 */
1826static void destroy_worker(struct worker *worker)
1827{
1828        struct worker_pool *pool = worker->pool;
1829
1830        lockdep_assert_held(&pool->manager_mutex);
1831        lockdep_assert_held(&pool->lock);
1832
1833        /* sanity check frenzy */
1834        if (WARN_ON(worker->current_work) ||
1835            WARN_ON(!list_empty(&worker->scheduled)))
1836                return;
1837
1838        if (worker->flags & WORKER_STARTED)
1839                pool->nr_workers--;
1840        if (worker->flags & WORKER_IDLE)
1841                pool->nr_idle--;
1842
1843        list_del_init(&worker->entry);
1844        worker->flags |= WORKER_DIE;
1845
1846        idr_remove(&pool->worker_idr, worker->id);
1847
1848        spin_unlock_irq(&pool->lock);
1849
1850        kthread_stop(worker->task);
1851        kfree(worker);
1852
1853        spin_lock_irq(&pool->lock);
1854}
1855
1856static void idle_worker_timeout(unsigned long __pool)
1857{
1858        struct worker_pool *pool = (void *)__pool;
1859
1860        spin_lock_irq(&pool->lock);
1861
1862        if (too_many_workers(pool)) {
1863                struct worker *worker;
1864                unsigned long expires;
1865
1866                /* idle_list is kept in LIFO order, check the last one */
1867                worker = list_entry(pool->idle_list.prev, struct worker, entry);
1868                expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1869
1870                if (time_before(jiffies, expires))
1871                        mod_timer(&pool->idle_timer, expires);
1872                else {
1873                        /* it's been idle for too long, wake up manager */
1874                        pool->flags |= POOL_MANAGE_WORKERS;
1875                        wake_up_worker(pool);
1876                }
1877        }
1878
1879        spin_unlock_irq(&pool->lock);
1880}
1881
1882static void send_mayday(struct work_struct *work)
1883{
1884        struct pool_workqueue *pwq = get_work_pwq(work);
1885        struct workqueue_struct *wq = pwq->wq;
1886
1887        lockdep_assert_held(&wq_mayday_lock);
1888
1889        if (!wq->rescuer)
1890                return;
1891
1892        /* mayday mayday mayday */
1893        if (list_empty(&pwq->mayday_node)) {
1894                list_add_tail(&pwq->mayday_node, &wq->maydays);
1895                wake_up_process(wq->rescuer->task);
1896        }
1897}
1898
1899static void pool_mayday_timeout(unsigned long __pool)
1900{
1901        struct worker_pool *pool = (void *)__pool;
1902        struct work_struct *work;
1903
1904        spin_lock_irq(&wq_mayday_lock);         /* for wq->maydays */
1905        spin_lock(&pool->lock);
1906
1907        if (need_to_create_worker(pool)) {
1908                /*
1909                 * We've been trying to create a new worker but
1910                 * haven't been successful.  We might be hitting an
1911                 * allocation deadlock.  Send distress signals to
1912                 * rescuers.
1913                 */
1914                list_for_each_entry(work, &pool->worklist, entry)
1915                        send_mayday(work);
1916        }
1917
1918        spin_unlock(&pool->lock);
1919        spin_unlock_irq(&wq_mayday_lock);
1920
1921        mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1922}
1923
1924/**
1925 * maybe_create_worker - create a new worker if necessary
1926 * @pool: pool to create a new worker for
1927 *
1928 * Create a new worker for @pool if necessary.  @pool is guaranteed to
1929 * have at least one idle worker on return from this function.  If
1930 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1931 * sent to all rescuers with works scheduled on @pool to resolve
1932 * possible allocation deadlock.
1933 *
1934 * On return, need_to_create_worker() is guaranteed to be %false and
1935 * may_start_working() %true.
1936 *
1937 * LOCKING:
1938 * spin_lock_irq(pool->lock) which may be released and regrabbed
1939 * multiple times.  Does GFP_KERNEL allocations.  Called only from
1940 * manager.
1941 *
1942 * Return:
1943 * %false if no action was taken and pool->lock stayed locked, %true
1944 * otherwise.
1945 */
1946static bool maybe_create_worker(struct worker_pool *pool)
1947__releases(&pool->lock)
1948__acquires(&pool->lock)
1949{
1950        if (!need_to_create_worker(pool))
1951                return false;
1952restart:
1953        spin_unlock_irq(&pool->lock);
1954
1955        /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1956        mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1957
1958        while (true) {
1959                struct worker *worker;
1960
1961                worker = create_worker(pool);
1962                if (worker) {
1963                        del_timer_sync(&pool->mayday_timer);
1964                        spin_lock_irq(&pool->lock);
1965                        start_worker(worker);
1966                        if (WARN_ON_ONCE(need_to_create_worker(pool)))
1967                                goto restart;
1968                        return true;
1969                }
1970
1971                if (!need_to_create_worker(pool))
1972                        break;
1973
1974                __set_current_state(TASK_INTERRUPTIBLE);
1975                schedule_timeout(CREATE_COOLDOWN);
1976
1977                if (!need_to_create_worker(pool))
1978                        break;
1979        }
1980
1981        del_timer_sync(&pool->mayday_timer);
1982        spin_lock_irq(&pool->lock);
1983        if (need_to_create_worker(pool))
1984                goto restart;
1985        return true;
1986}
1987
1988/**
1989 * maybe_destroy_worker - destroy workers which have been idle for a while
1990 * @pool: pool to destroy workers for
1991 *
1992 * Destroy @pool workers which have been idle for longer than
1993 * IDLE_WORKER_TIMEOUT.
1994 *
1995 * LOCKING:
1996 * spin_lock_irq(pool->lock) which may be released and regrabbed
1997 * multiple times.  Called only from manager.
1998 *
1999 * Return:
2000 * %false if no action was taken and pool->lock stayed locked, %true
2001 * otherwise.
2002 */
2003static bool maybe_destroy_workers(struct worker_pool *pool)
2004{
2005        bool ret = false;
2006
2007        while (too_many_workers(pool)) {
2008                struct worker *worker;
2009                unsigned long expires;
2010
2011                worker = list_entry(pool->idle_list.prev, struct worker, entry);
2012                expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2013
2014                if (time_before(jiffies, expires)) {
2015                        mod_timer(&pool->idle_timer, expires);
2016                        break;
2017                }
2018
2019                destroy_worker(worker);
2020                ret = true;
2021        }
2022
2023        return ret;
2024}
2025
2026/**
2027 * manage_workers - manage worker pool
2028 * @worker: self
2029 *
2030 * Assume the manager role and manage the worker pool @worker belongs
2031 * to.  At any given time, there can be only zero or one manager per
2032 * pool.  The exclusion is handled automatically by this function.
2033 *
2034 * The caller can safely start processing works on false return.  On
2035 * true return, it's guaranteed that need_to_create_worker() is false
2036 * and may_start_working() is true.
2037 *
2038 * CONTEXT:
2039 * spin_lock_irq(pool->lock) which may be released and regrabbed
2040 * multiple times.  Does GFP_KERNEL allocations.
2041 *
2042 * Return:
2043 * %false if the pool don't need management and the caller can safely start
2044 * processing works, %true indicates that the function released pool->lock
2045 * and reacquired it to perform some management function and that the
2046 * conditions that the caller verified while holding the lock before
2047 * calling the function might no longer be true.
2048 */
2049static bool manage_workers(struct worker *worker)
2050{
2051        struct worker_pool *pool = worker->pool;
2052        bool ret = false;
2053
2054        /*
2055         * Managership is governed by two mutexes - manager_arb and
2056         * manager_mutex.  manager_arb handles arbitration of manager role.
2057         * Anyone who successfully grabs manager_arb wins the arbitration
2058         * and becomes the manager.  mutex_trylock() on pool->manager_arb
2059         * failure while holding pool->lock reliably indicates that someone
2060         * else is managing the pool and the worker which failed trylock
2061         * can proceed to executing work items.  This means that anyone
2062         * grabbing manager_arb is responsible for actually performing
2063         * manager duties.  If manager_arb is grabbed and released without
2064         * actual management, the pool may stall indefinitely.
2065         *
2066         * manager_mutex is used for exclusion of actual management
2067         * operations.  The holder of manager_mutex can be sure that none
2068         * of management operations, including creation and destruction of
2069         * workers, won't take place until the mutex is released.  Because
2070         * manager_mutex doesn't interfere with manager role arbitration,
2071         * it is guaranteed that the pool's management, while may be
2072         * delayed, won't be disturbed by someone else grabbing
2073         * manager_mutex.
2074         */
2075        if (!mutex_trylock(&pool->manager_arb))
2076                return ret;
2077
2078        /*
2079         * With manager arbitration won, manager_mutex would be free in
2080         * most cases.  trylock first without dropping @pool->lock.
2081         */
2082        if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
2083                spin_unlock_irq(&pool->lock);
2084                mutex_lock(&pool->manager_mutex);
2085                spin_lock_irq(&pool->lock);
2086                ret = true;
2087        }
2088
2089        pool->flags &= ~POOL_MANAGE_WORKERS;
2090
2091        /*
2092         * Destroy and then create so that may_start_working() is true
2093         * on return.
2094         */
2095        ret |= maybe_destroy_workers(pool);
2096        ret |= maybe_create_worker(pool);
2097
2098        mutex_unlock(&pool->manager_mutex);
2099        mutex_unlock(&pool->manager_arb);
2100        return ret;
2101}
2102
2103/**
2104 * process_one_work - process single work
2105 * @worker: self
2106 * @work: work to process
2107 *
2108 * Process @work.  This function contains all the logics necessary to
2109 * process a single work including synchronization against and
2110 * interaction with other workers on the same cpu, queueing and
2111 * flushing.  As long as context requirement is met, any worker can
2112 * call this function to process a work.
2113 *
2114 * CONTEXT:
2115 * spin_lock_irq(pool->lock) which is released and regrabbed.
2116 */
2117static void process_one_work(struct worker *worker, struct work_struct *work)
2118__releases(&pool->lock)
2119__acquires(&pool->lock)
2120{
2121        struct pool_workqueue *pwq = get_work_pwq(work);
2122        struct worker_pool *pool = worker->pool;
2123        bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2124        int work_color;
2125        struct worker *collision;
2126#ifdef CONFIG_LOCKDEP
2127        /*
2128         * It is permissible to free the struct work_struct from
2129         * inside the function that is called from it, this we need to
2130         * take into account for lockdep too.  To avoid bogus "held
2131         * lock freed" warnings as well as problems when looking into
2132         * work->lockdep_map, make a copy and use that here.
2133         */
2134        struct lockdep_map lockdep_map;
2135
2136        lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2137#endif
2138        /*
2139         * Ensure we're on the correct CPU.  DISASSOCIATED test is
2140         * necessary to avoid spurious warnings from rescuers servicing the
2141         * unbound or a disassociated pool.
2142         */
2143        WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
2144                     !(pool->flags & POOL_DISASSOCIATED) &&
2145                     raw_smp_processor_id() != pool->cpu);
2146
2147        /*
2148         * A single work shouldn't be executed concurrently by
2149         * multiple workers on a single cpu.  Check whether anyone is
2150         * already processing the work.  If so, defer the work to the
2151         * currently executing one.
2152         */
2153        collision = find_worker_executing_work(pool, work);
2154        if (unlikely(collision)) {
2155                move_linked_works(work, &collision->scheduled, NULL);
2156                return;
2157        }
2158
2159        /* claim and dequeue */
2160        debug_work_deactivate(work);
2161        hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2162        worker->current_work = work;
2163        worker->current_func = work->func;
2164        worker->current_pwq = pwq;
2165        work_color = get_work_color(work);
2166
2167        list_del_init(&work->entry);
2168
2169        /*
2170         * CPU intensive works don't participate in concurrency
2171         * management.  They're the scheduler's responsibility.
2172         */
2173        if (unlikely(cpu_intensive))
2174                worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
2175
2176        /*
2177         * Unbound pool isn't concurrency managed and work items should be
2178         * executed ASAP.  Wake up another worker if necessary.
2179         */
2180        if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2181                wake_up_worker(pool);
2182
2183        /*
2184         * Record the last pool and clear PENDING which should be the last
2185         * update to @work.  Also, do this inside @pool->lock so that
2186         * PENDING and queued state changes happen together while IRQ is
2187         * disabled.
2188         */
2189        set_work_pool_and_clear_pending(work, pool->id);
2190
2191        spin_unlock_irq(&pool->lock);
2192
2193        lock_map_acquire_read(&pwq->wq->lockdep_map);
2194        lock_map_acquire(&lockdep_map);
2195        trace_workqueue_execute_start(work);
2196        worker->current_func(work);
2197        /*
2198         * While we must be careful to not use "work" after this, the trace
2199         * point will only record its address.
2200         */
2201        trace_workqueue_execute_end(work);
2202        lock_map_release(&lockdep_map);
2203        lock_map_release(&pwq->wq->lockdep_map);
2204
2205        if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
2206                pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2207                       "     last function: %pf\n",
2208                       current->comm, preempt_count(), task_pid_nr(current),
2209                       worker->current_func);
2210                debug_show_held_locks(current);
2211                dump_stack();
2212        }
2213
2214        /*
2215         * The following prevents a kworker from hogging CPU on !PREEMPT
2216         * kernels, where a requeueing work item waiting for something to
2217         * happen could deadlock with stop_machine as such work item could
2218         * indefinitely requeue itself while all other CPUs are trapped in
2219         * stop_machine.
2220         */
2221        cond_resched();
2222
2223        spin_lock_irq(&pool->lock);
2224
2225        /* clear cpu intensive status */
2226        if (unlikely(cpu_intensive))
2227                worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2228
2229        /* we're done with it, release */
2230        hash_del(&worker->hentry);
2231        worker->current_work = NULL;
2232        worker->current_func = NULL;
2233        worker->current_pwq = NULL;
2234        worker->desc_valid = false;
2235        pwq_dec_nr_in_flight(pwq, work_color);
2236}
2237
2238/**
2239 * process_scheduled_works - process scheduled works
2240 * @worker: self
2241 *
2242 * Process all scheduled works.  Please note that the scheduled list
2243 * may change while processing a work, so this function repeatedly
2244 * fetches a work from the top and executes it.
2245 *
2246 * CONTEXT:
2247 * spin_lock_irq(pool->lock) which may be released and regrabbed
2248 * multiple times.
2249 */
2250static void process_scheduled_works(struct worker *worker)
2251{
2252        while (!list_empty(&worker->scheduled)) {
2253                struct work_struct *work = list_first_entry(&worker->scheduled,
2254                                                struct work_struct, entry);
2255                process_one_work(worker, work);
2256        }
2257}
2258
2259/**
2260 * worker_thread - the worker thread function
2261 * @__worker: self
2262 *
2263 * The worker thread function.  All workers belong to a worker_pool -
2264 * either a per-cpu one or dynamic unbound one.  These workers process all
2265 * work items regardless of their specific target workqueue.  The only
2266 * exception is work items which belong to workqueues with a rescuer which
2267 * will be explained in rescuer_thread().
2268 *
2269 * Return: 0
2270 */
2271static int worker_thread(void *__worker)
2272{
2273        struct worker *worker = __worker;
2274        struct worker_pool *pool = worker->pool;
2275
2276        /* tell the scheduler that this is a workqueue worker */
2277        worker->task->flags |= PF_WQ_WORKER;
2278woke_up:
2279        spin_lock_irq(&pool->lock);
2280
2281        /* am I supposed to die? */
2282        if (unlikely(worker->flags & WORKER_DIE)) {
2283                spin_unlock_irq(&pool->lock);
2284                WARN_ON_ONCE(!list_empty(&worker->entry));
2285                worker->task->flags &= ~PF_WQ_WORKER;
2286                return 0;
2287        }
2288
2289        worker_leave_idle(worker);
2290recheck:
2291        /* no more worker necessary? */
2292        if (!need_more_worker(pool))
2293                goto sleep;
2294
2295        /* do we need to manage? */
2296        if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2297                goto recheck;
2298
2299        /*
2300         * ->scheduled list can only be filled while a worker is
2301         * preparing to process a work or actually processing it.
2302         * Make sure nobody diddled with it while I was sleeping.
2303         */
2304        WARN_ON_ONCE(!list_empty(&worker->scheduled));
2305
2306        /*
2307         * Finish PREP stage.  We're guaranteed to have at least one idle
2308         * worker or that someone else has already assumed the manager
2309         * role.  This is where @worker starts participating in concurrency
2310         * management if applicable and concurrency management is restored
2311         * after being rebound.  See rebind_workers() for details.
2312         */
2313        worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
2314
2315        do {
2316                struct work_struct *work =
2317                        list_first_entry(&pool->worklist,
2318                                         struct work_struct, entry);
2319
2320                if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2321                        /* optimization path, not strictly necessary */
2322                        process_one_work(worker, work);
2323                        if (unlikely(!list_empty(&worker->scheduled)))
2324                                process_scheduled_works(worker);
2325                } else {
2326                        move_linked_works(work, &worker->scheduled, NULL);
2327                        process_scheduled_works(worker);
2328                }
2329        } while (keep_working(pool));
2330
2331        worker_set_flags(worker, WORKER_PREP, false);
2332sleep:
2333        if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
2334                goto recheck;
2335
2336        /*
2337         * pool->lock is held and there's no work to process and no need to
2338         * manage, sleep.  Workers are woken up only while holding
2339         * pool->lock or from local cpu, so setting the current state
2340         * before releasing pool->lock is enough to prevent losing any
2341         * event.
2342         */
2343        worker_enter_idle(worker);
2344        __set_current_state(TASK_INTERRUPTIBLE);
2345        spin_unlock_irq(&pool->lock);
2346        schedule();
2347        goto woke_up;
2348}
2349
2350/**
2351 * rescuer_thread - the rescuer thread function
2352 * @__rescuer: self
2353 *
2354 * Workqueue rescuer thread function.  There's one rescuer for each
2355 * workqueue which has WQ_MEM_RECLAIM set.
2356 *
2357 * Regular work processing on a pool may block trying to create a new
2358 * worker which uses GFP_KERNEL allocation which has slight chance of
2359 * developing into deadlock if some works currently on the same queue
2360 * need to be processed to satisfy the GFP_KERNEL allocation.  This is
2361 * the problem rescuer solves.
2362 *
2363 * When such condition is possible, the pool summons rescuers of all
2364 * workqueues which have works queued on the pool and let them process
2365 * those works so that forward progress can be guaranteed.
2366 *
2367 * This should happen rarely.
2368 *
2369 * Return: 0
2370 */
2371static int rescuer_thread(void *__rescuer)
2372{
2373        struct worker *rescuer = __rescuer;
2374        struct workqueue_struct *wq = rescuer->rescue_wq;
2375        struct list_head *scheduled = &rescuer->scheduled;
2376
2377        set_user_nice(current, RESCUER_NICE_LEVEL);
2378
2379        /*
2380         * Mark rescuer as worker too.  As WORKER_PREP is never cleared, it
2381         * doesn't participate in concurrency management.
2382         */
2383        rescuer->task->flags |= PF_WQ_WORKER;
2384repeat:
2385        set_current_state(TASK_INTERRUPTIBLE);
2386
2387        if (kthread_should_stop()) {
2388                __set_current_state(TASK_RUNNING);
2389                rescuer->task->flags &= ~PF_WQ_WORKER;
2390                return 0;
2391        }
2392
2393        /* see whether any pwq is asking for help */
2394        spin_lock_irq(&wq_mayday_lock);
2395
2396        while (!list_empty(&wq->maydays)) {
2397                struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2398                                        struct pool_workqueue, mayday_node);
2399                struct worker_pool *pool = pwq->pool;
2400                struct work_struct *work, *n;
2401
2402                __set_current_state(TASK_RUNNING);
2403                list_del_init(&pwq->mayday_node);
2404
2405                spin_unlock_irq(&wq_mayday_lock);
2406
2407                /* migrate to the target cpu if possible */
2408                worker_maybe_bind_and_lock(pool);
2409                rescuer->pool = pool;
2410
2411                /*
2412                 * Slurp in all works issued via this workqueue and
2413                 * process'em.
2414                 */
2415                WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
2416                list_for_each_entry_safe(work, n, &pool->worklist, entry)
2417                        if (get_work_pwq(work) == pwq)
2418                                move_linked_works(work, scheduled, &n);
2419
2420                process_scheduled_works(rescuer);
2421
2422                /*
2423                 * Leave this pool.  If keep_working() is %true, notify a
2424                 * regular worker; otherwise, we end up with 0 concurrency
2425                 * and stalling the execution.
2426                 */
2427                if (keep_working(pool))
2428                        wake_up_worker(pool);
2429
2430                rescuer->pool = NULL;
2431                spin_unlock(&pool->lock);
2432                spin_lock(&wq_mayday_lock);
2433        }
2434
2435        spin_unlock_irq(&wq_mayday_lock);
2436
2437        /* rescuers should never participate in concurrency management */
2438        WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2439        schedule();
2440        goto repeat;
2441}
2442
2443struct wq_barrier {
2444        struct work_struct      work;
2445        struct completion       done;
2446};
2447
2448static void wq_barrier_func(struct work_struct *work)
2449{
2450        struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2451        complete(&barr->done);
2452}
2453
2454/**
2455 * insert_wq_barrier - insert a barrier work
2456 * @pwq: pwq to insert barrier into
2457 * @barr: wq_barrier to insert
2458 * @target: target work to attach @barr to
2459 * @worker: worker currently executing @target, NULL if @target is not executing
2460 *
2461 * @barr is linked to @target such that @barr is completed only after
2462 * @target finishes execution.  Please note that the ordering
2463 * guarantee is observed only with respect to @target and on the local
2464 * cpu.
2465 *
2466 * Currently, a queued barrier can't be canceled.  This is because
2467 * try_to_grab_pending() can't determine whether the work to be
2468 * grabbed is at the head of the queue and thus can't clear LINKED
2469 * flag of the previous work while there must be a valid next work
2470 * after a work with LINKED flag set.
2471 *
2472 * Note that when @worker is non-NULL, @target may be modified
2473 * underneath us, so we can't reliably determine pwq from @target.
2474 *
2475 * CONTEXT:
2476 * spin_lock_irq(pool->lock).
2477 */
2478static void insert_wq_barrier(struct pool_workqueue *pwq,
2479                              struct wq_barrier *barr,
2480                              struct work_struct *target, struct worker *worker)
2481{
2482        struct list_head *head;
2483        unsigned int linked = 0;
2484
2485        /*
2486         * debugobject calls are safe here even with pool->lock locked
2487         * as we know for sure that this will not trigger any of the
2488         * checks and call back into the fixup functions where we
2489         * might deadlock.
2490         */
2491        INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2492        __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2493        init_completion(&barr->done);
2494
2495        /*
2496         * If @target is currently being executed, schedule the
2497         * barrier to the worker; otherwise, put it after @target.
2498         */
2499        if (worker)
2500                head = worker->scheduled.next;
2501        else {
2502                unsigned long *bits = work_data_bits(target);
2503
2504                head = target->entry.next;
2505                /* there can already be other linked works, inherit and set */
2506                linked = *bits & WORK_STRUCT_LINKED;
2507                __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2508        }
2509
2510        debug_work_activate(&barr->work);
2511        insert_work(pwq, &barr->work, head,
2512                    work_color_to_flags(WORK_NO_COLOR) | linked);
2513}
2514
2515/**
2516 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2517 * @wq: workqueue being flushed
2518 * @flush_color: new flush color, < 0 for no-op
2519 * @work_color: new work color, < 0 for no-op
2520 *
2521 * Prepare pwqs for workqueue flushing.
2522 *
2523 * If @flush_color is non-negative, flush_color on all pwqs should be
2524 * -1.  If no pwq has in-flight commands at the specified color, all
2525 * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
2526 * has in flight commands, its pwq->flush_color is set to
2527 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2528 * wakeup logic is armed and %true is returned.
2529 *
2530 * The caller should have initialized @wq->first_flusher prior to
2531 * calling this function with non-negative @flush_color.  If
2532 * @flush_color is negative, no flush color update is done and %false
2533 * is returned.
2534 *
2535 * If @work_color is non-negative, all pwqs should have the same
2536 * work_color which is previous to @work_color and all will be
2537 * advanced to @work_color.
2538 *
2539 * CONTEXT:
2540 * mutex_lock(wq->mutex).
2541 *
2542 * Return:
2543 * %true if @flush_color >= 0 and there's something to flush.  %false
2544 * otherwise.
2545 */
2546static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2547                                      int flush_color, int work_color)
2548{
2549        bool wait = false;
2550        struct pool_workqueue *pwq;
2551
2552        if (flush_color >= 0) {
2553                WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2554                atomic_set(&wq->nr_pwqs_to_flush, 1);
2555        }
2556
2557        for_each_pwq(pwq, wq) {
2558                struct worker_pool *pool = pwq->pool;
2559
2560                spin_lock_irq(&pool->lock);
2561
2562                if (flush_color >= 0) {
2563                        WARN_ON_ONCE(pwq->flush_color != -1);
2564
2565                        if (pwq->nr_in_flight[flush_color]) {
2566                                pwq->flush_color = flush_color;
2567                                atomic_inc(&wq->nr_pwqs_to_flush);
2568                                wait = true;
2569                        }
2570                }
2571
2572                if (work_color >= 0) {
2573                        WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2574                        pwq->work_color = work_color;
2575                }
2576
2577                spin_unlock_irq(&pool->lock);
2578        }
2579
2580        if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2581                complete(&wq->first_flusher->done);
2582
2583        return wait;
2584}
2585
2586/**
2587 * flush_workqueue - ensure that any scheduled work has run to completion.
2588 * @wq: workqueue to flush
2589 *
2590 * This function sleeps until all work items which were queued on entry
2591 * have finished execution, but it is not livelocked by new incoming ones.
2592 */
2593void flush_workqueue(struct workqueue_struct *wq)
2594{
2595        struct wq_flusher this_flusher = {
2596                .list = LIST_HEAD_INIT(this_flusher.list),
2597                .flush_color = -1,
2598                .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2599        };
2600        int next_color;
2601
2602        lock_map_acquire(&wq->lockdep_map);
2603        lock_map_release(&wq->lockdep_map);
2604
2605        mutex_lock(&wq->mutex);
2606
2607        /*
2608         * Start-to-wait phase
2609         */
2610        next_color = work_next_color(wq->work_color);
2611
2612        if (next_color != wq->flush_color) {
2613                /*
2614                 * Color space is not full.  The current work_color
2615                 * becomes our flush_color and work_color is advanced
2616                 * by one.
2617                 */
2618                WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2619                this_flusher.flush_color = wq->work_color;
2620                wq->work_color = next_color;
2621
2622                if (!wq->first_flusher) {
2623                        /* no flush in progress, become the first flusher */
2624                        WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2625
2626                        wq->first_flusher = &this_flusher;
2627
2628                        if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2629                                                       wq->work_color)) {
2630                                /* nothing to flush, done */
2631                                wq->flush_color = next_color;
2632                                wq->first_flusher = NULL;
2633                                goto out_unlock;
2634                        }
2635                } else {
2636                        /* wait in queue */
2637                        WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2638                        list_add_tail(&this_flusher.list, &wq->flusher_queue);
2639                        flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2640                }
2641        } else {
2642                /*
2643                 * Oops, color space is full, wait on overflow queue.
2644                 * The next flush completion will assign us
2645                 * flush_color and transfer to flusher_queue.
2646                 */
2647                list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2648        }
2649
2650        mutex_unlock(&wq->mutex);
2651
2652        wait_for_completion(&this_flusher.done);
2653
2654        /*
2655         * Wake-up-and-cascade phase
2656         *
2657         * First flushers are responsible for cascading flushes and
2658         * handling overflow.  Non-first flushers can simply return.
2659         */
2660        if (wq->first_flusher != &this_flusher)
2661                return;
2662
2663        mutex_lock(&wq->mutex);
2664
2665        /* we might have raced, check again with mutex held */
2666        if (wq->first_flusher != &this_flusher)
2667                goto out_unlock;
2668
2669        wq->first_flusher = NULL;
2670
2671        WARN_ON_ONCE(!list_empty(&this_flusher.list));
2672        WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2673
2674        while (true) {
2675                struct wq_flusher *next, *tmp;
2676
2677                /* complete all the flushers sharing the current flush color */
2678                list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2679                        if (next->flush_color != wq->flush_color)
2680                                break;
2681                        list_del_init(&next->list);
2682                        complete(&next->done);
2683                }
2684
2685                WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2686                             wq->flush_color != work_next_color(wq->work_color));
2687
2688                /* this flush_color is finished, advance by one */
2689                wq->flush_color = work_next_color(wq->flush_color);
2690
2691                /* one color has been freed, handle overflow queue */
2692                if (!list_empty(&wq->flusher_overflow)) {
2693                        /*
2694                         * Assign the same color to all overflowed
2695                         * flushers, advance work_color and append to
2696                         * flusher_queue.  This is the start-to-wait
2697                         * phase for these overflowed flushers.
2698                         */
2699                        list_for_each_entry(tmp, &wq->flusher_overflow, list)
2700                                tmp->flush_color = wq->work_color;
2701
2702                        wq->work_color = work_next_color(wq->work_color);
2703
2704                        list_splice_tail_init(&wq->flusher_overflow,
2705                                              &wq->flusher_queue);
2706                        flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2707                }
2708
2709                if (list_empty(&wq->flusher_queue)) {
2710                        WARN_ON_ONCE(wq->flush_color != wq->work_color);
2711                        break;
2712                }
2713
2714                /*
2715                 * Need to flush more colors.  Make the next flusher
2716                 * the new first flusher and arm pwqs.
2717                 */
2718                WARN_ON_ONCE(wq->flush_color == wq->work_color);
2719                WARN_ON_ONCE(wq->flush_color != next->flush_color);
2720
2721                list_del_init(&next->list);
2722                wq->first_flusher = next;
2723
2724                if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2725                        break;
2726
2727                /*
2728                 * Meh... this color is already done, clear first
2729                 * flusher and repeat cascading.
2730                 */
2731                wq->first_flusher = NULL;
2732        }
2733
2734out_unlock:
2735        mutex_unlock(&wq->mutex);
2736}
2737EXPORT_SYMBOL_GPL(flush_workqueue);
2738
2739/**
2740 * drain_workqueue - drain a workqueue
2741 * @wq: workqueue to drain
2742 *
2743 * Wait until the workqueue becomes empty.  While draining is in progress,
2744 * only chain queueing is allowed.  IOW, only currently pending or running
2745 * work items on @wq can queue further work items on it.  @wq is flushed
2746 * repeatedly until it becomes empty.  The number of flushing is detemined
2747 * by the depth of chaining and should be relatively short.  Whine if it
2748 * takes too long.
2749 */
2750void drain_workqueue(struct workqueue_struct *wq)
2751{
2752        unsigned int flush_cnt = 0;
2753        struct pool_workqueue *pwq;
2754
2755        /*
2756         * __queue_work() needs to test whether there are drainers, is much
2757         * hotter than drain_workqueue() and already looks at @wq->flags.
2758         * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
2759         */
2760        mutex_lock(&wq->mutex);
2761        if (!wq->nr_drainers++)
2762                wq->flags |= __WQ_DRAINING;
2763        mutex_unlock(&wq->mutex);
2764reflush:
2765        flush_workqueue(wq);
2766
2767        mutex_lock(&wq->mutex);
2768
2769        for_each_pwq(pwq, wq) {
2770                bool drained;
2771
2772                spin_lock_irq(&pwq->pool->lock);
2773                drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2774                spin_unlock_irq(&pwq->pool->lock);
2775
2776                if (drained)
2777                        continue;
2778
2779                if (++flush_cnt == 10 ||
2780                    (flush_cnt % 100 == 0 && flush_cnt <= 1000))
2781                        pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
2782                                wq->name, flush_cnt);
2783
2784                mutex_unlock(&wq->mutex);
2785                goto reflush;
2786        }
2787
2788        if (!--wq->nr_drainers)
2789                wq->flags &= ~__WQ_DRAINING;
2790        mutex_unlock(&wq->mutex);
2791}
2792EXPORT_SYMBOL_GPL(drain_workqueue);
2793
2794static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
2795{
2796        struct worker *worker = NULL;
2797        struct worker_pool *pool;
2798        struct pool_workqueue *pwq;
2799
2800        might_sleep();
2801
2802        local_irq_disable();
2803        pool = get_work_pool(work);
2804        if (!pool) {
2805                local_irq_enable();
2806                return false;
2807        }
2808
2809        spin_lock(&pool->lock);
2810        /* see the comment in try_to_grab_pending() with the same code */
2811        pwq = get_work_pwq(work);
2812        if (pwq) {
2813                if (unlikely(pwq->pool != pool))
2814                        goto already_gone;
2815        } else {
2816                worker = find_worker_executing_work(pool, work);
2817                if (!worker)
2818                        goto already_gone;
2819                pwq = worker->current_pwq;
2820        }
2821
2822        insert_wq_barrier(pwq, barr, work, worker);
2823        spin_unlock_irq(&pool->lock);
2824
2825        /*
2826         * If @max_active is 1 or rescuer is in use, flushing another work
2827         * item on the same workqueue may lead to deadlock.  Make sure the
2828         * flusher is not running on the same workqueue by verifying write
2829         * access.
2830         */
2831        if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
2832                lock_map_acquire(&pwq->wq->lockdep_map);
2833        else
2834                lock_map_acquire_read(&pwq->wq->lockdep_map);
2835        lock_map_release(&pwq->wq->lockdep_map);
2836
2837        return true;
2838already_gone:
2839        spin_unlock_irq(&pool->lock);
2840        return false;
2841}
2842
2843static bool __flush_work(struct work_struct *work)
2844{
2845        struct wq_barrier barr;
2846
2847        if (start_flush_work(work, &barr)) {
2848                wait_for_completion(&barr.done);
2849                destroy_work_on_stack(&barr.work);
2850                return true;
2851        } else {
2852                return false;
2853        }
2854}
2855
2856/**
2857 * flush_work - wait for a work to finish executing the last queueing instance
2858 * @work: the work to flush
2859 *
2860 * Wait until @work has finished execution.  @work is guaranteed to be idle
2861 * on return if it hasn't been requeued since flush started.
2862 *
2863 * Return:
2864 * %true if flush_work() waited for the work to finish execution,
2865 * %false if it was already idle.
2866 */
2867bool flush_work(struct work_struct *work)
2868{
2869        lock_map_acquire(&work->lockdep_map);
2870        lock_map_release(&work->lockdep_map);
2871
2872        return __flush_work(work);
2873}
2874EXPORT_SYMBOL_GPL(flush_work);
2875
2876static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
2877{
2878        unsigned long flags;
2879        int ret;
2880
2881        do {
2882                ret = try_to_grab_pending(work, is_dwork, &flags);
2883                /*
2884                 * If someone else is canceling, wait for the same event it
2885                 * would be waiting for before retrying.
2886                 */
2887                if (unlikely(ret == -ENOENT))
2888                        flush_work(work);
2889        } while (unlikely(ret < 0));
2890
2891        /* tell other tasks trying to grab @work to back off */
2892        mark_work_canceling(work);
2893        local_irq_restore(flags);
2894
2895        flush_work(work);
2896        clear_work_data(work);
2897        return ret;
2898}
2899
2900/**
2901 * cancel_work_sync - cancel a work and wait for it to finish
2902 * @work: the work to cancel
2903 *
2904 * Cancel @work and wait for its execution to finish.  This function
2905 * can be used even if the work re-queues itself or migrates to
2906 * another workqueue.  On return from this function, @work is
2907 * guaranteed to be not pending or executing on any CPU.
2908 *
2909 * cancel_work_sync(&delayed_work->work) must not be used for
2910 * delayed_work's.  Use cancel_delayed_work_sync() instead.
2911 *
2912 * The caller must ensure that the workqueue on which @work was last
2913 * queued can't be destroyed before this function returns.
2914 *
2915 * Return:
2916 * %true if @work was pending, %false otherwise.
2917 */
2918bool cancel_work_sync(struct work_struct *work)
2919{
2920        return __cancel_work_timer(work, false);
2921}
2922EXPORT_SYMBOL_GPL(cancel_work_sync);
2923
2924/**
2925 * flush_delayed_work - wait for a dwork to finish executing the last queueing
2926 * @dwork: the delayed work to flush
2927 *
2928 * Delayed timer is cancelled and the pending work is queued for
2929 * immediate execution.  Like flush_work(), this function only
2930 * considers the last queueing instance of @dwork.
2931 *
2932 * Return:
2933 * %true if flush_work() waited for the work to finish execution,
2934 * %false if it was already idle.
2935 */
2936bool flush_delayed_work(struct delayed_work *dwork)
2937{
2938        local_irq_disable();
2939        if (del_timer_sync(&dwork->timer))
2940                __queue_work(dwork->cpu, dwork->wq, &dwork->work);
2941        local_irq_enable();
2942        return flush_work(&dwork->work);
2943}
2944EXPORT_SYMBOL(flush_delayed_work);
2945
2946/**
2947 * cancel_delayed_work - cancel a delayed work
2948 * @dwork: delayed_work to cancel
2949 *
2950 * Kill off a pending delayed_work.
2951 *
2952 * Return: %true if @dwork was pending and canceled; %false if it wasn't
2953 * pending.
2954 *
2955 * Note:
2956 * The work callback function may still be running on return, unless
2957 * it returns %true and the work doesn't re-arm itself.  Explicitly flush or
2958 * use cancel_delayed_work_sync() to wait on it.
2959 *
2960 * This function is safe to call from any context including IRQ handler.
2961 */
2962bool cancel_delayed_work(struct delayed_work *dwork)
2963{
2964        unsigned long flags;
2965        int ret;
2966
2967        do {
2968                ret = try_to_grab_pending(&dwork->work, true, &flags);
2969        } while (unlikely(ret == -EAGAIN));
2970
2971        if (unlikely(ret < 0))
2972                return false;
2973
2974        set_work_pool_and_clear_pending(&dwork->work,
2975                                        get_work_pool_id(&dwork->work));
2976        local_irq_restore(flags);
2977        return ret;
2978}
2979EXPORT_SYMBOL(cancel_delayed_work);
2980
2981/**
2982 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
2983 * @dwork: the delayed work cancel
2984 *
2985 * This is cancel_work_sync() for delayed works.
2986 *
2987 * Return:
2988 * %true if @dwork was pending, %false otherwise.
2989 */
2990bool cancel_delayed_work_sync(struct delayed_work *dwork)
2991{
2992        return __cancel_work_timer(&dwork->work, true);
2993}
2994EXPORT_SYMBOL(cancel_delayed_work_sync);
2995
2996/**
2997 * schedule_on_each_cpu - execute a function synchronously on each online CPU
2998 * @func: the function to call
2999 *
3000 * schedule_on_each_cpu() executes @func on each online CPU using the
3001 * system workqueue and blocks until all CPUs have completed.
3002 * schedule_on_each_cpu() is very slow.
3003 *
3004 * Return:
3005 * 0 on success, -errno on failure.
3006 */
3007int schedule_on_each_cpu(work_func_t func)
3008{
3009        int cpu;
3010        struct work_struct __percpu *works;
3011
3012        works = alloc_percpu(struct work_struct);
3013        if (!works)
3014                return -ENOMEM;
3015
3016        get_online_cpus();
3017
3018        for_each_online_cpu(cpu) {
3019                struct work_struct *work = per_cpu_ptr(works, cpu);
3020
3021                INIT_WORK(work, func);
3022                schedule_work_on(cpu, work);
3023        }
3024
3025        for_each_online_cpu(cpu)
3026                flush_work(per_cpu_ptr(works, cpu));
3027
3028        put_online_cpus();
3029        free_percpu(works);
3030        return 0;
3031}
3032
3033/**
3034 * flush_scheduled_work - ensure that any scheduled work has run to completion.
3035 *
3036 * Forces execution of the kernel-global workqueue and blocks until its
3037 * completion.
3038 *
3039 * Think twice before calling this function!  It's very easy to get into
3040 * trouble if you don't take great care.  Either of the following situations
3041 * will lead to deadlock:
3042 *
3043 *      One of the work items currently on the workqueue needs to acquire
3044 *      a lock held by your code or its caller.
3045 *
3046 *      Your code is running in the context of a work routine.
3047 *
3048 * They will be detected by lockdep when they occur, but the first might not
3049 * occur very often.  It depends on what work items are on the workqueue and
3050 * what locks they need, which you have no control over.
3051 *
3052 * In most situations flushing the entire workqueue is overkill; you merely
3053 * need to know that a particular work item isn't queued and isn't running.
3054 * In such cases you should use cancel_delayed_work_sync() or
3055 * cancel_work_sync() instead.
3056 */
3057void flush_scheduled_work(void)
3058{
3059        flush_workqueue(system_wq);
3060}
3061EXPORT_SYMBOL(flush_scheduled_work);
3062
3063/**
3064 * execute_in_process_context - reliably execute the routine with user context
3065 * @fn:         the function to execute
3066 * @ew:         guaranteed storage for the execute work structure (must
3067 *              be available when the work executes)
3068 *
3069 * Executes the function immediately if process context is available,
3070 * otherwise schedules the function for delayed execution.
3071 *
3072 * Return:      0 - function was executed
3073 *              1 - function was scheduled for execution
3074 */
3075int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3076{
3077        if (!in_interrupt()) {
3078                fn(&ew->work);
3079                return 0;
3080        }
3081
3082        INIT_WORK(&ew->work, fn);
3083        schedule_work(&ew->work);
3084
3085        return 1;
3086}
3087EXPORT_SYMBOL_GPL(execute_in_process_context);
3088
3089#ifdef CONFIG_SYSFS
3090/*
3091 * Workqueues with WQ_SYSFS flag set is visible to userland via
3092 * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the
3093 * following attributes.
3094 *
3095 *  per_cpu     RO bool : whether the workqueue is per-cpu or unbound
3096 *  max_active  RW int  : maximum number of in-flight work items
3097 *
3098 * Unbound workqueues have the following extra attributes.
3099 *
3100 *  id          RO int  : the associated pool ID
3101 *  nice        RW int  : nice value of the workers
3102 *  cpumask     RW mask : bitmask of allowed CPUs for the workers
3103 */
3104struct wq_device {
3105        struct workqueue_struct         *wq;
3106        struct device                   dev;
3107};
3108
3109static struct workqueue_struct *dev_to_wq(struct device *dev)
3110{
3111        struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3112
3113        return wq_dev->wq;
3114}
3115
3116static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
3117                            char *buf)
3118{
3119        struct workqueue_struct *wq = dev_to_wq(dev);
3120
3121        return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
3122}
3123static DEVICE_ATTR_RO(per_cpu);
3124
3125static ssize_t max_active_show(struct device *dev,
3126                               struct device_attribute *attr, char *buf)
3127{
3128        struct workqueue_struct *wq = dev_to_wq(dev);
3129
3130        return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
3131}
3132
3133static ssize_t max_active_store(struct device *dev,
3134                                struct device_attribute *attr, const char *buf,
3135                                size_t count)
3136{
3137        struct workqueue_struct *wq = dev_to_wq(dev);
3138        int val;
3139
3140        if (sscanf(buf, "%d", &val) != 1 || val <= 0)
3141                return -EINVAL;
3142
3143        workqueue_set_max_active(wq, val);
3144        return count;
3145}
3146static DEVICE_ATTR_RW(max_active);
3147
3148static struct attribute *wq_sysfs_attrs[] = {
3149        &dev_attr_per_cpu.attr,
3150        &dev_attr_max_active.attr,
3151        NULL,
3152};
3153ATTRIBUTE_GROUPS(wq_sysfs);
3154
3155static ssize_t wq_pool_ids_show(struct device *dev,
3156                                struct device_attribute *attr, char *buf)
3157{
3158        struct workqueue_struct *wq = dev_to_wq(dev);
3159        const char *delim = "";
3160        int node, written = 0;
3161
3162        rcu_read_lock_sched();
3163        for_each_node(node) {
3164                written += scnprintf(buf + written, PAGE_SIZE - written,
3165                                     "%s%d:%d", delim, node,
3166                                     unbound_pwq_by_node(wq, node)->pool->id);
3167                delim = " ";
3168        }
3169        written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3170        rcu_read_unlock_sched();
3171
3172        return written;
3173}
3174
3175static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
3176                            char *buf)
3177{
3178        struct workqueue_struct *wq = dev_to_wq(dev);
3179        int written;
3180
3181        mutex_lock(&wq->mutex);
3182        written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
3183        mutex_unlock(&wq->mutex);
3184
3185        return written;
3186}
3187
3188/* prepare workqueue_attrs for sysfs store operations */
3189static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
3190{
3191        struct workqueue_attrs *attrs;
3192
3193        attrs = alloc_workqueue_attrs(GFP_KERNEL);
3194        if (!attrs)
3195                return NULL;
3196
3197        mutex_lock(&wq->mutex);
3198        copy_workqueue_attrs(attrs, wq->unbound_attrs);
3199        mutex_unlock(&wq->mutex);
3200        return attrs;
3201}
3202
3203static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
3204                             const char *buf, size_t count)
3205{
3206        struct workqueue_struct *wq = dev_to_wq(dev);
3207        struct workqueue_attrs *attrs;
3208        int ret;
3209
3210        attrs = wq_sysfs_prep_attrs(wq);
3211        if (!attrs)
3212                return -ENOMEM;
3213
3214        if (sscanf(buf, "%d", &attrs->nice) == 1 &&
3215            attrs->nice >= -20 && attrs->nice <= 19)
3216                ret = apply_workqueue_attrs(wq, attrs);
3217        else
3218                ret = -EINVAL;
3219
3220        free_workqueue_attrs(attrs);
3221        return ret ?: count;
3222}
3223
3224static ssize_t wq_cpumask_show(struct device *dev,
3225                               struct device_attribute *attr, char *buf)
3226{
3227        struct workqueue_struct *wq = dev_to_wq(dev);
3228        int written;
3229
3230        mutex_lock(&wq->mutex);
3231        written = cpumask_scnprintf(buf, PAGE_SIZE, wq->unbound_attrs->cpumask);
3232        mutex_unlock(&wq->mutex);
3233
3234        written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3235        return written;
3236}
3237
3238static ssize_t wq_cpumask_store(struct device *dev,
3239                                struct device_attribute *attr,
3240                                const char *buf, size_t count)
3241{
3242        struct workqueue_struct *wq = dev_to_wq(dev);
3243        struct workqueue_attrs *attrs;
3244        int ret;
3245
3246        attrs = wq_sysfs_prep_attrs(wq);
3247        if (!attrs)
3248                return -ENOMEM;
3249
3250        ret = cpumask_parse(buf, attrs->cpumask);
3251        if (!ret)
3252                ret = apply_workqueue_attrs(wq, attrs);
3253
3254        free_workqueue_attrs(attrs);
3255        return ret ?: count;
3256}
3257
3258static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
3259                            char *buf)
3260{
3261        struct workqueue_struct *wq = dev_to_wq(dev);
3262        int written;
3263
3264        mutex_lock(&wq->mutex);
3265        written = scnprintf(buf, PAGE_SIZE, "%d\n",
3266                            !wq->unbound_attrs->no_numa);
3267        mutex_unlock(&wq->mutex);
3268
3269        return written;
3270}
3271
3272static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
3273                             const char *buf, size_t count)
3274{
3275        struct workqueue_struct *wq = dev_to_wq(dev);
3276        struct workqueue_attrs *attrs;
3277        int v, ret;
3278
3279        attrs = wq_sysfs_prep_attrs(wq);
3280        if (!attrs)
3281                return -ENOMEM;
3282
3283        ret = -EINVAL;
3284        if (sscanf(buf, "%d", &v) == 1) {
3285                attrs->no_numa = !v;
3286                ret = apply_workqueue_attrs(wq, attrs);
3287        }
3288
3289        free_workqueue_attrs(attrs);
3290        return ret ?: count;
3291}
3292
3293static struct device_attribute wq_sysfs_unbound_attrs[] = {
3294        __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
3295        __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
3296        __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
3297        __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
3298        __ATTR_NULL,
3299};
3300
3301static struct bus_type wq_subsys = {
3302        .name                           = "workqueue",
3303        .dev_groups                     = wq_sysfs_groups,
3304};
3305
3306static int __init wq_sysfs_init(void)
3307{
3308        return subsys_virtual_register(&wq_subsys, NULL);
3309}
3310core_initcall(wq_sysfs_init);
3311
3312static void wq_device_release(struct device *dev)
3313{
3314        struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3315
3316        kfree(wq_dev);
3317}
3318
3319/**
3320 * workqueue_sysfs_register - make a workqueue visible in sysfs
3321 * @wq: the workqueue to register
3322 *
3323 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
3324 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
3325 * which is the preferred method.
3326 *
3327 * Workqueue user should use this function directly iff it wants to apply
3328 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
3329 * apply_workqueue_attrs() may race against userland updating the
3330 * attributes.
3331 *
3332 * Return: 0 on success, -errno on failure.
3333 */
3334int workqueue_sysfs_register(struct workqueue_struct *wq)
3335{
3336        struct wq_device *wq_dev;
3337        int ret;
3338
3339        /*
3340         * Adjusting max_active or creating new pwqs by applyting
3341         * attributes breaks ordering guarantee.  Disallow exposing ordered
3342         * workqueues.
3343         */
3344        if (WARN_ON(wq->flags & __WQ_ORDERED))
3345                return -EINVAL;
3346
3347        wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
3348        if (!wq_dev)
3349                return -ENOMEM;
3350
3351        wq_dev->wq = wq;
3352        wq_dev->dev.bus = &wq_subsys;
3353        wq_dev->dev.init_name = wq->name;
3354        wq_dev->dev.release = wq_device_release;
3355
3356        /*
3357         * unbound_attrs are created separately.  Suppress uevent until
3358         * everything is ready.
3359         */
3360        dev_set_uevent_suppress(&wq_dev->dev, true);
3361
3362        ret = device_register(&wq_dev->dev);
3363        if (ret) {
3364                kfree(wq_dev);
3365                wq->wq_dev = NULL;
3366                return ret;
3367        }
3368
3369        if (wq->flags & WQ_UNBOUND) {
3370                struct device_attribute *attr;
3371
3372                for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
3373                        ret = device_create_file(&wq_dev->dev, attr);
3374                        if (ret) {
3375                                device_unregister(&wq_dev->dev);
3376                                wq->wq_dev = NULL;
3377                                return ret;
3378                        }
3379                }
3380        }
3381
3382        kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
3383        return 0;
3384}
3385
3386/**
3387 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
3388 * @wq: the workqueue to unregister
3389 *
3390 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
3391 */
3392static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
3393{
3394        struct wq_device *wq_dev = wq->wq_dev;
3395
3396        if (!wq->wq_dev)
3397                return;
3398
3399        wq->wq_dev = NULL;
3400        device_unregister(&wq_dev->dev);
3401}
3402#else   /* CONFIG_SYSFS */
3403static void workqueue_sysfs_unregister(struct workqueue_struct *wq)     { }
3404#endif  /* CONFIG_SYSFS */
3405
3406/**
3407 * free_workqueue_attrs - free a workqueue_attrs
3408 * @attrs: workqueue_attrs to free
3409 *
3410 * Undo alloc_workqueue_attrs().
3411 */
3412void free_workqueue_attrs(struct workqueue_attrs *attrs)
3413{
3414        if (attrs) {
3415                free_cpumask_var(attrs->cpumask);
3416                kfree(attrs);
3417        }
3418}
3419
3420/**
3421 * alloc_workqueue_attrs - allocate a workqueue_attrs
3422 * @gfp_mask: allocation mask to use
3423 *
3424 * Allocate a new workqueue_attrs, initialize with default settings and
3425 * return it.
3426 *
3427 * Return: The allocated new workqueue_attr on success. %NULL on failure.
3428 */
3429struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3430{
3431        struct workqueue_attrs *attrs;
3432
3433        attrs = kzalloc(sizeof(*attrs), gfp_mask);
3434        if (!attrs)
3435                goto fail;
3436        if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3437                goto fail;
3438
3439        cpumask_copy(attrs->cpumask, cpu_possible_mask);
3440        return attrs;
3441fail:
3442        free_workqueue_attrs(attrs);
3443        return NULL;
3444}
3445
3446static void copy_workqueue_attrs(struct workqueue_attrs *to,
3447                                 const struct workqueue_attrs *from)
3448{
3449        to->nice = from->nice;
3450        cpumask_copy(to->cpumask, from->cpumask);
3451        /*
3452         * Unlike hash and equality test, this function doesn't ignore
3453         * ->no_numa as it is used for both pool and wq attrs.  Instead,
3454         * get_unbound_pool() explicitly clears ->no_numa after copying.
3455         */
3456        to->no_numa = from->no_numa;
3457}
3458
3459/* hash value of the content of @attr */
3460static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3461{
3462        u32 hash = 0;
3463
3464        hash = jhash_1word(attrs->nice, hash);
3465        hash = jhash(cpumask_bits(attrs->cpumask),
3466                     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3467        return hash;
3468}
3469
3470/* content equality test */
3471static bool wqattrs_equal(const struct workqueue_attrs *a,
3472                          const struct workqueue_attrs *b)
3473{
3474        if (a->nice != b->nice)
3475                return false;
3476        if (!cpumask_equal(a->cpumask, b->cpumask))
3477                return false;
3478        return true;
3479}
3480
3481/**
3482 * init_worker_pool - initialize a newly zalloc'd worker_pool
3483 * @pool: worker_pool to initialize
3484 *
3485 * Initiailize a newly zalloc'd @pool.  It also allocates @pool->attrs.
3486 *
3487 * Return: 0 on success, -errno on failure.  Even on failure, all fields
3488 * inside @pool proper are initialized and put_unbound_pool() can be called
3489 * on @pool safely to release it.
3490 */
3491static int init_worker_pool(struct worker_pool *pool)
3492{
3493        spin_lock_init(&pool->lock);
3494        pool->id = -1;
3495        pool->cpu = -1;
3496        pool->node = NUMA_NO_NODE;
3497        pool->flags |= POOL_DISASSOCIATED;
3498        INIT_LIST_HEAD(&pool->worklist);
3499        INIT_LIST_HEAD(&pool->idle_list);
3500        hash_init(pool->busy_hash);
3501
3502        init_timer_deferrable(&pool->idle_timer);
3503        pool->idle_timer.function = idle_worker_timeout;
3504        pool->idle_timer.data = (unsigned long)pool;
3505
3506        setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3507                    (unsigned long)pool);
3508
3509        mutex_init(&pool->manager_arb);
3510        mutex_init(&pool->manager_mutex);
3511        idr_init(&pool->worker_idr);
3512
3513        INIT_HLIST_NODE(&pool->hash_node);
3514        pool->refcnt = 1;
3515
3516        /* shouldn't fail above this point */
3517        pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3518        if (!pool->attrs)
3519                return -ENOMEM;
3520        return 0;
3521}
3522
3523static void rcu_free_pool(struct rcu_head *rcu)
3524{
3525        struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3526
3527        idr_destroy(&pool->worker_idr);
3528        free_workqueue_attrs(pool->attrs);
3529        kfree(pool);
3530}
3531
3532/**
3533 * put_unbound_pool - put a worker_pool
3534 * @pool: worker_pool to put
3535 *
3536 * Put @pool.  If its refcnt reaches zero, it gets destroyed in sched-RCU
3537 * safe manner.  get_unbound_pool() calls this function on its failure path
3538 * and this function should be able to release pools which went through,
3539 * successfully or not, init_worker_pool().
3540 *
3541 * Should be called with wq_pool_mutex held.
3542 */
3543static void put_unbound_pool(struct worker_pool *pool)
3544{
3545        struct worker *worker;
3546
3547        lockdep_assert_held(&wq_pool_mutex);
3548
3549        if (--pool->refcnt)
3550                return;
3551
3552        /* sanity checks */
3553        if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
3554            WARN_ON(!list_empty(&pool->worklist)))
3555                return;
3556
3557        /* release id and unhash */
3558        if (pool->id >= 0)
3559                idr_remove(&worker_pool_idr, pool->id);
3560        hash_del(&pool->hash_node);
3561
3562        /*
3563         * Become the manager and destroy all workers.  Grabbing
3564         * manager_arb prevents @pool's workers from blocking on
3565         * manager_mutex.
3566         */
3567        mutex_lock(&pool->manager_arb);
3568        mutex_lock(&pool->manager_mutex);
3569        spin_lock_irq(&pool->lock);
3570
3571        while ((worker = first_worker(pool)))
3572                destroy_worker(worker);
3573        WARN_ON(pool->nr_workers || pool->nr_idle);
3574
3575        spin_unlock_irq(&pool->lock);
3576        mutex_unlock(&pool->manager_mutex);
3577        mutex_unlock(&pool->manager_arb);
3578
3579        /* shut down the timers */
3580        del_timer_sync(&pool->idle_timer);
3581        del_timer_sync(&pool->mayday_timer);
3582
3583        /* sched-RCU protected to allow dereferences from get_work_pool() */
3584        call_rcu_sched(&pool->rcu, rcu_free_pool);
3585}
3586
3587/**
3588 * get_unbound_pool - get a worker_pool with the specified attributes
3589 * @attrs: the attributes of the worker_pool to get
3590 *
3591 * Obtain a worker_pool which has the same attributes as @attrs, bump the
3592 * reference count and return it.  If there already is a matching
3593 * worker_pool, it will be used; otherwise, this function attempts to
3594 * create a new one.
3595 *
3596 * Should be called with wq_pool_mutex held.
3597 *
3598 * Return: On success, a worker_pool with the same attributes as @attrs.
3599 * On failure, %NULL.
3600 */
3601static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3602{
3603        u32 hash = wqattrs_hash(attrs);
3604        struct worker_pool *pool;
3605        int node;
3606
3607        lockdep_assert_held(&wq_pool_mutex);
3608
3609        /* do we already have a matching pool? */
3610        hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3611                if (wqattrs_equal(pool->attrs, attrs)) {
3612                        pool->refcnt++;
3613                        goto out_unlock;
3614                }
3615        }
3616
3617        /* nope, create a new one */
3618        pool = kzalloc(sizeof(*pool), GFP_KERNEL);
3619        if (!pool || init_worker_pool(pool) < 0)
3620                goto fail;
3621
3622        if (workqueue_freezing)
3623                pool->flags |= POOL_FREEZING;
3624
3625        lockdep_set_subclass(&pool->lock, 1);   /* see put_pwq() */
3626        copy_workqueue_attrs(pool->attrs, attrs);
3627
3628        /*
3629         * no_numa isn't a worker_pool attribute, always clear it.  See
3630         * 'struct workqueue_attrs' comments for detail.
3631         */
3632        pool->attrs->no_numa = false;
3633
3634        /* if cpumask is contained inside a NUMA node, we belong to that node */
3635        if (wq_numa_enabled) {
3636                for_each_node(node) {
3637                        if (cpumask_subset(pool->attrs->cpumask,
3638                                           wq_numa_possible_cpumask[node])) {
3639                                pool->node = node;
3640                                break;
3641                        }
3642                }
3643        }
3644
3645        if (worker_pool_assign_id(pool) < 0)
3646                goto fail;
3647
3648        /* create and start the initial worker */
3649        if (create_and_start_worker(pool) < 0)
3650                goto fail;
3651
3652        /* install */
3653        hash_add(unbound_pool_hash, &pool->hash_node, hash);
3654out_unlock:
3655        return pool;
3656fail:
3657        if (pool)
3658                put_unbound_pool(pool);
3659        return NULL;
3660}
3661
3662static void rcu_free_pwq(struct rcu_head *rcu)
3663{
3664        kmem_cache_free(pwq_cache,
3665                        container_of(rcu, struct pool_workqueue, rcu));
3666}
3667
3668/*
3669 * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3670 * and needs to be destroyed.
3671 */
3672static void pwq_unbound_release_workfn(struct work_struct *work)
3673{
3674        struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3675                                                  unbound_release_work);
3676        struct workqueue_struct *wq = pwq->wq;
3677        struct worker_pool *pool = pwq->pool;
3678        bool is_last;
3679
3680        if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3681                return;
3682
3683        /*
3684         * Unlink @pwq.  Synchronization against wq->mutex isn't strictly
3685         * necessary on release but do it anyway.  It's easier to verify
3686         * and consistent with the linking path.
3687         */
3688        mutex_lock(&wq->mutex);
3689        list_del_rcu(&pwq->pwqs_node);
3690        is_last = list_empty(&wq->pwqs);
3691        mutex_unlock(&wq->mutex);
3692
3693        mutex_lock(&wq_pool_mutex);
3694        put_unbound_pool(pool);
3695        mutex_unlock(&wq_pool_mutex);
3696
3697        call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3698
3699        /*
3700         * If we're the last pwq going away, @wq is already dead and no one
3701         * is gonna access it anymore.  Free it.
3702         */
3703        if (is_last) {
3704                free_workqueue_attrs(wq->unbound_attrs);
3705                kfree(wq);
3706        }
3707}
3708
3709/**
3710 * pwq_adjust_max_active - update a pwq's max_active to the current setting
3711 * @pwq: target pool_workqueue
3712 *
3713 * If @pwq isn't freezing, set @pwq->max_active to the associated
3714 * workqueue's saved_max_active and activate delayed work items
3715 * accordingly.  If @pwq is freezing, clear @pwq->max_active to zero.
3716 */
3717static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3718{
3719        struct workqueue_struct *wq = pwq->wq;
3720        bool freezable = wq->flags & WQ_FREEZABLE;
3721
3722        /* for @wq->saved_max_active */
3723        lockdep_assert_held(&wq->mutex);
3724
3725        /* fast exit for non-freezable wqs */
3726        if (!freezable && pwq->max_active == wq->saved_max_active)
3727                return;
3728
3729        spin_lock_irq(&pwq->pool->lock);
3730
3731        if (!freezable || !(pwq->pool->flags & POOL_FREEZING)) {
3732                pwq->max_active = wq->saved_max_active;
3733
3734                while (!list_empty(&pwq->delayed_works) &&
3735                       pwq->nr_active < pwq->max_active)
3736                        pwq_activate_first_delayed(pwq);
3737
3738                /*
3739                 * Need to kick a worker after thawed or an unbound wq's
3740                 * max_active is bumped.  It's a slow path.  Do it always.
3741                 */
3742                wake_up_worker(pwq->pool);
3743        } else {
3744                pwq->max_active = 0;
3745        }
3746
3747        spin_unlock_irq(&pwq->pool->lock);
3748}
3749
3750/* initialize newly alloced @pwq which is associated with @wq and @pool */
3751static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
3752                     struct worker_pool *pool)
3753{
3754        BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3755
3756        memset(pwq, 0, sizeof(*pwq));
3757
3758        pwq->pool = pool;
3759        pwq->wq = wq;
3760        pwq->flush_color = -1;
3761        pwq->refcnt = 1;
3762        INIT_LIST_HEAD(&pwq->delayed_works);
3763        INIT_LIST_HEAD(&pwq->pwqs_node);
3764        INIT_LIST_HEAD(&pwq->mayday_node);
3765        INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3766}
3767
3768/* sync @pwq with the current state of its associated wq and link it */
3769static void link_pwq(struct pool_workqueue *pwq)
3770{
3771        struct workqueue_struct *wq = pwq->wq;
3772
3773        lockdep_assert_held(&wq->mutex);
3774
3775        /* may be called multiple times, ignore if already linked */
3776        if (!list_empty(&pwq->pwqs_node))
3777                return;
3778
3779        /*
3780         * Set the matching work_color.  This is synchronized with
3781         * wq->mutex to avoid confusing flush_workqueue().
3782         */
3783        pwq->work_color = wq->work_color;
3784
3785        /* sync max_active to the current setting */
3786        pwq_adjust_max_active(pwq);
3787
3788        /* link in @pwq */
3789        list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3790}
3791
3792/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3793static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
3794                                        const struct workqueue_attrs *attrs)
3795{
3796        struct worker_pool *pool;
3797        struct pool_workqueue *pwq;
3798
3799        lockdep_assert_held(&wq_pool_mutex);
3800
3801        pool = get_unbound_pool(attrs);
3802        if (!pool)
3803                return NULL;
3804
3805        pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3806        if (!pwq) {
3807                put_unbound_pool(pool);
3808                return NULL;
3809        }
3810
3811        init_pwq(pwq, wq, pool);
3812        return pwq;
3813}
3814
3815/* undo alloc_unbound_pwq(), used only in the error path */
3816static void free_unbound_pwq(struct pool_workqueue *pwq)
3817{
3818        lockdep_assert_held(&wq_pool_mutex);
3819
3820        if (pwq) {
3821                put_unbound_pool(pwq->pool);
3822                kmem_cache_free(pwq_cache, pwq);
3823        }
3824}
3825
3826/**
3827 * wq_calc_node_mask - calculate a wq_attrs' cpumask for the specified node
3828 * @attrs: the wq_attrs of interest
3829 * @node: the target NUMA node
3830 * @cpu_going_down: if >= 0, the CPU to consider as offline
3831 * @cpumask: outarg, the resulting cpumask
3832 *
3833 * Calculate the cpumask a workqueue with @attrs should use on @node.  If
3834 * @cpu_going_down is >= 0, that cpu is considered offline during
3835 * calculation.  The result is stored in @cpumask.
3836 *
3837 * If NUMA affinity is not enabled, @attrs->cpumask is always used.  If
3838 * enabled and @node has online CPUs requested by @attrs, the returned
3839 * cpumask is the intersection of the possible CPUs of @node and
3840 * @attrs->cpumask.
3841 *
3842 * The caller is responsible for ensuring that the cpumask of @node stays
3843 * stable.
3844 *
3845 * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3846 * %false if equal.
3847 */
3848static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3849                                 int cpu_going_down, cpumask_t *cpumask)
3850{
3851        if (!wq_numa_enabled || attrs->no_numa)
3852                goto use_dfl;
3853
3854        /* does @node have any online CPUs @attrs wants? */
3855        cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3856        if (cpu_going_down >= 0)
3857                cpumask_clear_cpu(cpu_going_down, cpumask);
3858
3859        if (cpumask_empty(cpumask))
3860                goto use_dfl;
3861
3862        /* yeap, return possible CPUs in @node that @attrs wants */
3863        cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3864        return !cpumask_equal(cpumask, attrs->cpumask);
3865
3866use_dfl:
3867        cpumask_copy(cpumask, attrs->cpumask);
3868        return false;
3869}
3870
3871/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3872static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
3873                                                   int node,
3874                                                   struct pool_workqueue *pwq)
3875{
3876        struct pool_workqueue *old_pwq;
3877
3878        lockdep_assert_held(&wq->mutex);
3879
3880        /* link_pwq() can handle duplicate calls */
3881        link_pwq(pwq);
3882
3883        old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3884        rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3885        return old_pwq;
3886}
3887
3888/**
3889 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3890 * @wq: the target workqueue
3891 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3892 *
3893 * Apply @attrs to an unbound workqueue @wq.  Unless disabled, on NUMA
3894 * machines, this function maps a separate pwq to each NUMA node with
3895 * possibles CPUs in @attrs->cpumask so that work items are affine to the
3896 * NUMA node it was issued on.  Older pwqs are released as in-flight work
3897 * items finish.  Note that a work item which repeatedly requeues itself
3898 * back-to-back will stay on its current pwq.
3899 *
3900 * Performs GFP_KERNEL allocations.
3901 *
3902 * Return: 0 on success and -errno on failure.
3903 */
3904int apply_workqueue_attrs(struct workqueue_struct *wq,
3905                          const struct workqueue_attrs *attrs)
3906{
3907        struct workqueue_attrs *new_attrs, *tmp_attrs;
3908        struct pool_workqueue **pwq_tbl, *dfl_pwq;
3909        int node, ret;
3910
3911        /* only unbound workqueues can change attributes */
3912        if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3913                return -EINVAL;
3914
3915        /* creating multiple pwqs breaks ordering guarantee */
3916        if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3917                return -EINVAL;
3918
3919        pwq_tbl = kzalloc(wq_numa_tbl_len * sizeof(pwq_tbl[0]), GFP_KERNEL);
3920        new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3921        tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3922        if (!pwq_tbl || !new_attrs || !tmp_attrs)
3923                goto enomem;
3924
3925        /* make a copy of @attrs and sanitize it */
3926        copy_workqueue_attrs(new_attrs, attrs);
3927        cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
3928
3929        /*
3930         * We may create multiple pwqs with differing cpumasks.  Make a
3931         * copy of @new_attrs which will be modified and used to obtain
3932         * pools.
3933         */
3934        copy_workqueue_attrs(tmp_attrs, new_attrs);
3935
3936        /*
3937         * CPUs should stay stable across pwq creations and installations.
3938         * Pin CPUs, determine the target cpumask for each node and create
3939         * pwqs accordingly.
3940         */
3941        get_online_cpus();
3942
3943        mutex_lock(&wq_pool_mutex);
3944
3945        /*
3946         * If something goes wrong during CPU up/down, we'll fall back to
3947         * the default pwq covering whole @attrs->cpumask.  Always create
3948         * it even if we don't use it immediately.
3949         */
3950        dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3951        if (!dfl_pwq)
3952                goto enomem_pwq;
3953
3954        for_each_node(node) {
3955                if (wq_calc_node_cpumask(attrs, node, -1, tmp_attrs->cpumask)) {
3956                        pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
3957                        if (!pwq_tbl[node])
3958                                goto enomem_pwq;
3959                } else {
3960                        dfl_pwq->refcnt++;
3961                        pwq_tbl[node] = dfl_pwq;
3962                }
3963        }
3964
3965        mutex_unlock(&wq_pool_mutex);
3966
3967        /* all pwqs have been created successfully, let's install'em */
3968        mutex_lock(&wq->mutex);
3969
3970        copy_workqueue_attrs(wq->unbound_attrs, new_attrs);
3971
3972        /* save the previous pwq and install the new one */
3973        for_each_node(node)
3974                pwq_tbl[node] = numa_pwq_tbl_install(wq, node, pwq_tbl[node]);
3975
3976        /* @dfl_pwq might not have been used, ensure it's linked */
3977        link_pwq(dfl_pwq);
3978        swap(wq->dfl_pwq, dfl_pwq);
3979
3980        mutex_unlock(&wq->mutex);
3981
3982        /* put the old pwqs */
3983        for_each_node(node)
3984                put_pwq_unlocked(pwq_tbl[node]);
3985        put_pwq_unlocked(dfl_pwq);
3986
3987        put_online_cpus();
3988        ret = 0;
3989        /* fall through */
3990out_free:
3991        free_workqueue_attrs(tmp_attrs);
3992        free_workqueue_attrs(new_attrs);
3993        kfree(pwq_tbl);
3994        return ret;
3995
3996enomem_pwq:
3997        free_unbound_pwq(dfl_pwq);
3998        for_each_node(node)
3999                if (pwq_tbl && pwq_tbl[node] != dfl_pwq)
4000                        free_unbound_pwq(pwq_tbl[node]);
4001        mutex_unlock(&wq_pool_mutex);
4002        put_online_cpus();
4003enomem:
4004        ret = -ENOMEM;
4005        goto out_free;
4006}
4007
4008/**
4009 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4010 * @wq: the target workqueue
4011 * @cpu: the CPU coming up or going down
4012 * @online: whether @cpu is coming up or going down
4013 *
4014 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
4015 * %CPU_DOWN_FAILED.  @cpu is being hot[un]plugged, update NUMA affinity of
4016 * @wq accordingly.
4017 *
4018 * If NUMA affinity can't be adjusted due to memory allocation failure, it
4019 * falls back to @wq->dfl_pwq which may not be optimal but is always
4020 * correct.
4021 *
4022 * Note that when the last allowed CPU of a NUMA node goes offline for a
4023 * workqueue with a cpumask spanning multiple nodes, the workers which were
4024 * already executing the work items for the workqueue will lose their CPU
4025 * affinity and may execute on any CPU.  This is similar to how per-cpu
4026 * workqueues behave on CPU_DOWN.  If a workqueue user wants strict
4027 * affinity, it's the user's responsibility to flush the work item from
4028 * CPU_DOWN_PREPARE.
4029 */
4030static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4031                                   bool online)
4032{
4033        int node = cpu_to_node(cpu);
4034        int cpu_off = online ? -1 : cpu;
4035        struct pool_workqueue *old_pwq = NULL, *pwq;
4036        struct workqueue_attrs *target_attrs;
4037        cpumask_t *cpumask;
4038
4039        lockdep_assert_held(&wq_pool_mutex);
4040
4041        if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND))
4042                return;
4043
4044        /*
4045         * We don't wanna alloc/free wq_attrs for each wq for each CPU.
4046         * Let's use a preallocated one.  The following buf is protected by
4047         * CPU hotplug exclusion.
4048         */
4049        target_attrs = wq_update_unbound_numa_attrs_buf;
4050        cpumask = target_attrs->cpumask;
4051
4052        mutex_lock(&wq->mutex);
4053        if (wq->unbound_attrs->no_numa)
4054                goto out_unlock;
4055
4056        copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4057        pwq = unbound_pwq_by_node(wq, node);
4058
4059        /*
4060         * Let's determine what needs to be done.  If the target cpumask is
4061         * different from wq's, we need to compare it to @pwq's and create
4062         * a new one if they don't match.  If the target cpumask equals
4063         * wq's, the default pwq should be used.  If @pwq is already the
4064         * default one, nothing to do; otherwise, install the default one.
4065         */
4066        if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {
4067                if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4068                        goto out_unlock;
4069        } else {
4070                if (pwq == wq->dfl_pwq)
4071                        goto out_unlock;
4072                else
4073                        goto use_dfl_pwq;
4074        }
4075
4076        mutex_unlock(&wq->mutex);
4077
4078        /* create a new pwq */
4079        pwq = alloc_unbound_pwq(wq, target_attrs);
4080        if (!pwq) {
4081                pr_warning("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4082                           wq->name);
4083                goto out_unlock;
4084        }
4085
4086        /*
4087         * Install the new pwq.  As this function is called only from CPU
4088         * hotplug callbacks and applying a new attrs is wrapped with
4089         * get/put_online_cpus(), @wq->unbound_attrs couldn't have changed
4090         * inbetween.
4091         */
4092        mutex_lock(&wq->mutex);
4093        old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4094        goto out_unlock;
4095
4096use_dfl_pwq:
4097        spin_lock_irq(&wq->dfl_pwq->pool->lock);
4098        get_pwq(wq->dfl_pwq);
4099        spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4100        old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4101out_unlock:
4102        mutex_unlock(&wq->mutex);
4103        put_pwq_unlocked(old_pwq);
4104}
4105
4106static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4107{
4108        bool highpri = wq->flags & WQ_HIGHPRI;
4109        int cpu;
4110
4111        if (!(wq->flags & WQ_UNBOUND)) {
4112                wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4113                if (!wq->cpu_pwqs)
4114                        return -ENOMEM;
4115
4116                for_each_possible_cpu(cpu) {
4117                        struct pool_workqueue *pwq =
4118                                per_cpu_ptr(wq->cpu_pwqs, cpu);
4119                        struct worker_pool *cpu_pools =
4120                                per_cpu(cpu_worker_pools, cpu);
4121
4122                        init_pwq(pwq, wq, &cpu_pools[highpri]);
4123
4124                        mutex_lock(&wq->mutex);
4125                        link_pwq(pwq);
4126                        mutex_unlock(&wq->mutex);
4127                }
4128                return 0;
4129        } else {
4130                return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4131        }
4132}
4133
4134static int wq_clamp_max_active(int max_active, unsigned int flags,
4135                               const char *name)
4136{
4137        int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4138
4139        if (max_active < 1 || max_active > lim)
4140                pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4141                        max_active, name, 1, lim);
4142
4143        return clamp_val(max_active, 1, lim);
4144}
4145
4146struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
4147                                               unsigned int flags,
4148                                               int max_active,
4149                                               struct lock_class_key *key,
4150                                               const char *lock_name, ...)
4151{
4152        size_t tbl_size = 0;
4153        va_list args;
4154        struct workqueue_struct *wq;
4155        struct pool_workqueue *pwq;
4156
4157        /* see the comment above the definition of WQ_POWER_EFFICIENT */
4158        if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4159                flags |= WQ_UNBOUND;
4160
4161        /* allocate wq and format name */
4162        if (flags & WQ_UNBOUND)
4163                tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
4164
4165        wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4166        if (!wq)
4167                return NULL;
4168
4169        if (flags & WQ_UNBOUND) {
4170                wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4171                if (!wq->unbound_attrs)
4172                        goto err_free_wq;
4173        }
4174
4175        va_start(args, lock_name);
4176        vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4177        va_end(args);
4178
4179        max_active = max_active ?: WQ_DFL_ACTIVE;
4180        max_active = wq_clamp_max_active(max_active, flags, wq->name);
4181
4182        /* init wq */
4183        wq->flags = flags;
4184        wq->saved_max_active = max_active;
4185        mutex_init(&wq->mutex);
4186        atomic_set(&wq->nr_pwqs_to_flush, 0);
4187        INIT_LIST_HEAD(&wq->pwqs);
4188        INIT_LIST_HEAD(&wq->flusher_queue);
4189        INIT_LIST_HEAD(&wq->flusher_overflow);
4190        INIT_LIST_HEAD(&wq->maydays);
4191
4192        lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
4193        INIT_LIST_HEAD(&wq->list);
4194
4195        if (alloc_and_link_pwqs(wq) < 0)
4196                goto err_free_wq;
4197
4198        /*
4199         * Workqueues which may be used during memory reclaim should
4200         * have a rescuer to guarantee forward progress.
4201         */
4202        if (flags & WQ_MEM_RECLAIM) {
4203                struct worker *rescuer;
4204
4205                rescuer = alloc_worker();
4206                if (!rescuer)
4207                        goto err_destroy;
4208
4209                rescuer->rescue_wq = wq;
4210                rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
4211                                               wq->name);
4212                if (IS_ERR(rescuer->task)) {
4213                        kfree(rescuer);
4214                        goto err_destroy;
4215                }
4216
4217                wq->rescuer = rescuer;
4218                rescuer->task->flags |= PF_NO_SETAFFINITY;
4219                wake_up_process(rescuer->task);
4220        }
4221
4222        if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4223                goto err_destroy;
4224
4225        /*
4226         * wq_pool_mutex protects global freeze state and workqueues list.
4227         * Grab it, adjust max_active and add the new @wq to workqueues
4228         * list.
4229         */
4230        mutex_lock(&wq_pool_mutex);
4231
4232        mutex_lock(&wq->mutex);
4233        for_each_pwq(pwq, wq)
4234                pwq_adjust_max_active(pwq);
4235        mutex_unlock(&wq->mutex);
4236
4237        list_add(&wq->list, &workqueues);
4238
4239        mutex_unlock(&wq_pool_mutex);
4240
4241        return wq;
4242
4243err_free_wq:
4244        free_workqueue_attrs(wq->unbound_attrs);
4245        kfree(wq);
4246        return NULL;
4247err_destroy:
4248        destroy_workqueue(wq);
4249        return NULL;
4250}
4251EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
4252
4253/**
4254 * destroy_workqueue - safely terminate a workqueue
4255 * @wq: target workqueue
4256 *
4257 * Safely destroy a workqueue. All work currently pending will be done first.
4258 */
4259void destroy_workqueue(struct workqueue_struct *wq)
4260{
4261        struct pool_workqueue *pwq;
4262        int node;
4263
4264        /* drain it before proceeding with destruction */
4265        drain_workqueue(wq);
4266
4267        /* sanity checks */
4268        mutex_lock(&wq->mutex);
4269        for_each_pwq(pwq, wq) {
4270                int i;
4271
4272                for (i = 0; i < WORK_NR_COLORS; i++) {
4273                        if (WARN_ON(pwq->nr_in_flight[i])) {
4274                                mutex_unlock(&wq->mutex);
4275                                return;
4276                        }
4277                }
4278
4279                if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||
4280                    WARN_ON(pwq->nr_active) ||
4281                    WARN_ON(!list_empty(&pwq->delayed_works))) {
4282                        mutex_unlock(&wq->mutex);
4283                        return;
4284                }
4285        }
4286        mutex_unlock(&wq->mutex);
4287
4288        /*
4289         * wq list is used to freeze wq, remove from list after
4290         * flushing is complete in case freeze races us.
4291         */
4292        mutex_lock(&wq_pool_mutex);
4293        list_del_init(&wq->list);
4294        mutex_unlock(&wq_pool_mutex);
4295
4296        workqueue_sysfs_unregister(wq);
4297
4298        if (wq->rescuer) {
4299                kthread_stop(wq->rescuer->task);
4300                kfree(wq->rescuer);
4301                wq->rescuer = NULL;
4302        }
4303
4304        if (!(wq->flags & WQ_UNBOUND)) {
4305                /*
4306                 * The base ref is never dropped on per-cpu pwqs.  Directly
4307                 * free the pwqs and wq.
4308                 */
4309                free_percpu(wq->cpu_pwqs);
4310                kfree(wq);
4311        } else {
4312                /*
4313                 * We're the sole accessor of @wq at this point.  Directly
4314                 * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4315                 * @wq will be freed when the last pwq is released.
4316                 */
4317                for_each_node(node) {
4318                        pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4319                        RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4320                        put_pwq_unlocked(pwq);
4321                }
4322
4323                /*
4324                 * Put dfl_pwq.  @wq may be freed any time after dfl_pwq is
4325                 * put.  Don't access it afterwards.
4326                 */
4327                pwq = wq->dfl_pwq;
4328                wq->dfl_pwq = NULL;
4329                put_pwq_unlocked(pwq);
4330        }
4331}
4332EXPORT_SYMBOL_GPL(destroy_workqueue);
4333
4334/**
4335 * workqueue_set_max_active - adjust max_active of a workqueue
4336 * @wq: target workqueue
4337 * @max_active: new max_active value.
4338 *
4339 * Set max_active of @wq to @max_active.
4340 *
4341 * CONTEXT:
4342 * Don't call from IRQ context.
4343 */
4344void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4345{
4346        struct pool_workqueue *pwq;
4347
4348        /* disallow meddling with max_active for ordered workqueues */
4349        if (WARN_ON(wq->flags & __WQ_ORDERED))
4350                return;
4351
4352        max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4353
4354        mutex_lock(&wq->mutex);
4355
4356        wq->saved_max_active = max_active;
4357
4358        for_each_pwq(pwq, wq)
4359                pwq_adjust_max_active(pwq);
4360
4361        mutex_unlock(&wq->mutex);
4362}
4363EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4364
4365/**
4366 * current_is_workqueue_rescuer - is %current workqueue rescuer?
4367 *
4368 * Determine whether %current is a workqueue rescuer.  Can be used from
4369 * work functions to determine whether it's being run off the rescuer task.
4370 *
4371 * Return: %true if %current is a workqueue rescuer. %false otherwise.
4372 */
4373bool current_is_workqueue_rescuer(void)
4374{
4375        struct worker *worker = current_wq_worker();
4376
4377        return worker && worker->rescue_wq;
4378}
4379
4380/**
4381 * workqueue_congested - test whether a workqueue is congested
4382 * @cpu: CPU in question
4383 * @wq: target workqueue
4384 *
4385 * Test whether @wq's cpu workqueue for @cpu is congested.  There is
4386 * no synchronization around this function and the test result is
4387 * unreliable and only useful as advisory hints or for debugging.
4388 *
4389 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4390 * Note that both per-cpu and unbound workqueues may be associated with
4391 * multiple pool_workqueues which have separate congested states.  A
4392 * workqueue being congested on one CPU doesn't mean the workqueue is also
4393 * contested on other CPUs / NUMA nodes.
4394 *
4395 * Return:
4396 * %true if congested, %false otherwise.
4397 */
4398bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4399{
4400        struct pool_workqueue *pwq;
4401        bool ret;
4402
4403        rcu_read_lock_sched();
4404
4405        if (cpu == WORK_CPU_UNBOUND)
4406                cpu = smp_processor_id();
4407
4408        if (!(wq->flags & WQ_UNBOUND))
4409                pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4410        else
4411                pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4412
4413        ret = !list_empty(&pwq->delayed_works);
4414        rcu_read_unlock_sched();
4415
4416        return ret;
4417}
4418EXPORT_SYMBOL_GPL(workqueue_congested);
4419
4420/**
4421 * work_busy - test whether a work is currently pending or running
4422 * @work: the work to be tested
4423 *
4424 * Test whether @work is currently pending or running.  There is no
4425 * synchronization around this function and the test result is
4426 * unreliable and only useful as advisory hints or for debugging.
4427 *
4428 * Return:
4429 * OR'd bitmask of WORK_BUSY_* bits.
4430 */
4431unsigned int work_busy(struct work_struct *work)
4432{
4433        struct worker_pool *pool;
4434        unsigned long flags;
4435        unsigned int ret = 0;
4436
4437        if (work_pending(work))
4438                ret |= WORK_BUSY_PENDING;
4439
4440        local_irq_save(flags);
4441        pool = get_work_pool(work);
4442        if (pool) {
4443                spin_lock(&pool->lock);
4444                if (find_worker_executing_work(pool, work))
4445                        ret |= WORK_BUSY_RUNNING;
4446                spin_unlock(&pool->lock);
4447        }
4448        local_irq_restore(flags);
4449
4450        return ret;
4451}
4452EXPORT_SYMBOL_GPL(work_busy);
4453
4454/**
4455 * set_worker_desc - set description for the current work item
4456 * @fmt: printf-style format string
4457 * @...: arguments for the format string
4458 *
4459 * This function can be called by a running work function to describe what
4460 * the work item is about.  If the worker task gets dumped, this
4461 * information will be printed out together to help debugging.  The
4462 * description can be at most WORKER_DESC_LEN including the trailing '\0'.
4463 */
4464void set_worker_desc(const char *fmt, ...)
4465{
4466        struct worker *worker = current_wq_worker();
4467        va_list args;
4468
4469        if (worker) {
4470                va_start(args, fmt);
4471                vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4472                va_end(args);
4473                worker->desc_valid = true;
4474        }
4475}
4476
4477/**
4478 * print_worker_info - print out worker information and description
4479 * @log_lvl: the log level to use when printing
4480 * @task: target task
4481 *
4482 * If @task is a worker and currently executing a work item, print out the
4483 * name of the workqueue being serviced and worker description set with
4484 * set_worker_desc() by the currently executing work item.
4485 *
4486 * This function can be safely called on any task as long as the
4487 * task_struct itself is accessible.  While safe, this function isn't
4488 * synchronized and may print out mixups or garbages of limited length.
4489 */
4490void print_worker_info(const char *log_lvl, struct task_struct *task)
4491{
4492        work_func_t *fn = NULL;
4493        char name[WQ_NAME_LEN] = { };
4494        char desc[WORKER_DESC_LEN] = { };
4495        struct pool_workqueue *pwq = NULL;
4496        struct workqueue_struct *wq = NULL;
4497        bool desc_valid = false;
4498        struct worker *worker;
4499
4500        if (!(task->flags & PF_WQ_WORKER))
4501                return;
4502
4503        /*
4504         * This function is called without any synchronization and @task
4505         * could be in any state.  Be careful with dereferences.
4506         */
4507        worker = probe_kthread_data(task);
4508
4509        /*
4510         * Carefully copy the associated workqueue's workfn and name.  Keep
4511         * the original last '\0' in case the original contains garbage.
4512         */
4513        probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4514        probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4515        probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4516        probe_kernel_read(name, wq->name, sizeof(name) - 1);
4517
4518        /* copy worker description */
4519        probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4520        if (desc_valid)
4521                probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4522
4523        if (fn || name[0] || desc[0]) {
4524                printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4525                if (desc[0])
4526                        pr_cont(" (%s)", desc);
4527                pr_cont("\n");
4528        }
4529}
4530
4531/*
4532 * CPU hotplug.
4533 *
4534 * There are two challenges in supporting CPU hotplug.  Firstly, there
4535 * are a lot of assumptions on strong associations among work, pwq and
4536 * pool which make migrating pending and scheduled works very
4537 * difficult to implement without impacting hot paths.  Secondly,
4538 * worker pools serve mix of short, long and very long running works making
4539 * blocked draining impractical.
4540 *
4541 * This is solved by allowing the pools to be disassociated from the CPU
4542 * running as an unbound one and allowing it to be reattached later if the
4543 * cpu comes back online.
4544 */
4545
4546static void wq_unbind_fn(struct work_struct *work)
4547{
4548        int cpu = smp_processor_id();
4549        struct worker_pool *pool;
4550        struct worker *worker;
4551        int wi;
4552
4553        for_each_cpu_worker_pool(pool, cpu) {
4554                WARN_ON_ONCE(cpu != smp_processor_id());
4555
4556                mutex_lock(&pool->manager_mutex);
4557                spin_lock_irq(&pool->lock);
4558
4559                /*
4560                 * We've blocked all manager operations.  Make all workers
4561                 * unbound and set DISASSOCIATED.  Before this, all workers
4562                 * except for the ones which are still executing works from
4563                 * before the last CPU down must be on the cpu.  After
4564                 * this, they may become diasporas.
4565                 */
4566                for_each_pool_worker(worker, wi, pool)
4567                        worker->flags |= WORKER_UNBOUND;
4568
4569                pool->flags |= POOL_DISASSOCIATED;
4570
4571                spin_unlock_irq(&pool->lock);
4572                mutex_unlock(&pool->manager_mutex);
4573
4574                /*
4575                 * Call schedule() so that we cross rq->lock and thus can
4576                 * guarantee sched callbacks see the %WORKER_UNBOUND flag.
4577                 * This is necessary as scheduler callbacks may be invoked
4578                 * from other cpus.
4579                 */
4580                schedule();
4581
4582                /*
4583                 * Sched callbacks are disabled now.  Zap nr_running.
4584                 * After this, nr_running stays zero and need_more_worker()
4585                 * and keep_working() are always true as long as the
4586                 * worklist is not empty.  This pool now behaves as an
4587                 * unbound (in terms of concurrency management) pool which
4588                 * are served by workers tied to the pool.
4589                 */
4590                atomic_set(&pool->nr_running, 0);
4591
4592                /*
4593                 * With concurrency management just turned off, a busy
4594                 * worker blocking could lead to lengthy stalls.  Kick off
4595                 * unbound chain execution of currently pending work items.
4596                 */
4597                spin_lock_irq(&pool->lock);
4598                wake_up_worker(pool);
4599                spin_unlock_irq(&pool->lock);
4600        }
4601}
4602
4603/**
4604 * rebind_workers - rebind all workers of a pool to the associated CPU
4605 * @pool: pool of interest
4606 *
4607 * @pool->cpu is coming online.  Rebind all workers to the CPU.
4608 */
4609static void rebind_workers(struct worker_pool *pool)
4610{
4611        struct worker *worker;
4612        int wi;
4613
4614        lockdep_assert_held(&pool->manager_mutex);
4615
4616        /*
4617         * Restore CPU affinity of all workers.  As all idle workers should
4618         * be on the run-queue of the associated CPU before any local
4619         * wake-ups for concurrency management happen, restore CPU affinty
4620         * of all workers first and then clear UNBOUND.  As we're called
4621         * from CPU_ONLINE, the following shouldn't fail.
4622         */
4623        for_each_pool_worker(worker, wi, pool)
4624                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4625                                                  pool->attrs->cpumask) < 0);
4626
4627        spin_lock_irq(&pool->lock);
4628
4629        for_each_pool_worker(worker, wi, pool) {
4630                unsigned int worker_flags = worker->flags;
4631
4632                /*
4633                 * A bound idle worker should actually be on the runqueue
4634                 * of the associated CPU for local wake-ups targeting it to
4635                 * work.  Kick all idle workers so that they migrate to the
4636                 * associated CPU.  Doing this in the same loop as
4637                 * replacing UNBOUND with REBOUND is safe as no worker will
4638                 * be bound before @pool->lock is released.
4639                 */
4640                if (worker_flags & WORKER_IDLE)
4641                        wake_up_process(worker->task);
4642
4643                /*
4644                 * We want to clear UNBOUND but can't directly call
4645                 * worker_clr_flags() or adjust nr_running.  Atomically
4646                 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
4647                 * @worker will clear REBOUND using worker_clr_flags() when
4648                 * it initiates the next execution cycle thus restoring
4649                 * concurrency management.  Note that when or whether
4650                 * @worker clears REBOUND doesn't affect correctness.
4651                 *
4652                 * ACCESS_ONCE() is necessary because @worker->flags may be
4653                 * tested without holding any lock in
4654                 * wq_worker_waking_up().  Without it, NOT_RUNNING test may
4655                 * fail incorrectly leading to premature concurrency
4656                 * management operations.
4657                 */
4658                WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4659                worker_flags |= WORKER_REBOUND;
4660                worker_flags &= ~WORKER_UNBOUND;
4661                ACCESS_ONCE(worker->flags) = worker_flags;
4662        }
4663
4664        spin_unlock_irq(&pool->lock);
4665}
4666
4667/**
4668 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
4669 * @pool: unbound pool of interest
4670 * @cpu: the CPU which is coming up
4671 *
4672 * An unbound pool may end up with a cpumask which doesn't have any online
4673 * CPUs.  When a worker of such pool get scheduled, the scheduler resets
4674 * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any
4675 * online CPU before, cpus_allowed of all its workers should be restored.
4676 */
4677static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4678{
4679        static cpumask_t cpumask;
4680        struct worker *worker;
4681        int wi;
4682
4683        lockdep_assert_held(&pool->manager_mutex);
4684
4685        /* is @cpu allowed for @pool? */
4686        if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4687                return;
4688
4689        /* is @cpu the only online CPU? */
4690        cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4691        if (cpumask_weight(&cpumask) != 1)
4692                return;
4693
4694        /* as we're called from CPU_ONLINE, the following shouldn't fail */
4695        for_each_pool_worker(worker, wi, pool)
4696                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4697                                                  pool->attrs->cpumask) < 0);
4698}
4699
4700/*
4701 * Workqueues should be brought up before normal priority CPU notifiers.
4702 * This will be registered high priority CPU notifier.
4703 */
4704static int workqueue_cpu_up_callback(struct notifier_block *nfb,
4705                                               unsigned long action,
4706                                               void *hcpu)
4707{
4708        int cpu = (unsigned long)hcpu;
4709        struct worker_pool *pool;
4710        struct workqueue_struct *wq;
4711        int pi;
4712
4713        switch (action & ~CPU_TASKS_FROZEN) {
4714        case CPU_UP_PREPARE:
4715                for_each_cpu_worker_pool(pool, cpu) {
4716                        if (pool->nr_workers)
4717                                continue;
4718                        if (create_and_start_worker(pool) < 0)
4719                                return NOTIFY_BAD;
4720                }
4721                break;
4722
4723        case CPU_DOWN_FAILED:
4724        case CPU_ONLINE:
4725                mutex_lock(&wq_pool_mutex);
4726
4727                for_each_pool(pool, pi) {
4728                        mutex_lock(&pool->manager_mutex);
4729
4730                        if (pool->cpu == cpu) {
4731                                spin_lock_irq(&pool->lock);
4732                                pool->flags &= ~POOL_DISASSOCIATED;
4733                                spin_unlock_irq(&pool->lock);
4734
4735                                rebind_workers(pool);
4736                        } else if (pool->cpu < 0) {
4737                                restore_unbound_workers_cpumask(pool, cpu);
4738                        }
4739
4740                        mutex_unlock(&pool->manager_mutex);
4741                }
4742
4743                /* update NUMA affinity of unbound workqueues */
4744                list_for_each_entry(wq, &workqueues, list)
4745                        wq_update_unbound_numa(wq, cpu, true);
4746
4747                mutex_unlock(&wq_pool_mutex);
4748                break;
4749        }
4750        return NOTIFY_OK;
4751}
4752
4753/*
4754 * Workqueues should be brought down after normal priority CPU notifiers.
4755 * This will be registered as low priority CPU notifier.
4756 */
4757static int workqueue_cpu_down_callback(struct notifier_block *nfb,
4758                                                 unsigned long action,
4759                                                 void *hcpu)
4760{
4761        int cpu = (unsigned long)hcpu;
4762        struct work_struct unbind_work;
4763        struct workqueue_struct *wq;
4764
4765        switch (action & ~CPU_TASKS_FROZEN) {
4766        case CPU_DOWN_PREPARE:
4767                /* unbinding per-cpu workers should happen on the local CPU */
4768                INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
4769                queue_work_on(cpu, system_highpri_wq, &unbind_work);
4770
4771                /* update NUMA affinity of unbound workqueues */
4772                mutex_lock(&wq_pool_mutex);
4773                list_for_each_entry(wq, &workqueues, list)
4774                        wq_update_unbound_numa(wq, cpu, false);
4775                mutex_unlock(&wq_pool_mutex);
4776
4777                /* wait for per-cpu unbinding to finish */
4778                flush_work(&unbind_work);
4779                break;
4780        }
4781        return NOTIFY_OK;
4782}
4783
4784#ifdef CONFIG_SMP
4785
4786struct work_for_cpu {
4787        struct work_struct work;
4788        long (*fn)(void *);
4789        void *arg;
4790        long ret;
4791};
4792
4793static void work_for_cpu_fn(struct work_struct *work)
4794{
4795        struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4796
4797        wfc->ret = wfc->fn(wfc->arg);
4798}
4799
4800/**
4801 * work_on_cpu - run a function in user context on a particular cpu
4802 * @cpu: the cpu to run on
4803 * @fn: the function to run
4804 * @arg: the function arg
4805 *
4806 * It is up to the caller to ensure that the cpu doesn't go offline.
4807 * The caller must not hold any locks which would prevent @fn from completing.
4808 *
4809 * Return: The value @fn returns.
4810 */
4811long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
4812{
4813        struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4814
4815        INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4816        schedule_work_on(cpu, &wfc.work);
4817
4818        /*
4819         * The work item is on-stack and can't lead to deadlock through
4820         * flushing.  Use __flush_work() to avoid spurious lockdep warnings
4821         * when work_on_cpu()s are nested.
4822         */
4823        __flush_work(&wfc.work);
4824
4825        return wfc.ret;
4826}
4827EXPORT_SYMBOL_GPL(work_on_cpu);
4828#endif /* CONFIG_SMP */
4829
4830#ifdef CONFIG_FREEZER
4831
4832/**
4833 * freeze_workqueues_begin - begin freezing workqueues
4834 *
4835 * Start freezing workqueues.  After this function returns, all freezable
4836 * workqueues will queue new works to their delayed_works list instead of
4837 * pool->worklist.
4838 *
4839 * CONTEXT:
4840 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4841 */
4842void freeze_workqueues_begin(void)
4843{
4844        struct worker_pool *pool;
4845        struct workqueue_struct *wq;
4846        struct pool_workqueue *pwq;
4847        int pi;
4848
4849        mutex_lock(&wq_pool_mutex);
4850
4851        WARN_ON_ONCE(workqueue_freezing);
4852        workqueue_freezing = true;
4853
4854        /* set FREEZING */
4855        for_each_pool(pool, pi) {
4856                spin_lock_irq(&pool->lock);
4857                WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4858                pool->flags |= POOL_FREEZING;
4859                spin_unlock_irq(&pool->lock);
4860        }
4861
4862        list_for_each_entry(wq, &workqueues, list) {
4863                mutex_lock(&wq->mutex);
4864                for_each_pwq(pwq, wq)
4865                        pwq_adjust_max_active(pwq);
4866                mutex_unlock(&wq->mutex);
4867        }
4868
4869        mutex_unlock(&wq_pool_mutex);
4870}
4871
4872/**
4873 * freeze_workqueues_busy - are freezable workqueues still busy?
4874 *
4875 * Check whether freezing is complete.  This function must be called
4876 * between freeze_workqueues_begin() and thaw_workqueues().
4877 *
4878 * CONTEXT:
4879 * Grabs and releases wq_pool_mutex.
4880 *
4881 * Return:
4882 * %true if some freezable workqueues are still busy.  %false if freezing
4883 * is complete.
4884 */
4885bool freeze_workqueues_busy(void)
4886{
4887        bool busy = false;
4888        struct workqueue_struct *wq;
4889        struct pool_workqueue *pwq;
4890
4891        mutex_lock(&wq_pool_mutex);
4892
4893        WARN_ON_ONCE(!workqueue_freezing);
4894
4895        list_for_each_entry(wq, &workqueues, list) {
4896                if (!(wq->flags & WQ_FREEZABLE))
4897                        continue;
4898                /*
4899                 * nr_active is monotonically decreasing.  It's safe
4900                 * to peek without lock.
4901                 */
4902                rcu_read_lock_sched();
4903                for_each_pwq(pwq, wq) {
4904                        WARN_ON_ONCE(pwq->nr_active < 0);
4905                        if (pwq->nr_active) {
4906                                busy = true;
4907                                rcu_read_unlock_sched();
4908                                goto out_unlock;
4909                        }
4910                }
4911                rcu_read_unlock_sched();
4912        }
4913out_unlock:
4914        mutex_unlock(&wq_pool_mutex);
4915        return busy;
4916}
4917
4918/**
4919 * thaw_workqueues - thaw workqueues
4920 *
4921 * Thaw workqueues.  Normal queueing is restored and all collected
4922 * frozen works are transferred to their respective pool worklists.
4923 *
4924 * CONTEXT:
4925 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4926 */
4927void thaw_workqueues(void)
4928{
4929        struct workqueue_struct *wq;
4930        struct pool_workqueue *pwq;
4931        struct worker_pool *pool;
4932        int pi;
4933
4934        mutex_lock(&wq_pool_mutex);
4935
4936        if (!workqueue_freezing)
4937                goto out_unlock;
4938
4939        /* clear FREEZING */
4940        for_each_pool(pool, pi) {
4941                spin_lock_irq(&pool->lock);
4942                WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
4943                pool->flags &= ~POOL_FREEZING;
4944                spin_unlock_irq(&pool->lock);
4945        }
4946
4947        /* restore max_active and repopulate worklist */
4948        list_for_each_entry(wq, &workqueues, list) {
4949                mutex_lock(&wq->mutex);
4950                for_each_pwq(pwq, wq)
4951                        pwq_adjust_max_active(pwq);
4952                mutex_unlock(&wq->mutex);
4953        }
4954
4955        workqueue_freezing = false;
4956out_unlock:
4957        mutex_unlock(&wq_pool_mutex);
4958}
4959#endif /* CONFIG_FREEZER */
4960
4961static void __init wq_numa_init(void)
4962{
4963        cpumask_var_t *tbl;
4964        int node, cpu;
4965
4966        /* determine NUMA pwq table len - highest node id + 1 */
4967        for_each_node(node)
4968                wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
4969
4970        if (num_possible_nodes() <= 1)
4971                return;
4972
4973        if (wq_disable_numa) {
4974                pr_info("workqueue: NUMA affinity support disabled\n");
4975                return;
4976        }
4977
4978        wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
4979        BUG_ON(!wq_update_unbound_numa_attrs_buf);
4980
4981        /*
4982         * We want masks of possible CPUs of each node which isn't readily
4983         * available.  Build one from cpu_to_node() which should have been
4984         * fully initialized by now.
4985         */
4986        tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
4987        BUG_ON(!tbl);
4988
4989        for_each_node(node)
4990                BUG_ON(!alloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
4991                                node_online(node) ? node : NUMA_NO_NODE));
4992
4993        for_each_possible_cpu(cpu) {
4994                node = cpu_to_node(cpu);
4995                if (WARN_ON(node == NUMA_NO_NODE)) {
4996                        pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
4997                        /* happens iff arch is bonkers, let's just proceed */
4998                        return;
4999                }
5000                cpumask_set_cpu(cpu, tbl[node]);
5001        }
5002
5003        wq_numa_possible_cpumask = tbl;
5004        wq_numa_enabled = true;
5005}
5006
5007static int __init init_workqueues(void)
5008{
5009        int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5010        int i, cpu;
5011
5012        /* make sure we have enough bits for OFFQ pool ID */
5013        BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
5014                     WORK_CPU_END * NR_STD_WORKER_POOLS);
5015
5016        WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5017
5018        pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
5019
5020        cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
5021        hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
5022
5023        wq_numa_init();
5024
5025        /* initialize CPU pools */
5026        for_each_possible_cpu(cpu) {
5027                struct worker_pool *pool;
5028
5029                i = 0;
5030                for_each_cpu_worker_pool(pool, cpu) {
5031                        BUG_ON(init_worker_pool(pool));
5032                        pool->cpu = cpu;
5033                        cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
5034                        pool->attrs->nice = std_nice[i++];
5035                        pool->node = cpu_to_node(cpu);
5036
5037                        /* alloc pool ID */
5038                        mutex_lock(&wq_pool_mutex);
5039                        BUG_ON(worker_pool_assign_id(pool));
5040                        mutex_unlock(&wq_pool_mutex);
5041                }
5042        }
5043
5044        /* create the initial worker */
5045        for_each_online_cpu(cpu) {
5046                struct worker_pool *pool;
5047
5048                for_each_cpu_worker_pool(pool, cpu) {
5049                        pool->flags &= ~POOL_DISASSOCIATED;
5050                        BUG_ON(create_and_start_worker(pool) < 0);
5051                }
5052        }
5053
5054        /* create default unbound wq attrs */
5055        for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5056                struct workqueue_attrs *attrs;
5057
5058                BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5059                attrs->nice = std_nice[i];
5060                unbound_std_wq_attrs[i] = attrs;
5061        }
5062
5063        system_wq = alloc_workqueue("events", 0, 0);
5064        system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
5065        system_long_wq = alloc_workqueue("events_long", 0, 0);
5066        system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5067                                            WQ_UNBOUND_MAX_ACTIVE);
5068        system_freezable_wq = alloc_workqueue("events_freezable",
5069                                              WQ_FREEZABLE, 0);
5070        system_power_efficient_wq = alloc_workqueue("events_power_efficient",
5071                                              WQ_POWER_EFFICIENT, 0);
5072        system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
5073                                              WQ_FREEZABLE | WQ_POWER_EFFICIENT,
5074                                              0);
5075        BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
5076               !system_unbound_wq || !system_freezable_wq ||
5077               !system_power_efficient_wq ||
5078               !system_freezable_power_efficient_wq);
5079        return 0;
5080}
5081early_initcall(init_workqueues);
5082
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.