1/* 2 * Read-Copy Update mechanism for mutual exclusion (tree-based version) 3 * Internal non-public definitions. 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * Copyright IBM Corporation, 2008 20 * 21 * Author: Ingo Molnar <mingo@elte.hu> 22 * Paul E. McKenney <paulmck@linux.vnet.ibm.com> 23 */ 24 25#include <linux/cache.h> 26#include <linux/spinlock.h> 27#include <linux/threads.h> 28#include <linux/cpumask.h> 29#include <linux/seqlock.h> 30 31/* 32 * Define shape of hierarchy based on NR_CPUS, CONFIG_RCU_FANOUT, and 33 * CONFIG_RCU_FANOUT_LEAF. 34 * In theory, it should be possible to add more levels straightforwardly. 35 * In practice, this did work well going from three levels to four. 36 * Of course, your mileage may vary. 37 */ 38#define MAX_RCU_LVLS 4 39#define RCU_FANOUT_1 (CONFIG_RCU_FANOUT_LEAF) 40#define RCU_FANOUT_2 (RCU_FANOUT_1 * CONFIG_RCU_FANOUT) 41#define RCU_FANOUT_3 (RCU_FANOUT_2 * CONFIG_RCU_FANOUT) 42#define RCU_FANOUT_4 (RCU_FANOUT_3 * CONFIG_RCU_FANOUT) 43 44#if NR_CPUS <= RCU_FANOUT_1 45# define RCU_NUM_LVLS 1 46# define NUM_RCU_LVL_0 1 47# define NUM_RCU_LVL_1 (NR_CPUS) 48# define NUM_RCU_LVL_2 0 49# define NUM_RCU_LVL_3 0 50# define NUM_RCU_LVL_4 0 51#elif NR_CPUS <= RCU_FANOUT_2 52# define RCU_NUM_LVLS 2 53# define NUM_RCU_LVL_0 1 54# define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_1) 55# define NUM_RCU_LVL_2 (NR_CPUS) 56# define NUM_RCU_LVL_3 0 57# define NUM_RCU_LVL_4 0 58#elif NR_CPUS <= RCU_FANOUT_3 59# define RCU_NUM_LVLS 3 60# define NUM_RCU_LVL_0 1 61# define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_2) 62# define NUM_RCU_LVL_2 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_1) 63# define NUM_RCU_LVL_3 (NR_CPUS) 64# define NUM_RCU_LVL_4 0 65#elif NR_CPUS <= RCU_FANOUT_4 66# define RCU_NUM_LVLS 4 67# define NUM_RCU_LVL_0 1 68# define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_3) 69# define NUM_RCU_LVL_2 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_2) 70# define NUM_RCU_LVL_3 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_1) 71# define NUM_RCU_LVL_4 (NR_CPUS) 72#else 73# error "CONFIG_RCU_FANOUT insufficient for NR_CPUS" 74#endif /* #if (NR_CPUS) <= RCU_FANOUT_1 */ 75 76#define RCU_SUM (NUM_RCU_LVL_0 + NUM_RCU_LVL_1 + NUM_RCU_LVL_2 + NUM_RCU_LVL_3 + NUM_RCU_LVL_4) 77#define NUM_RCU_NODES (RCU_SUM - NR_CPUS) 78 79extern int rcu_num_lvls; 80extern int rcu_num_nodes; 81 82/* 83 * Dynticks per-CPU state. 84 */ 85struct rcu_dynticks { 86 long long dynticks_nesting; /* Track irq/process nesting level. */ 87 /* Process level is worth LLONG_MAX/2. */ 88 int dynticks_nmi_nesting; /* Track NMI nesting level. */ 89 atomic_t dynticks; /* Even value for idle, else odd. */ 90#ifdef CONFIG_RCU_FAST_NO_HZ 91 int dyntick_drain; /* Prepare-for-idle state variable. */ 92 unsigned long dyntick_holdoff; 93 /* No retries for the jiffy of failure. */ 94 struct timer_list idle_gp_timer; 95 /* Wake up CPU sleeping with callbacks. */ 96 unsigned long idle_gp_timer_expires; 97 /* When to wake up CPU (for repost). */ 98 bool idle_first_pass; /* First pass of attempt to go idle? */ 99 unsigned long nonlazy_posted; 100 /* # times non-lazy CBs posted to CPU. */ 101 unsigned long nonlazy_posted_snap; 102 /* idle-period nonlazy_posted snapshot. */ 103 int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */ 104#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ 105#ifdef CONFIG_RCU_USER_QS 106 bool ignore_user_qs; /* Treat userspace as extended QS or not */ 107 bool in_user; /* Is the CPU in userland from RCU POV? */ 108#endif 109}; 110 111/* RCU's kthread states for tracing. */ 112#define RCU_KTHREAD_STOPPED 0 113#define RCU_KTHREAD_RUNNING 1 114#define RCU_KTHREAD_WAITING 2 115#define RCU_KTHREAD_OFFCPU 3 116#define RCU_KTHREAD_YIELDING 4 117#define RCU_KTHREAD_MAX 4 118 119/* 120 * Definition for node within the RCU grace-period-detection hierarchy. 121 */ 122struct rcu_node { 123 raw_spinlock_t lock; /* Root rcu_node's lock protects some */ 124 /* rcu_state fields as well as following. */ 125 unsigned long gpnum; /* Current grace period for this node. */ 126 /* This will either be equal to or one */ 127 /* behind the root rcu_node's gpnum. */ 128 unsigned long completed; /* Last GP completed for this node. */ 129 /* This will either be equal to or one */ 130 /* behind the root rcu_node's gpnum. */ 131 unsigned long qsmask; /* CPUs or groups that need to switch in */ 132 /* order for current grace period to proceed.*/ 133 /* In leaf rcu_node, each bit corresponds to */ 134 /* an rcu_data structure, otherwise, each */ 135 /* bit corresponds to a child rcu_node */ 136 /* structure. */ 137 unsigned long expmask; /* Groups that have ->blkd_tasks */ 138 /* elements that need to drain to allow the */ 139 /* current expedited grace period to */ 140 /* complete (only for TREE_PREEMPT_RCU). */ 141 atomic_t wakemask; /* CPUs whose kthread needs to be awakened. */ 142 /* Since this has meaning only for leaf */ 143 /* rcu_node structures, 32 bits suffices. */ 144 unsigned long qsmaskinit; 145 /* Per-GP initial value for qsmask & expmask. */ 146 unsigned long grpmask; /* Mask to apply to parent qsmask. */ 147 /* Only one bit will be set in this mask. */ 148 int grplo; /* lowest-numbered CPU or group here. */ 149 int grphi; /* highest-numbered CPU or group here. */ 150 u8 grpnum; /* CPU/group number for next level up. */ 151 u8 level; /* root is at level 0. */ 152 struct rcu_node *parent; 153 struct list_head blkd_tasks; 154 /* Tasks blocked in RCU read-side critical */ 155 /* section. Tasks are placed at the head */ 156 /* of this list and age towards the tail. */ 157 struct list_head *gp_tasks; 158 /* Pointer to the first task blocking the */ 159 /* current grace period, or NULL if there */ 160 /* is no such task. */ 161 struct list_head *exp_tasks; 162 /* Pointer to the first task blocking the */ 163 /* current expedited grace period, or NULL */ 164 /* if there is no such task. If there */ 165 /* is no current expedited grace period, */ 166 /* then there can cannot be any such task. */ 167#ifdef CONFIG_RCU_BOOST 168 struct list_head *boost_tasks; 169 /* Pointer to first task that needs to be */ 170 /* priority boosted, or NULL if no priority */ 171 /* boosting is needed for this rcu_node */ 172 /* structure. If there are no tasks */ 173 /* queued on this rcu_node structure that */ 174 /* are blocking the current grace period, */ 175 /* there can be no such task. */ 176 unsigned long boost_time; 177 /* When to start boosting (jiffies). */ 178 struct task_struct *boost_kthread_task; 179 /* kthread that takes care of priority */ 180 /* boosting for this rcu_node structure. */ 181 unsigned int boost_kthread_status; 182 /* State of boost_kthread_task for tracing. */ 183 unsigned long n_tasks_boosted; 184 /* Total number of tasks boosted. */ 185 unsigned long n_exp_boosts; 186 /* Number of tasks boosted for expedited GP. */ 187 unsigned long n_normal_boosts; 188 /* Number of tasks boosted for normal GP. */ 189 unsigned long n_balk_blkd_tasks; 190 /* Refused to boost: no blocked tasks. */ 191 unsigned long n_balk_exp_gp_tasks; 192 /* Refused to boost: nothing blocking GP. */ 193 unsigned long n_balk_boost_tasks; 194 /* Refused to boost: already boosting. */ 195 unsigned long n_balk_notblocked; 196 /* Refused to boost: RCU RS CS still running. */ 197 unsigned long n_balk_notyet; 198 /* Refused to boost: not yet time. */ 199 unsigned long n_balk_nos; 200 /* Refused to boost: not sure why, though. */ 201 /* This can happen due to race conditions. */ 202#endif /* #ifdef CONFIG_RCU_BOOST */ 203 raw_spinlock_t fqslock ____cacheline_internodealigned_in_smp; 204} ____cacheline_internodealigned_in_smp; 205 206/* 207 * Do a full breadth-first scan of the rcu_node structures for the 208 * specified rcu_state structure. 209 */ 210#define rcu_for_each_node_breadth_first(rsp, rnp) \ 211 for ((rnp) = &(rsp)->node[0]; \ 212 (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++) 213 214/* 215 * Do a breadth-first scan of the non-leaf rcu_node structures for the 216 * specified rcu_state structure. Note that if there is a singleton 217 * rcu_node tree with but one rcu_node structure, this loop is a no-op. 218 */ 219#define rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) \ 220 for ((rnp) = &(rsp)->node[0]; \ 221 (rnp) < (rsp)->level[rcu_num_lvls - 1]; (rnp)++) 222 223/* 224 * Scan the leaves of the rcu_node hierarchy for the specified rcu_state 225 * structure. Note that if there is a singleton rcu_node tree with but 226 * one rcu_node structure, this loop -will- visit the rcu_node structure. 227 * It is still a leaf node, even if it is also the root node. 228 */ 229#define rcu_for_each_leaf_node(rsp, rnp) \ 230 for ((rnp) = (rsp)->level[rcu_num_lvls - 1]; \ 231 (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++) 232 233/* Index values for nxttail array in struct rcu_data. */ 234#define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */ 235#define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */ 236#define RCU_NEXT_READY_TAIL 2 /* Also RCU_NEXT head. */ 237#define RCU_NEXT_TAIL 3 238#define RCU_NEXT_SIZE 4 239 240/* Per-CPU data for read-copy update. */ 241struct rcu_data { 242 /* 1) quiescent-state and grace-period handling : */ 243 unsigned long completed; /* Track rsp->completed gp number */ 244 /* in order to detect GP end. */ 245 unsigned long gpnum; /* Highest gp number that this CPU */ 246 /* is aware of having started. */ 247 bool passed_quiesce; /* User-mode/idle loop etc. */ 248 bool qs_pending; /* Core waits for quiesc state. */ 249 bool beenonline; /* CPU online at least once. */ 250 bool preemptible; /* Preemptible RCU? */ 251 struct rcu_node *mynode; /* This CPU's leaf of hierarchy */ 252 unsigned long grpmask; /* Mask to apply to leaf qsmask. */ 253#ifdef CONFIG_RCU_CPU_STALL_INFO 254 unsigned long ticks_this_gp; /* The number of scheduling-clock */ 255 /* ticks this CPU has handled */ 256 /* during and after the last grace */ 257 /* period it is aware of. */ 258#endif /* #ifdef CONFIG_RCU_CPU_STALL_INFO */ 259 260 /* 2) batch handling */ 261 /* 262 * If nxtlist is not NULL, it is partitioned as follows. 263 * Any of the partitions might be empty, in which case the 264 * pointer to that partition will be equal to the pointer for 265 * the following partition. When the list is empty, all of 266 * the nxttail elements point to the ->nxtlist pointer itself, 267 * which in that case is NULL. 268 * 269 * [nxtlist, *nxttail[RCU_DONE_TAIL]): 270 * Entries that batch # <= ->completed 271 * The grace period for these entries has completed, and 272 * the other grace-period-completed entries may be moved 273 * here temporarily in rcu_process_callbacks(). 274 * [*nxttail[RCU_DONE_TAIL], *nxttail[RCU_WAIT_TAIL]): 275 * Entries that batch # <= ->completed - 1: waiting for current GP 276 * [*nxttail[RCU_WAIT_TAIL], *nxttail[RCU_NEXT_READY_TAIL]): 277 * Entries known to have arrived before current GP ended 278 * [*nxttail[RCU_NEXT_READY_TAIL], *nxttail[RCU_NEXT_TAIL]): 279 * Entries that might have arrived after current GP ended 280 * Note that the value of *nxttail[RCU_NEXT_TAIL] will 281 * always be NULL, as this is the end of the list. 282 */ 283 struct rcu_head *nxtlist; 284 struct rcu_head **nxttail[RCU_NEXT_SIZE]; 285 long qlen_lazy; /* # of lazy queued callbacks */ 286 long qlen; /* # of queued callbacks, incl lazy */ 287 long qlen_last_fqs_check; 288 /* qlen at last check for QS forcing */ 289 unsigned long n_cbs_invoked; /* count of RCU cbs invoked. */ 290 unsigned long n_cbs_orphaned; /* RCU cbs orphaned by dying CPU */ 291 unsigned long n_cbs_adopted; /* RCU cbs adopted from dying CPU */ 292 unsigned long n_force_qs_snap; 293 /* did other CPU force QS recently? */ 294 long blimit; /* Upper limit on a processed batch */ 295 296 /* 3) dynticks interface. */ 297 struct rcu_dynticks *dynticks; /* Shared per-CPU dynticks state. */ 298 int dynticks_snap; /* Per-GP tracking for dynticks. */ 299 300 /* 4) reasons this CPU needed to be kicked by force_quiescent_state */ 301 unsigned long dynticks_fqs; /* Kicked due to dynticks idle. */ 302 unsigned long offline_fqs; /* Kicked due to being offline. */ 303 304 /* 5) __rcu_pending() statistics. */ 305 unsigned long n_rcu_pending; /* rcu_pending() calls since boot. */ 306 unsigned long n_rp_qs_pending; 307 unsigned long n_rp_report_qs; 308 unsigned long n_rp_cb_ready; 309 unsigned long n_rp_cpu_needs_gp; 310 unsigned long n_rp_gp_completed; 311 unsigned long n_rp_gp_started; 312 unsigned long n_rp_need_nothing; 313 314 /* 6) _rcu_barrier() and OOM callbacks. */ 315 struct rcu_head barrier_head; 316#ifdef CONFIG_RCU_FAST_NO_HZ 317 struct rcu_head oom_head; 318#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ 319 320 int cpu; 321 struct rcu_state *rsp; 322}; 323 324/* Values for fqs_state field in struct rcu_state. */ 325#define RCU_GP_IDLE 0 /* No grace period in progress. */ 326#define RCU_GP_INIT 1 /* Grace period being initialized. */ 327#define RCU_SAVE_DYNTICK 2 /* Need to scan dyntick state. */ 328#define RCU_FORCE_QS 3 /* Need to force quiescent state. */ 329#define RCU_SIGNAL_INIT RCU_SAVE_DYNTICK 330 331#define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */ 332 333#ifdef CONFIG_PROVE_RCU 334#define RCU_STALL_DELAY_DELTA (5 * HZ) 335#else 336#define RCU_STALL_DELAY_DELTA 0 337#endif 338#define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time */ 339 /* to take at least one */ 340 /* scheduling clock irq */ 341 /* before ratting on them. */ 342 343#define rcu_wait(cond) \ 344do { \ 345 for (;;) { \ 346 set_current_state(TASK_INTERRUPTIBLE); \ 347 if (cond) \ 348 break; \ 349 schedule(); \ 350 } \ 351 __set_current_state(TASK_RUNNING); \ 352} while (0) 353 354/* 355 * RCU global state, including node hierarchy. This hierarchy is 356 * represented in "heap" form in a dense array. The root (first level) 357 * of the hierarchy is in ->node[0] (referenced by ->level[0]), the second 358 * level in ->node[1] through ->node[m] (->node[1] referenced by ->level[1]), 359 * and the third level in ->node[m+1] and following (->node[m+1] referenced 360 * by ->level[2]). The number of levels is determined by the number of 361 * CPUs and by CONFIG_RCU_FANOUT. Small systems will have a "hierarchy" 362 * consisting of a single rcu_node. 363 */ 364struct rcu_state { 365 struct rcu_node node[NUM_RCU_NODES]; /* Hierarchy. */ 366 struct rcu_node *level[RCU_NUM_LVLS]; /* Hierarchy levels. */ 367 u32 levelcnt[MAX_RCU_LVLS + 1]; /* # nodes in each level. */ 368 u8 levelspread[RCU_NUM_LVLS]; /* kids/node in each level. */ 369 struct rcu_data __percpu *rda; /* pointer of percu rcu_data. */ 370 void (*call)(struct rcu_head *head, /* call_rcu() flavor. */ 371 void (*func)(struct rcu_head *head)); 372 373 /* The following fields are guarded by the root rcu_node's lock. */ 374 375 u8 fqs_state ____cacheline_internodealigned_in_smp; 376 /* Force QS state. */ 377 u8 boost; /* Subject to priority boost. */ 378 unsigned long gpnum; /* Current gp number. */ 379 unsigned long completed; /* # of last completed gp. */ 380 struct task_struct *gp_kthread; /* Task for grace periods. */ 381 wait_queue_head_t gp_wq; /* Where GP task waits. */ 382 int gp_flags; /* Commands for GP task. */ 383 384 /* End of fields guarded by root rcu_node's lock. */ 385 386 raw_spinlock_t onofflock ____cacheline_internodealigned_in_smp; 387 /* exclude on/offline and */ 388 /* starting new GP. */ 389 struct rcu_head *orphan_nxtlist; /* Orphaned callbacks that */ 390 /* need a grace period. */ 391 struct rcu_head **orphan_nxttail; /* Tail of above. */ 392 struct rcu_head *orphan_donelist; /* Orphaned callbacks that */ 393 /* are ready to invoke. */ 394 struct rcu_head **orphan_donetail; /* Tail of above. */ 395 long qlen_lazy; /* Number of lazy callbacks. */ 396 long qlen; /* Total number of callbacks. */ 397 /* End of fields guarded by onofflock. */ 398 399 struct mutex onoff_mutex; /* Coordinate hotplug & GPs. */ 400 401 struct mutex barrier_mutex; /* Guards barrier fields. */ 402 atomic_t barrier_cpu_count; /* # CPUs waiting on. */ 403 struct completion barrier_completion; /* Wake at barrier end. */ 404 unsigned long n_barrier_done; /* ++ at start and end of */ 405 /* _rcu_barrier(). */ 406 /* End of fields guarded by barrier_mutex. */ 407 408 unsigned long jiffies_force_qs; /* Time at which to invoke */ 409 /* force_quiescent_state(). */ 410 unsigned long n_force_qs; /* Number of calls to */ 411 /* force_quiescent_state(). */ 412 unsigned long n_force_qs_lh; /* ~Number of calls leaving */ 413 /* due to lock unavailable. */ 414 unsigned long n_force_qs_ngp; /* Number of calls leaving */ 415 /* due to no GP active. */ 416 unsigned long gp_start; /* Time at which GP started, */ 417 /* but in jiffies. */ 418 unsigned long jiffies_stall; /* Time at which to check */ 419 /* for CPU stalls. */ 420 unsigned long gp_max; /* Maximum GP duration in */ 421 /* jiffies. */ 422 char *name; /* Name of structure. */ 423 struct list_head flavors; /* List of RCU flavors. */ 424}; 425 426/* Values for rcu_state structure's gp_flags field. */ 427#define RCU_GP_FLAG_INIT 0x1 /* Need grace-period initialization. */ 428#define RCU_GP_FLAG_FQS 0x2 /* Need grace-period quiescent-state forcing. */ 429 430extern struct list_head rcu_struct_flavors; 431#define for_each_rcu_flavor(rsp) \ 432 list_for_each_entry((rsp), &rcu_struct_flavors, flavors) 433 434/* Return values for rcu_preempt_offline_tasks(). */ 435 436#define RCU_OFL_TASKS_NORM_GP 0x1 /* Tasks blocking normal */ 437 /* GP were moved to root. */ 438#define RCU_OFL_TASKS_EXP_GP 0x2 /* Tasks blocking expedited */ 439 /* GP were moved to root. */ 440 441/* 442 * RCU implementation internal declarations: 443 */ 444extern struct rcu_state rcu_sched_state; 445DECLARE_PER_CPU(struct rcu_data, rcu_sched_data); 446 447extern struct rcu_state rcu_bh_state; 448DECLARE_PER_CPU(struct rcu_data, rcu_bh_data); 449 450#ifdef CONFIG_TREE_PREEMPT_RCU 451extern struct rcu_state rcu_preempt_state; 452DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data); 453#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ 454 455#ifdef CONFIG_RCU_BOOST 456DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); 457DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu); 458DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); 459DECLARE_PER_CPU(char, rcu_cpu_has_work); 460#endif /* #ifdef CONFIG_RCU_BOOST */ 461 462#ifndef RCU_TREE_NONCORE 463 464/* Forward declarations for rcutree_plugin.h */ 465static void rcu_bootup_announce(void); 466long rcu_batches_completed(void); 467static void rcu_preempt_note_context_switch(int cpu); 468static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp); 469#ifdef CONFIG_HOTPLUG_CPU 470static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, 471 unsigned long flags); 472#endif /* #ifdef CONFIG_HOTPLUG_CPU */ 473static void rcu_print_detail_task_stall(struct rcu_state *rsp); 474static int rcu_print_task_stall(struct rcu_node *rnp); 475static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); 476#ifdef CONFIG_HOTPLUG_CPU 477static int rcu_preempt_offline_tasks(struct rcu_state *rsp, 478 struct rcu_node *rnp, 479 struct rcu_data *rdp); 480#endif /* #ifdef CONFIG_HOTPLUG_CPU */ 481static void rcu_preempt_check_callbacks(int cpu); 482void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 483#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) 484static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, 485 bool wake); 486#endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) */ 487static void __init __rcu_init_preempt(void); 488static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); 489static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); 490static void invoke_rcu_callbacks_kthread(void); 491static bool rcu_is_callbacks_kthread(void); 492#ifdef CONFIG_RCU_BOOST 493static void rcu_preempt_do_callbacks(void); 494static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, 495 struct rcu_node *rnp); 496#endif /* #ifdef CONFIG_RCU_BOOST */ 497static void __cpuinit rcu_prepare_kthreads(int cpu); 498static void rcu_prepare_for_idle_init(int cpu); 499static void rcu_cleanup_after_idle(int cpu); 500static void rcu_prepare_for_idle(int cpu); 501static void rcu_idle_count_callbacks_posted(void); 502static void print_cpu_stall_info_begin(void); 503static void print_cpu_stall_info(struct rcu_state *rsp, int cpu); 504static void print_cpu_stall_info_end(void); 505static void zero_cpu_stall_ticks(struct rcu_data *rdp); 506static void increment_cpu_stall_ticks(void); 507 508#endif /* #ifndef RCU_TREE_NONCORE */ 509