linux/kernel/srcu.c
<<
>>
Prefs
   1/*
   2 * Sleepable Read-Copy Update mechanism for mutual exclusion.
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License as published by
   6 * the Free Software Foundation; either version 2 of the License, or
   7 * (at your option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write to the Free Software
  16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17 *
  18 * Copyright (C) IBM Corporation, 2006
  19 *
  20 * Author: Paul McKenney <paulmck@us.ibm.com>
  21 *
  22 * For detailed explanation of Read-Copy Update mechanism see -
  23 *              Documentation/RCU/ *.txt
  24 *
  25 */
  26
  27#include <linux/export.h>
  28#include <linux/mutex.h>
  29#include <linux/percpu.h>
  30#include <linux/preempt.h>
  31#include <linux/rcupdate.h>
  32#include <linux/sched.h>
  33#include <linux/smp.h>
  34#include <linux/delay.h>
  35#include <linux/srcu.h>
  36
  37/*
  38 * Initialize an rcu_batch structure to empty.
  39 */
  40static inline void rcu_batch_init(struct rcu_batch *b)
  41{
  42        b->head = NULL;
  43        b->tail = &b->head;
  44}
  45
  46/*
  47 * Enqueue a callback onto the tail of the specified rcu_batch structure.
  48 */
  49static inline void rcu_batch_queue(struct rcu_batch *b, struct rcu_head *head)
  50{
  51        *b->tail = head;
  52        b->tail = &head->next;
  53}
  54
  55/*
  56 * Is the specified rcu_batch structure empty?
  57 */
  58static inline bool rcu_batch_empty(struct rcu_batch *b)
  59{
  60        return b->tail == &b->head;
  61}
  62
  63/*
  64 * Remove the callback at the head of the specified rcu_batch structure
  65 * and return a pointer to it, or return NULL if the structure is empty.
  66 */
  67static inline struct rcu_head *rcu_batch_dequeue(struct rcu_batch *b)
  68{
  69        struct rcu_head *head;
  70
  71        if (rcu_batch_empty(b))
  72                return NULL;
  73
  74        head = b->head;
  75        b->head = head->next;
  76        if (b->tail == &head->next)
  77                rcu_batch_init(b);
  78
  79        return head;
  80}
  81
  82/*
  83 * Move all callbacks from the rcu_batch structure specified by "from" to
  84 * the structure specified by "to".
  85 */
  86static inline void rcu_batch_move(struct rcu_batch *to, struct rcu_batch *from)
  87{
  88        if (!rcu_batch_empty(from)) {
  89                *to->tail = from->head;
  90                to->tail = from->tail;
  91                rcu_batch_init(from);
  92        }
  93}
  94
  95/* single-thread state-machine */
  96static void process_srcu(struct work_struct *work);
  97
  98static int init_srcu_struct_fields(struct srcu_struct *sp)
  99{
 100        sp->completed = 0;
 101        spin_lock_init(&sp->queue_lock);
 102        sp->running = false;
 103        rcu_batch_init(&sp->batch_queue);
 104        rcu_batch_init(&sp->batch_check0);
 105        rcu_batch_init(&sp->batch_check1);
 106        rcu_batch_init(&sp->batch_done);
 107        INIT_DELAYED_WORK(&sp->work, process_srcu);
 108        sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array);
 109        return sp->per_cpu_ref ? 0 : -ENOMEM;
 110}
 111
 112#ifdef CONFIG_DEBUG_LOCK_ALLOC
 113
 114int __init_srcu_struct(struct srcu_struct *sp, const char *name,
 115                       struct lock_class_key *key)
 116{
 117        /* Don't re-initialize a lock while it is held. */
 118        debug_check_no_locks_freed((void *)sp, sizeof(*sp));
 119        lockdep_init_map(&sp->dep_map, name, key, 0);
 120        return init_srcu_struct_fields(sp);
 121}
 122EXPORT_SYMBOL_GPL(__init_srcu_struct);
 123
 124#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 125
 126/**
 127 * init_srcu_struct - initialize a sleep-RCU structure
 128 * @sp: structure to initialize.
 129 *
 130 * Must invoke this on a given srcu_struct before passing that srcu_struct
 131 * to any other function.  Each srcu_struct represents a separate domain
 132 * of SRCU protection.
 133 */
 134int init_srcu_struct(struct srcu_struct *sp)
 135{
 136        return init_srcu_struct_fields(sp);
 137}
 138EXPORT_SYMBOL_GPL(init_srcu_struct);
 139
 140#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 141
 142/*
 143 * Returns approximate total of the readers' ->seq[] values for the
 144 * rank of per-CPU counters specified by idx.
 145 */
 146static unsigned long srcu_readers_seq_idx(struct srcu_struct *sp, int idx)
 147{
 148        int cpu;
 149        unsigned long sum = 0;
 150        unsigned long t;
 151
 152        for_each_possible_cpu(cpu) {
 153                t = ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->seq[idx]);
 154                sum += t;
 155        }
 156        return sum;
 157}
 158
 159/*
 160 * Returns approximate number of readers active on the specified rank
 161 * of the per-CPU ->c[] counters.
 162 */
 163static unsigned long srcu_readers_active_idx(struct srcu_struct *sp, int idx)
 164{
 165        int cpu;
 166        unsigned long sum = 0;
 167        unsigned long t;
 168
 169        for_each_possible_cpu(cpu) {
 170                t = ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[idx]);
 171                sum += t;
 172        }
 173        return sum;
 174}
 175
 176/*
 177 * Return true if the number of pre-existing readers is determined to
 178 * be stably zero.  An example unstable zero can occur if the call
 179 * to srcu_readers_active_idx() misses an __srcu_read_lock() increment,
 180 * but due to task migration, sees the corresponding __srcu_read_unlock()
 181 * decrement.  This can happen because srcu_readers_active_idx() takes
 182 * time to sum the array, and might in fact be interrupted or preempted
 183 * partway through the summation.
 184 */
 185static bool srcu_readers_active_idx_check(struct srcu_struct *sp, int idx)
 186{
 187        unsigned long seq;
 188
 189        seq = srcu_readers_seq_idx(sp, idx);
 190
 191        /*
 192         * The following smp_mb() A pairs with the smp_mb() B located in
 193         * __srcu_read_lock().  This pairing ensures that if an
 194         * __srcu_read_lock() increments its counter after the summation
 195         * in srcu_readers_active_idx(), then the corresponding SRCU read-side
 196         * critical section will see any changes made prior to the start
 197         * of the current SRCU grace period.
 198         *
 199         * Also, if the above call to srcu_readers_seq_idx() saw the
 200         * increment of ->seq[], then the call to srcu_readers_active_idx()
 201         * must see the increment of ->c[].
 202         */
 203        smp_mb(); /* A */
 204
 205        /*
 206         * Note that srcu_readers_active_idx() can incorrectly return
 207         * zero even though there is a pre-existing reader throughout.
 208         * To see this, suppose that task A is in a very long SRCU
 209         * read-side critical section that started on CPU 0, and that
 210         * no other reader exists, so that the sum of the counters
 211         * is equal to one.  Then suppose that task B starts executing
 212         * srcu_readers_active_idx(), summing up to CPU 1, and then that
 213         * task C starts reading on CPU 0, so that its increment is not
 214         * summed, but finishes reading on CPU 2, so that its decrement
 215         * -is- summed.  Then when task B completes its sum, it will
 216         * incorrectly get zero, despite the fact that task A has been
 217         * in its SRCU read-side critical section the whole time.
 218         *
 219         * We therefore do a validation step should srcu_readers_active_idx()
 220         * return zero.
 221         */
 222        if (srcu_readers_active_idx(sp, idx) != 0)
 223                return false;
 224
 225        /*
 226         * The remainder of this function is the validation step.
 227         * The following smp_mb() D pairs with the smp_mb() C in
 228         * __srcu_read_unlock().  If the __srcu_read_unlock() was seen
 229         * by srcu_readers_active_idx() above, then any destructive
 230         * operation performed after the grace period will happen after
 231         * the corresponding SRCU read-side critical section.
 232         *
 233         * Note that there can be at most NR_CPUS worth of readers using
 234         * the old index, which is not enough to overflow even a 32-bit
 235         * integer.  (Yes, this does mean that systems having more than
 236         * a billion or so CPUs need to be 64-bit systems.)  Therefore,
 237         * the sum of the ->seq[] counters cannot possibly overflow.
 238         * Therefore, the only way that the return values of the two
 239         * calls to srcu_readers_seq_idx() can be equal is if there were
 240         * no increments of the corresponding rank of ->seq[] counts
 241         * in the interim.  But the missed-increment scenario laid out
 242         * above includes an increment of the ->seq[] counter by
 243         * the corresponding __srcu_read_lock().  Therefore, if this
 244         * scenario occurs, the return values from the two calls to
 245         * srcu_readers_seq_idx() will differ, and thus the validation
 246         * step below suffices.
 247         */
 248        smp_mb(); /* D */
 249
 250        return srcu_readers_seq_idx(sp, idx) == seq;
 251}
 252
 253/**
 254 * srcu_readers_active - returns approximate number of readers.
 255 * @sp: which srcu_struct to count active readers (holding srcu_read_lock).
 256 *
 257 * Note that this is not an atomic primitive, and can therefore suffer
 258 * severe errors when invoked on an active srcu_struct.  That said, it
 259 * can be useful as an error check at cleanup time.
 260 */
 261static int srcu_readers_active(struct srcu_struct *sp)
 262{
 263        int cpu;
 264        unsigned long sum = 0;
 265
 266        for_each_possible_cpu(cpu) {
 267                sum += ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[0]);
 268                sum += ACCESS_ONCE(per_cpu_ptr(sp->per_cpu_ref, cpu)->c[1]);
 269        }
 270        return sum;
 271}
 272
 273/**
 274 * cleanup_srcu_struct - deconstruct a sleep-RCU structure
 275 * @sp: structure to clean up.
 276 *
 277 * Must invoke this after you are finished using a given srcu_struct that
 278 * was initialized via init_srcu_struct(), else you leak memory.
 279 */
 280void cleanup_srcu_struct(struct srcu_struct *sp)
 281{
 282        int sum;
 283
 284        sum = srcu_readers_active(sp);
 285        WARN_ON(sum);  /* Leakage unless caller handles error. */
 286        if (sum != 0)
 287                return;
 288        free_percpu(sp->per_cpu_ref);
 289        sp->per_cpu_ref = NULL;
 290}
 291EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
 292
 293/*
 294 * Counts the new reader in the appropriate per-CPU element of the
 295 * srcu_struct.  Must be called from process context.
 296 * Returns an index that must be passed to the matching srcu_read_unlock().
 297 */
 298int __srcu_read_lock(struct srcu_struct *sp)
 299{
 300        int idx;
 301
 302        preempt_disable();
 303        idx = rcu_dereference_index_check(sp->completed,
 304                                          rcu_read_lock_sched_held()) & 0x1;
 305        ACCESS_ONCE(this_cpu_ptr(sp->per_cpu_ref)->c[idx]) += 1;
 306        smp_mb(); /* B */  /* Avoid leaking the critical section. */
 307        ACCESS_ONCE(this_cpu_ptr(sp->per_cpu_ref)->seq[idx]) += 1;
 308        preempt_enable();
 309        return idx;
 310}
 311EXPORT_SYMBOL_GPL(__srcu_read_lock);
 312
 313/*
 314 * Removes the count for the old reader from the appropriate per-CPU
 315 * element of the srcu_struct.  Note that this may well be a different
cpu;
 317       *be called from process context.
 318       an>
 319__srcu_read_lock
u.c#L234" 3 span clascpu_ref" class=m    *3We theass="sref">srcu_struct ernel/src3.c#L220" id="L220" class3"line321" class="sref">seq[ 301
idx]) += 1;
 306        smp_mb( 307        ACCESS_ONCE(this_cpu_ptr(sp-> 232idx]) += 1;
fal3e3
 309        return 
 310}
       an>
cpu"> 226 227         * The following s" id="L312" class="line" name="L312"L228"> 238   328"> 128  n3d="L129" class="line" name="L12synchronizet">  _expedient
    3invoke this on a given srcu_str(defi>   nd th)ruct  theppen after
 231
 132
 233smp_mb 233sp-5_mb" class="sref">smp_mbcpuspsmp_mb12_mb" class="sref">smp_mb 23238 3( 2393/a> 243/*
   3     * in the interim.  But t*="kernelvclk()drank of froes for the"  Ther>
3d="L142" class="line" name="L14pan>
3243 3ns approximate total of the reasnd /a> = ( 31) ^ 13" id="L233" class="line" name="3e234"> 233 2443span class="comment"> * rank of per-CPU counters specified by i3"L245"> 235   3="L145" classel/srcu.c#L283"184"> 184 */ 30_read307" id="L307" cry3"> 30_readhref="kernel/srcu.c#L307" id="L219" c3ass="line" name="L219"> 319__srcu_read_lock__srcu_cryspan 307" id="L307" cryspan  can therefore suffer
u.cu.c#L2463 id="L246" class="line" 3ame="3de=sp" class="sref">sp, int 3>
 266         248         111111111ame="L221"> 221 .c#L185" id="L185" class="line" name="L185"> 185staspan a href="+code=cleanu_read_unlock() ="sref">__srcu_read_lock
u.>(); /* D */3/span3
307" id="L307" cru!= 0)
sp, __srcu_cryspan 307" id="L307" cryspan  can <or. */
se3;3
idx) != 0)
 2313ef">for_each_possible_cpu(idxstaspan a href="+cpen _RETRY_CHECK_DELAYan class="commepen _RETRY_CHECK_DELAY"srehref="kernel/srcu.c#L308" id> 2523 309        return <" name="L354"> 254 309        return <"> 255 *3@sp: 3hich s"> 309        return <"u.c#L2463
 237 * 3ote that this is not an atomic Ilaid out<=">
 238 * 3evere errors when invoked on anus/span>arted oa href==">
 2593span 3d="L159" class="line" name="L15u huctwai] fel/src>/*

33struct 
 298int __srcu_rea3*s3)3 266        idx =  36 309        return sum = 036ich s"> 309        return , , pan>
-a href=srcueadpan>ameitan class=l

c[1]);
<3 href37ass="comment"> * can be useful as an error check at cleanup ti3l/srcu.c#3270" id="L270" class="li3e" na3e="L27" class="line" name="  cu_dereference_i  href="kernel/srcu.c#L307" id="L219" c3ass="line" name="L219"> 319rcu_dereferenca href="+3ode=sum" class="sref">su3;37/ss="sref">idx/_lock="kernel/srcu.c#L307" i"L21h
u."L271" cl3ss="line" name="L271"> 23137a hre266"> 266         2723 263        int idx 274c[idx =  289        /* Ahidx = /ss="sref">idx/_locclass="line" name=t">/ss="sref">idx/_locref="+code=sp" class="sref3us="sref"3a>3* Mus37lass="line" name="L247"> 247 el/srcu.c#L283" "+code=idx" class="sref">idx = __srcu_flagsss="sref">idx 278sp-> el/srcu.c#L283" "+code=idx" class="sref">idx = __srcu_h3idx =  266        3struct idx = 307" id="L307" cru!= 0)
s3)38staspan a href="+coef="k_nrt_w>this_cpu_ptref="k_nrt_w>href) 04"> el/srcu.c#L283" "+code=idx" class="sref">idx = for_each_p9"> 309        return <s3m3

 el/srcu.c#L283" "+code=idx" class="sref">idx = __srcu_flagsss="sref">idx 309        return su3)38class=="kernel/srcu.c#L310" id="L310" class="line" name="L310"> 310}
  cu_dereference_i  hrefhref="kernel/srcu.c#L308" id p.sum<3a> != 0)
 266        sp->"kernel/srcu.c#L307" i"L21h__srcu_hsp-&3t;sp->"kernel/srcu.c#L307" iindex_cioncu_dereference_index_cc#L24/a3="sref">__srcu_index_cioncu_dereference_index_cc#L24/aref="kernel/srcu.c#L308" id>ruct3e=NULL" class="sref">NUL3;3EXPORT_SYMBOL_GP3(3a href="+code=cleanup_srcu_stru3t" class=3sref">cleanup_srcu_struc3)3
 292  n 294

 295 * srcu_struct.  l as an error check at cleanup ti3"> 296 *3Retur3s an i" class" c3ass="line" name=waknme_class_>  cu_dereference_waknme_class_>  href="kernel/srcu.c#L307" i"L21hsum<3 266        3struct sp->"kernel/srcu.c#L307" i"L21synchronizeel/srcu.c#L304" id=synchronizea hres4)4NUL4line"401" class="sref">seq[  cu_dereference_>  hrefclass="line" name=lockas="r_oESS_ONCE(__srcu_hi4x4
 el/srcu.c#L283">  cu_dereference_>  hreff">idx = cleanup_srcu_struc4 4      9"> 309        return 4e_index_c4eck(smp_mbidx 319__srcu_cryspan 307" id="L307" cryspan  can ref="kernel/srcu.c#L308" i4<6       4  name="L295"> 295ass="="ks="ludx>
 298int  *4per_c4u_ref)-> 408/*  n  _expedient
id4;411ock_" class" c3ass="line" name=__synchronizet">  +code=__srcu_re__synchronizet">  href="kernel/srcu.c#L307" id="L219" c3ass="line" name="L219"> 319__srcu_cryspan 307" id="L307" cryspan  can error check at cleanup ti4aa href="4 class="sref">EXPORT_SYM4OL_GP41t" class="sref">srcu_struct4lock" cla4s="sref">__srcu_read_loc4)41f">for_each_p"kernel/srcu.c#L307" i"L21synchronizeel/srcu.c#L304" id=synchronizea hrespan a href="+c>  cu_dereference_>  hrefref="kernel/srcu.c#L308" i4"> 312for_each_p"kernel/srcu.c#L307" i"L21h el/srcu.c#L283">  cu_dereference_>  href.="sref">__srcu_h4314 41lock_sched_held" class="sref184"> 184 */ 184idx) != 0)
 345 * 41ich s"> 309        return 4f">cpu/* A id= id=dep_refer 307" id="L307"  id= id=dep_refer href=!el/srcu.c#L283"
 el/srcu.c#L283" "+code=idx" class="sref">idx =  04"> "> 309        return 4ff">per_c47" class="line" name="L247"> 341cpu" class="sref">cpuuuuuuuuuuuu!el/srcu.c#L283"
 el/srcu.c#L283" id=bh4" clama"+code=idx" clas id=bh4" clama"9 04"> "> 309        return 4f"L308" c4 111111111uuuuuuuuuuu!el/srcu.c#L283"
 el/srcu.c#L283" id=" clama"+code=idx" clas id=" clama"9 04"> "> 309        return 4c#L234" 34span clascpu_ref" class=4    *42 el/srcu.c#L283" id=ss="li" clama"+code=idx" clas id=ss="li" clama"9rcu_dereferen4ernel/src4.c#L220" id="L220" class4"line42"+code=c" class="sreffffffffffff* read-side c"kepane>"Illeg     n

 el/srcu.c#L283">  cu_dereference_>  href.="sref">__srcu_="+codeioncu_dereference_index_cc#L24/a ref="kernel/srcu.c#L308" i43" id="L243" class="line" name="L243"> 242=smp_mb" class="sref">smp_mbfal4e42lock_sched_held" class="srefhidx =  289        c[idx = /ss="sref">idx/_locclass="line" name=waknme_class_>  cu_dereference_waknme_class_>  hrefref="+code=sp" class="sref4a">cpu 226/* A"pin4" clairqrcu.c#L284" id="pin4" clairqhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx = per_c47"> 227idx =  266         248   42ame="L148"> 111111111 id="L306" class="line" nsterefown*43idx = 307" id="L307" cru!= 0)
    43"+code=c" class="srefel/srcu.c#L283" id=b hre_queu)el/srcu.c#L304" id=b hre_queu)href=04"> el/srcu.c#L283" "+code=idx" class="sref">idx = __srcu_h 231 = 
 el/srcu.c#L283" "+code=idx" class="sref">idx = 
 282        int4n" id="L24/a>sp->idx__srcu_cryspan 307" id="L307" cryspan  can ref="kernel/srcu.c#L308" i4L234"> 234sp->yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  184 266         234sp->->idx =  184__srcu_h__srcu_hcpu 286       ss="srefel/srcu.c#L283" id=b hre_dequeu)el/srcu.c#L304" id=b hre_dequeu)href=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  184per_c4cpuuuuuuuuumment"> */ 184307" id="L307" cru!= 0)
 24238 43ame="L148"> 1111111119"> 309        return 4239"> 2394/a> 244ass="="ks="ludx>
int    44 266        44f">for_each_possible_cpu( el/srcu.c#L283" "+code=idx" class="sref">idx = __srcu_h4243 44="sref">sp->
 el/srcu.c#L283" "+code=idx" class="sref">idx =  234 244445"sref">sp 309        return 42235"> 2345   44ich s"> 309        return 4cu.c#L2464 id="L246" class="line" 4ame="44L286"> 286 184
cpu)-> el/srcu.c#L283">  cu_dereference_>  href.="sref">__srcu_="+codeioncu_dereference_index_cc#L24/a ref="kernel/srcu.c#L308" i4name="L244"> 248         309        return 4>(); /* D */4/span45a hreef="kernel/srcu.c#L308" i4de=sp" cl4ss="sref">sp, se4;45ny other function.  Each srcu_   -twai] fel/sriel/pen after
 24145="L142" class="line" name="L14- deco"L219"> 31 withay welluct 23454"> 254 234 *4@sp: 45ement of the srcu_struct.  NotAs withae srcicsen ,/="coupda/srccess us/s-omspsein mte means oEa>="L292" class="line" name="L294>u.c#L2464
 * 4ote that this is not an atomic srcuead_lock().  Ther>
 248 * 4evere errors when invoked on a Ther>
 2594span 4d="L159" class="line" name="L15er-CPU  n

4struct   n
s4)46="L142" class="line" name="L14o"L219"> 31 span>-omsparted o"L219"> 31'safter
4c#L263" id="L263" class=4line"46="L293" class="line" name="L2l as an error check at cleanup ti464" id="L464" class="line" name="L464"> 46  +code=__srcu_resynchronizet">  href="kernel/srcu.c#L307" id="L219" c3ass="line" name="L219"> 319sum = 046ich s266"> 266        /* A__synchronizet">  +code=__srcu_re__synchronizet">  href=span a href="+code=cleanu_read_unlock() ="sref">__srcu_pYNCHRONIZE_pen _TRYCOUNTan class="commepYNCHRONIZE_pen _TRYCOUNT"sre ref="kernel/srcu.c#L308" i4ss="sref"4per_cpu_ref,  309        return 4ss="sref"4per_cpu_ref, __srcu_c#L310" id="L310" class="line" name="L310"> 310}
  +code=__srcu_resynchronizet">  href ref="kernel/srcu.c#L308" i4+code=c" 4lass="sref">c[1]);
<4 href47a hreef="kernel/srcu.c#L308" i4l/srcu.c#4270" id="L270" class="li4e" na47eturns approximate number o" id="L272" class="line" name="L2724a href="+4ode=sum" class="sref">su4;47ny other function.  Each srcu_  _expedient -tBrute-felc apen apan>
 24147="L142" class="line" name="L14- deco"L219"> 31 withay welluct 274
arted at thb
when wai]pan  Ther>
4* Mus47te that this is not an atomic er-CPU
/*any lid=ocher>
 2784  _expedient


s4)48ny other function.  Each srcu_perf"kely leg    sle  te  _expedient
 319
 31'safter
4code=sum" class="sref">s4m48="L293" class="line" name="L29
panphref=o"L219"> 31spis acyclic  Ther>
 * rank of per-CPU counters specified by i4a href="+4ode=sum" class="sref">su4)48class" c3ass="line" name=aynchronizet">  _expedient+code=__srcu_resynchronizet">  _expedienthref="kernel/srcu.c#L307" id="L219" c3ass="line" name="L219"> 319sp, int 4ref">sum<4a> != 0)
 247__synchronizet">  +code=__srcu_re__synchronizet">  href=span a href="+code=cleanu_read_unlock() ="sref">__srcu_pYNCHRONIZE_pen _ame_TRYCOUNTan class="commepYNCHRONIZE_pen _ame_TRYCOUNT"sre ref="kernel/srcu.c#L308" i4tname="L24a>( 309        return 4>sp-&4t;spe"sref">__srcu_c#L310" id="L310" class="line" name="L310"> 310}
  _expedient+code=__srcu_resynchronizet">  _expedienthref ref="kernel/srcu.c#L308" i4>ruct4e=NULL" class="sref">NUL4;49ame="L300"> 300        int4>EXPORT_SYMBOL_GP4(49ny other function.  Each sr" id="L272" class="line" name="L2724t" class=4sref">cleanup_srcu_struc4)4
  
4
 294 319 295 266         296 *4Retur49  /* A"ynchronizet">  +code=__srcu_resynchronizet">  href=int sum<4 309        return 4_lock4struct __srcu_c#L310" id="L310" class="line" name="L310"> 310}
s5)5NUL5line"50eturns approximate number o" id="L272" class="line" name="L2725i5x50ny other function.  Each srcu_< id=b hrees5="+codent -tf="+codb hreeshredex_cheod="L272" class="line" name="L2725<3 href="5cref">cleanup_srcu_struc5 50="L142" class="line" name="L14- deco"L219"> 31 /spy welluctreportdb hre index_cc#Lod="L272" class="line" name="L2725<4 href="5c 295
s cl a ,d


 *5per_c5u_ref)-> 319 508 266        idx = id5;511ock_9"> 309        return 5aa href="5 class="sref">EXPORT_SYM5OL_GP51t" cle"sref">__srcu_c#L310" id="L310" class="line" name="L310"> 310}
__srcu_read_loc5)51ame="L282"> 282        int5"> 312fo#defi> nel/srcu.c#L283"pen _CALLBACK_BATCHan class="commepen _CALLBACK_BATCHa href="k10L282"> 282        int5"        5314 51lock_#defi> nel/srcu.c#L283"pen _INTERVA" class="line" npen _INTERVA"pu" class="sref1L282"> 282        int5"6       55 * 51ich s"> 309        return 5f">cpuper_c57" class="line" name="L257"> 351te that this is not an atomic Movemmnysnewfpen a

pen apan>
id="L312" class="line" name="L315f"L308" c5
 319srcu_struct5rcu_reade5s_active_idx(yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  266         252="sref">sp-> el/srcu.c#L283" "+code=idx" class="sref">idx = fal5e525"sref">sp-> el/srcu.c#L283" "+code=idx" class="sref">idx =  el/srcu.c#L283" "+code=idx" class="sref">idx = sp->
 el/srcu.c#L283" "+code=idx" class="sref">idx = cpu 226 309        return 53f">per_c57"> 227 309        return 5"L228"> 258   52/a>(53  * calls to srcu_readers_s" id="L312" class="line" name="L315mrnel/src5f">idb hre_185 231idb hre_185idb hre_doneras>

idx 319__srcu_cryspan 307" id="L307" cryspan  can error check at cleanup ti5nef="+cod5 266         235sp__srcu_read_lock 221<"+code=idx" class="sref">idx =  31)ref="kernel/srcu.c#L308" i5m">cpuper_c5 25238 53        * To see this, suppooooooooo* Becaus/s
 2395/a> 245x=0 snd /a>=1n  W"f   54ny other function.  Each srcoooooooo* needhuctwai] fel/
both0"f fr values befor
id="L312" class="line" name="L315L>
54="L142" class="line" name="Loooooooo* invo pan>aa

5243 54="L293" class="line" name="Loooooooo*l as an error check at cleanup ti5e234"> 235 24454a href="kernel/srcu.c#L174" id52235"> 2355   546"sref">sp 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  04"> "> 309        return 5cu.c#L2465 id="L246" class="line" 5ame="54L286"> 286   "L221"> 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  309        return 5cf">per_c5"kernel/srcu.c#L247" id=5L247"54cpu" class="sref">cpuf="+co;neid="L306" class="line" nnoa
 25"> 248        (); /* D */5/span55a href="kernelf (!el/srcu.c#L283"cry_185__srcu_read_lock__srcu_cryspan 307" id="L307" cryspan  can  "> 309        return 5de=sp" cl5ss="sref">sp, se5;55 href="+code=cleanup_srcu_stru5ae="L242"5ss="line" name="L251"> 251553pu" class="salls to srcu_readers_s" id="L312" class="line" name="L315> 2525idb hre_185234"> 23554"> 254 235 *5@sp: 55ement of the srcu_struct.  Noooooooo* f">idb hre_185u.c#L2465
per_c57 * 5ote that this is not an atomioooooooo* invocu.c#L, leavpan Uanded o"cf">idb hre_185L228"> 258 * 5evere errors when invoked on oooooooo* They=cr"fidb hre_doneaid="L292" class="line" name="L295 class="l5ne" name="L259"> 2595span 5d="L159" class="line" name="Loooooooo*l as an error check at cleanup ti5 e=sp" cl5seq[  =b hre_mo/srcu.c#L284" id= id=b hre_mo/shref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  184 el/srcu.c#L283" "+code=idx" class="sref">idx =  309        return 5ctive5struct s5)56      221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  309        return 5c 2525c#L263" id="L263" class=5line"56="sref">sp-> 56lock_sched_held" class="srefo"L21flidass="line" name="L21flidhref=int sum = 056ich s"> 309        return 5f="+code=5or_each_possible_cpu" cl5ss="s56  , idb hre_185, idb hre_185c[1]);
<5 href57     * read-side critical seoooooooo* foo>
su5;57 el/srcu.c#L283" "+code=idx" class="sref">idx =  el/srcu.c#L283" "+code=idx" class="sref">idx =  25157ame="L282"> 282        int5> 2725 2745* Mus57lass="line" name="L247"> 247cryspan 307" id="L307" cryspan  canclass="line" name= 278__srcu_read_lock__srcu_cryspan 307" id="L307" cryspan  can  "> 309        return 5y.5 300        int5y href="+5+code=sp" class="sref">s5)582 idb hre_185
s5m58="L293" class="line" name="Loooooooo* pre-exis pan>
both0">x values.  They=cr"f
 * rank oooooooo* 
idb hre_doneaid="L292" class="line" name="L295a href="+5ode=sum" class="sref">su5)58ement of the srcu_struct.  Noooooooo*l as an error check at cleanup ti5y="+code=56" id="L286" class="line5 name58  /* A id=b hre_mo/srcu.c#L284" id= id=b hre_mo/shref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  184 el/srcu.c#L283" "+code=idx" class="sref">idx = ror check at cleanup ti5ys="sref"5a> != 0)
 309        return 5tname="L25a>(sp-&5t;NUL5;59eturns approximate number of /Invo e a limi]ed numbtr ef=pen a
EXPORT_SYMBOL_GP5(59ny other function.  Each srL29


="L312" class="line" name="L315>e="L242"5sref">cleanup_srcu_struc5)5
 294 319 309        return 5lass="lin5" name="L295"> 295 266         296 *5Retur59  __srcu_rd_locksum<5(s6)60__srcu_rd_lock__srcu_pen _CALLBACK_BATCHan class="commepen _CALLBACK_BATCHa hr;s="sref">__srcu_rd_lock 266        NUL6line"60"+code=c" class="srefel/srcu.c#L283"h/* A id=b hre_dequeu)el/srcu.c#L304" id=b hre_dequeu)href=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  184i6x60cleanup_srcu_struc6 60f">for_each_possible_________b
sp->lo
sp->idx = /ss="sref">idx/_loc=span a href="+ch 295sp->lo
 *6 309        return 6ef">per_c6u_ref)-> 309        return 6="L308" c6ass="line" name="L308"> 608(id6;61eturns approximate number of /Finishedaonerround ef=pen apan>
EXPORT_SYM6OL_GP61ny other function.  Each srL29mot apen a
__srcu_read_loc6)61="L142" class="line" name="L1l as an error check at cleanup ti6"> 312ass="="ks="ludx>
 319 309        return 6"        6314 615"sre266"> 266         * 61e=c" class="sref">c[ 184c[ 184307" id="L307" cru!= 0)
cpuper_c67" class="line" name="L267"> 361cpu" class="slf ("L221"> 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  184 04"> "> 309        return 6f"L308" c6 11111"L221"> 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  04"> "> 309        return 6c#L234" 36span clascpu_ref" class=6    *62 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  04"> "> 309        return 6ernel/src6.c#L220" id="L220" class6"line62"+code=c" class=""L221"> 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  266        
 el/srcu.c#L283" "+code=idx" class="sref">idx = (for_each_possible_lf ("L221"> 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  184 04"> "> 309        return 63" id="L263" class="line" name="L263"> 262="sref">sp-> 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  04"> "> 309        return 6c        6=false" class="sref">fal6e625"sref">sp-> 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  04"> "> 309        return 6e6       6="kernel/srcu.c#L225" id6"L225626"sref">sp-> 221 id=b hre_yss="sref">idxyhref=04"> el/srcu.c#L283" "+code=idx" class="sref">idx =  266        ">cpu 226 286       ss="srefel/srcu.c#L283" "+code=idx" class="sref">idx = ) !ss="sref">idx) != 0)
per_c67"> 227cpuuuuuuuuumment"> */ 184) !ss="sref">idx) != 0)
   62ame="L148"> 1111111119"> 309        return 6me="L229"6 22963
 el/srcu.c#L283" "+code=idx" class="sref">idx =  309        return 6m
 231
 221penel/s> 184 309        return 6n" id="L26/a>sp-> el/srcu.c#L283" "+code=idx" class="sref">idx = __srcu_pen _INTERVA" class="line" npen _INTERVA"pu"  ref="kernel/srcu.c#L308" i6m        6 309        return 6m6       6 309        return 6m">cpuper_c6/*
 26238 63        * To see this, suppo1l as an error check at cleanup ti6239"> 2396/a> 184
 319 309        return 6L240"> 246 266           64 319
64ame="L282"> 282        int6e="L243">6243 64="sref">sp 319__srcu_work"L311" class="lworkhref.="sref">__srcu_work"L311" class="lworkhref ref="kernel/srcu.c#L308" i6L        6 24464a href="kernel/srcu.c#L174" id62235"> 2365   64e=c" class="sref">c[cpu/* A"rs="advance=b hreesss="sref">idxper_c6"kernel/srcu.c#L247" id=6L247"64lass="line" name="L247"> 247ass="invo e_
 26"> 248         247ass="="ks="ludx" class="sref">ass="="ks="ludx>
(); /* D */6/span65a hre9"> 309        return 6de=sp" cl6ss="sref">sp, sfooter">
Theaoriginal LXR softwcr"fbspan>
> 309http://sourc
forge.net/projects/lxr">LXR his,uni>yhref,ean>

bsp> 309mailto:lxr@plxr@p
ssubfooter">
lxr.p 309http://www.redpill-pRedpill Lsnd operaernes=serviuea sincea1995.
s/div>