linux/include/linux/seqlock.h
<<
>>
Prefs
   1#ifndef __LINUX_SEQLOCK_H
   2#define __LINUX_SEQLOCK_H
   3/*
   4 * Reader/writer consistent mechanism without starving writers. This type of
   5 * lock for data where the reader wants a consistent set of information
   6 * and is willing to retry if the information changes.  Readers never
   7 * block but they may have to retry if a writer is in
   8 * progress. Writers do not wait for readers. 
   9 *
  10 * This is not as cache friendly as brlock. Also, this will not work
  11 * for data that contains pointers, because any writer could
  12 * invalidate a pointer that a reader was following.
  13 *
  14 * Expected reader usage:
  15 *      do {
  16 *          seq = read_seqbegin(&foo);
  17 *      ...
  18 *      } while (read_seqretry(&foo, seq));
  19 *
  20 *
  21 * On non-SMP the spin locks disappear but the writer still needs
  22 * to increment the sequence variables because an interrupt routine could
  23 * change the state of the data.
  24 *
  25 * Based on x86_64 vsyscall gettimeofday 
  26 * by Keith Owens and Andrea Arcangeli
  27 */
  28
  29#include <linux/spinlock.h>
  30#include <linux/preempt.h>
  31#include <asm/processor.h>
  32
  33typedef struct {
  34        unsigned sequence;
  35        spinlock_t lock;
  36} seqlock_t;
  37
  38/*
  39 * These macros triggered gcc-3.x compile-time problems.  We think these are
  40 * OK now.  Be cautious.
  41 */
  42#define __SEQLOCK_UNLOCKED(lockname) \
  43                 { 0, __SPIN_LOCK_UNLOCKED(lockname) }
  44
  45#define seqlock_init(x)                                 \
  46        do {                                            \
  47                (x)->sequence = 0;                      \
  48                spin_lock_init(&(x)->lock);             \
  49        } while (0)
  50
  51#define DEFINE_SEQLOCK(x) \
  52                seqlock_t x = __SEQLOCK_UNLOCKED(x)
  53
  54/* Lock out other writers and update the count.
  55 * Acts like a normal spin_lock/unlock.
  56 * Don't need preempt_disable() because that is in the spin_lock already.
  57 */
  58static inline void write_seqlock(seqlock_t *sl)
  59{
  60        spin_lock(&sl->lock);
  61        ++sl->sequence;
  62        smp_wmb();
  63}
  64
  65static inline void write_sequnlock(seqlock_t *sl)
  66{
  67        smp_wmb();
  68        sl->sequence++;
  69        spin_unlock(&sl->lock);
  70}
  71
  72static inline int write_tryseqlock(seqlock_t *sl)
  73{
  74        int ret = spin_trylock(&sl->lock);
  75
  76        if (ret) {
  77                ++sl->sequence;
  78                smp_wmb();
  79        }
  80        return ret;
  81}
  82
  83/* Start of read calculation -- fetch last complete writer token */
  84static __always_inline unsigned read_seqbegin(const seqlock_t *sl)
  85{
  86        unsigned ret;
  87
  88repeat:
  89        ret = ACCESS_ONCE(sl->sequence);
  90        if (unlikely(ret & 1)) {
  91                cpu_relax();
  92                goto repeat;
  93        }
  94        smp_rmb();
  95
  96        return ret;
  97}
  98
  99/*
 100 * Test if reader processed invalid data.
 101 *
 102 * If sequence value changed then writer changed data while in section.
 103 */
 104static __always_inline int read_seqretry(const seqlock_t *sl, unsigned start)
 105{
 106        smp_rmb();
 107
 108        return unlikely(sl->sequence != start);
 109}
 110
 111
 112/*
 113 * Version using sequence counter only.
 114 * This can be used when code has its own mutex protecting the
 115 * updating starting before the write_seqcountbeqin() and ending
 116 * after the write_seqcount_end().
 117 */
 118
 119typedef struct seqcount {
 120        unsigned sequence;
 121} seqcount_t;
 122
 123#define SEQCNT_ZERO { 0 }
 124#define seqcount_init(x)        do { *(x) = (seqcount_t) SEQCNT_ZERO; } while (0)
 125
 126/**
 127 * __read_seqcount_begin - begin a seq-read critical section (without barrier)
 128 * @s: pointer to seqcount_t
 129 * Returns: count to be passed to read_seqcount_retry
 130 *
 131 * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb()
 132 * barrier. Callers should ensure that smp_rmb() or equivalent ordering is
 133 * provided before actually loading any of the variables that are to be
 134 * protected in this critical section.
 135 *
 136 * Use carefully, only in critical code, and comment how the barrier is
 137 * provided.
 138 */
 139static inline unsigned __read_seqcount_begin(const seqcount_t *s)
 140{
 141        unsigned ret;
 142
 143repeat:
 144        ret = ACCESS_ONCE(s->sequence);
 145        if (unlikely(ret & 1)) {
 146                cpu_relax();
 147                goto repeat;
 148        }
 149        return ret;
 150}
 151
 152/**
 153 * read_seqcount_begin - begin a seq-read critical section
 154 * @s: pointer to seqcount_t
 155 * Returns: count to be passed to read_seqcount_retry
 156 *
 157 * read_seqcount_begin opens a read critical section of the given seqcount.
 158 * Validity of the critical section is tested by checking read_seqcount_retry
 159 * function.
 160 */
 161static inline unsigned read_seqcount_begin(const seqcount_t *s)
 162{
 163        unsigned ret = __read_seqcount_begin(s);
 164        smp_rmb();
 165        return ret;
 166}
 167
 168/**
 169 * raw_seqcount_begin - begin a seq-read critical section
 170 * @s: pointer to seqcount_t
 171 * Returns: count to be passed to read_seqcount_retry
 172 *
 173 * raw_seqcount_begin opens a read critical section of the given seqcount.
 174 * Validity of the critical section is tested by checking read_seqcount_retry
 175 * function.
 176 *
 177 * Unlike read_seqcount_begin(), this function will not wait for the count
 178 * to stabilize. If a writer is active when we begin, we will fail the
 179 * read_seqcount_retry() instead of stabilizing at the beginning of the
 180 * critical section.
 181 */
 182static inline unsigned raw_seqcount_begin(const seqcount_t *s)
 183{
 184        unsigned ret = ACCESS_ONCE(s->sequence);
 185        smp_rmb();
 186        return ret & ~1;
 187}
 188
 189/**
 190 * __read_seqcount_retry - end a seq-read critical section (without barrier)
 191 * @s: pointer to seqcount_t
 192 * @start: count, from read_seqcount_begin
 193 * Returns: 1 if retry is required, else 0
 194 *
 195 * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb()
 196 * barrier. Callers should ensure that smp_rmb() or equivalent ordering is
 197 * provided before actually loading any of the variables that are to be
 198 * protected in this critical section.
 199 *
 200 * Use carefully, only in critical code, and comment how the barrier is
 201 * provided.
 202 */
 203static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start)
 204{
 205        return unlikely(s->sequence != start);
 206}
 207
 208/**
 209 * read_seqcount_retry - end a seq-read critical section
 210 * @s: pointer to seqcount_t
 211 * @start: count, from read_seqcount_begin
 212 * Returns: 1 if retry is required, else 0
 213 *
 214 * read_seqcount_retry closes a read critical section of the given seqcount.
 215 * If the critical section was invalid, it must be ignored (and typically
 216 * retried).
 217 */
 218static inline int read_seqcount_retry(const seqcount_t *s, unsigned start)
 219{
 220        smp_rmb();
 221
 222        return __read_seqcount_retry(s, start);
 223}
 224
 225
 226/*
 227 * Sequence counter only version assumes that callers are using their
 228 * own mutexing.
 229 */
 230static inline void write_seqcount_begin(seqcount_t *s)
 231{
 232        s->sequence++;
 233        smp_wmb();
 234}
 235
 236static inline void write_seqcount_end(seqcount_t *s)
 237{
 238        smp_wmb();
 239        s->sequence++;
 240}
 241
 242/**
 243 * write_seqcount_barrier - invalidate in-progress read-side seq operations
 244 * @s: pointer to seqcount_t
 245 *
 246 * After write_seqcount_barrier, no read-side seq operations will complete
 247 * successfully and see data older than this.
 248 */
 249static inline void write_seqcount_barrier(seqcount_t *s)
 250{
 251        smp_wmb();
 252        s->sequence+=2;
 253}
 254
 255/*
 256 * Possible sw/hw IRQ protected versions of the interfaces.
 257 */
 258#define write_seqlock_irqsave(lock, flags)                              \
 259        do { local_irq_save(flags); write_seqlock(lock); } while (0)
 260#define write_seqlock_irq(lock)                                         \
 261        do { local_irq_disable();   write_seqlock(lock); } while (0)
 262#define write_seqlock_bh(lock)                                          \
 263        do { local_bh_disable();    write_seqlock(lock); } while (0)
 264
 265#define write_sequnlock_irqrestore(lock, flags)                         \
 266        do { write_sequnlock(lock); local_irq_restore(flags); } while(0)
 267#define write_sequnlock_irq(lock)                                       \
 268        do { write_sequnlock(lock); local_irq_enable(); } while(0)
 269#define write_sequnlock_bh(lock)                                        \
 270        do { write_sequnlock(lock); local_bh_enable(); } while(0)
 271
 272#define read_seqbegin_irqsave(lock, flags)                              \
 273        ({ local_irq_save(flags);   read_seqbegin(lock); })
 274
 275#define read_seqretry_irqrestore(lock, iv, flags)                       \
 276        ({                                                              \
 277                int ret = read_seqretry(lock, iv);                      \
 278                local_irq_restore(flags);                               \
 279                ret;                                                    \
 280        })
 281
 282#endif /* __LINUX_SEQLOCK_H */
 283
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.