LXR linux/include/linux/percpu.h

   1#ifndef __LINUX_PERCPU_H
   2#define __LINUX_PERCPU_H
   3
   4#include <linux/preempt.h>
   5#include <linux/slab.h> /* For kmalloc() */
   6#include <linux/smp.h>
   7#include <linux/cpumask.h>
   8
   9#include <asm/percpu.h>
  10
  11#ifdef CONFIG_SMP
  12#define DEFINE_PER_CPU(type, name)                                      \
  13        __attribute__((__section__(".data.percpu")))                    \
  14        PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  15
  16#ifdef MODULE
  17#define SHARED_ALIGNED_SECTION ".data.percpu"
  18#else
  19#define SHARED_ALIGNED_SECTION ".data.percpu.shared_aligned"
  20#endif
  21
  22#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
  23        __attribute__((__section__(SHARED_ALIGNED_SECTION)))            \
  24        PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name             \
  25        ____cacheline_aligned_in_smp
  26
  27#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)                 \
  28        __attribute__((__section__(".data.percpu.page_aligned")))       \
  29        PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  30#else
  31#define DEFINE_PER_CPU(type, name)                                      \
  32        PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  33
  34#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                     \
  35        DEFINE_PER_CPU(type, name)
  36
  37#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)               \
  38        DEFINE_PER_CPU(type, name)
  39#endif
  40
  41#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
  42#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
  43
  44/* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
  45#ifndef PERCPU_ENOUGH_ROOM
  46#ifdef CONFIG_MODULES
  47#define PERCPU_MODULE_RESERVE   8192
  48#else
  49#define PERCPU_MODULE_RESERVE   0
  50#endif
  51
  52#define PERCPU_ENOUGH_ROOM                                              \
  53        (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
  54#endif  /* PERCPU_ENOUGH_ROOM */
  55
  56/*
  57 * Must be an lvalue. Since @var must be a simple identifier,
  58 * we force a syntax error here if it isn't.
  59 */
  60#define get_cpu_var(var) (*({                           \
  61        extern int simple_identifier_##var(void);       \
  62        preempt_disable();                              \
  63        &__get_cpu_var(var); }))
  64#define put_cpu_var(var) preempt_enable()
  65
  66#ifdef CONFIG_SMP
  67
  68struct percpu_data {
  69        void *ptrs[1];
  70};
  71
  72#define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
  73/* 
  74 * Use this to get to a cpu's version of the per-cpu object dynamically
  75 * allocated. Non-atomic access to the current CPU's version should
  76 * probably be combined with get_cpu()/put_cpu().
  77 */ 
  78#define percpu_ptr(ptr, cpu)                              \
  79({                                                        \
  80        struct percpu_data *__p = __percpu_disguise(ptr); \
  81        (__typeof__(ptr))__p->ptrs[(cpu)];                \
  82})
  83
  84extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
  85extern void percpu_free(void *__pdata);
  86
  87#else /* CONFIG_SMP */
  88
  89#define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
  90
  91static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
  92{
  93        return kzalloc(size, gfp);
  94}
  95
  96static inline void percpu_free(void *__pdata)
  97{
  98        kfree(__pdata);
  99}
 100
 101#endif /* CONFIG_SMP */
 102
 103#define percpu_alloc_mask(size, gfp, mask) \
 104        __percpu_alloc_mask((size), (gfp), &(mask))
 105
 106#define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
 107
 108/* (legacy) interface for use without CPU hotplug handling */
 109
 110#define __alloc_percpu(size)    percpu_alloc_mask((size), GFP_KERNEL, \
 111                                                  cpu_possible_map)
 112#define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type))
 113#define free_percpu(ptr)        percpu_free((ptr))
 114#define per_cpu_ptr(ptr, cpu)   percpu_ptr((ptr), (cpu))
 115
 116#endif /* __LINUX_PERCPU_H */
 117