LXR linux/mm/allocpercpu.c

   1/*
   2 * linux/mm/allocpercpu.c
   3 *
   4 * Separated from slab.c August 11, 2006 Christoph Lameter <clameter@sgi.com>
   5 */
   6#include <linux/mm.h>
   7#include <linux/module.h>
   8
   9/**
  10 * percpu_depopulate - depopulate per-cpu data for given cpu
  11 * @__pdata: per-cpu data to depopulate
  12 * @cpu: depopulate per-cpu data for this cpu
  13 *
  14 * Depopulating per-cpu data for a cpu going offline would be a typical
  15 * use case. You need to register a cpu hotplug handler for that purpose.
  16 */
  17void percpu_depopulate(void *__pdata, int cpu)
  18{
  19        struct percpu_data *pdata = __percpu_disguise(__pdata);
  20
  21        kfree(pdata->ptrs[cpu]);
  22        pdata->ptrs[cpu] = NULL;
  23}
  24EXPORT_SYMBOL_GPL(percpu_depopulate);
  25
  26/**
  27 * percpu_depopulate_mask - depopulate per-cpu data for some cpu's
  28 * @__pdata: per-cpu data to depopulate
  29 * @mask: depopulate per-cpu data for cpu's selected through mask bits
  30 */
  31void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
  32{
  33        int cpu;
  34        for_each_cpu_mask(cpu, *mask)
  35                percpu_depopulate(__pdata, cpu);
  36}
  37EXPORT_SYMBOL_GPL(__percpu_depopulate_mask);
  38
  39/**
  40 * percpu_populate - populate per-cpu data for given cpu
  41 * @__pdata: per-cpu data to populate further
  42 * @size: size of per-cpu object
  43 * @gfp: may sleep or not etc.
  44 * @cpu: populate per-data for this cpu
  45 *
  46 * Populating per-cpu data for a cpu coming online would be a typical
  47 * use case. You need to register a cpu hotplug handler for that purpose.
  48 * Per-cpu object is populated with zeroed buffer.
  49 */
  50void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu)
  51{
  52        struct percpu_data *pdata = __percpu_disguise(__pdata);
  53        int node = cpu_to_node(cpu);
  54
  55        BUG_ON(pdata->ptrs[cpu]);
  56        if (node_online(node)) {
  57                /* FIXME: kzalloc_node(size, gfp, node) */
  58                pdata->ptrs[cpu] = kmalloc_node(size, gfp, node);
  59                if (pdata->ptrs[cpu])
  60                        memset(pdata->ptrs[cpu], 0, size);
  61        } else
  62                pdata->ptrs[cpu] = kzalloc(size, gfp);
  63        return pdata->ptrs[cpu];
  64}
  65EXPORT_SYMBOL_GPL(percpu_populate);
  66
  67/**
  68 * percpu_populate_mask - populate per-cpu data for more cpu's
  69 * @__pdata: per-cpu data to populate further
  70 * @size: size of per-cpu object
  71 * @gfp: may sleep or not etc.
  72 * @mask: populate per-cpu data for cpu's selected through mask bits
  73 *
  74 * Per-cpu objects are populated with zeroed buffers.
  75 */
  76int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  77                           cpumask_t *mask)
  78{
  79        cpumask_t populated = CPU_MASK_NONE;
  80        int cpu;
  81
  82        for_each_cpu_mask(cpu, *mask)
  83                if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) {
  84                        __percpu_depopulate_mask(__pdata, &populated);
  85                        return -ENOMEM;
  86                } else
  87                        cpu_set(cpu, populated);
  88        return 0;
  89}
  90EXPORT_SYMBOL_GPL(__percpu_populate_mask);
  91
  92/**
  93 * percpu_alloc_mask - initial setup of per-cpu data
  94 * @size: size of per-cpu object
  95 * @gfp: may sleep or not etc.
  96 * @mask: populate per-data for cpu's selected through mask bits
  97 *
  98 * Populating per-cpu data for all online cpu's would be a typical use case,
  99 * which is simplified by the percpu_alloc() wrapper.
 100 * Per-cpu objects are populated with zeroed buffers.
 101 */
 102void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
 103{
 104        void *pdata = kzalloc(sizeof(struct percpu_data), gfp);
 105        void *__pdata = __percpu_disguise(pdata);
 106
 107        if (unlikely(!pdata))
 108                return NULL;
 109        if (likely(!__percpu_populate_mask(__pdata, size, gfp, mask)))
 110                return __pdata;
 111        kfree(pdata);
 112        return NULL;
 113}
 114EXPORT_SYMBOL_GPL(__percpu_alloc_mask);
 115
 116/**
 117 * percpu_free - final cleanup of per-cpu data
 118 * @__pdata: object to clean up
 119 *
 120 * We simply clean up any per-cpu object left. No need for the client to
 121 * track and specify through a bis mask which per-cpu objects are to free.
 122 */
 123void percpu_free(void *__pdata)
 124{
 125        if (unlikely(!__pdata))
 126                return;
 127        __percpu_depopulate_mask(__pdata, &cpu_possible_map);
 128        kfree(__percpu_disguise(__pdata));
 129}
 130EXPORT_SYMBOL_GPL(percpu_free);
 131