linux/mm/mempool.c
<<
>>
Prefs
   1/*
   2 *  linux/mm/mempool.c
   3 *
   4 *  memory buffer pool support. Such pools are mostly used
   5 *  for guaranteed, deadlock-free memory allocations during
   6 *  extreme VM load.
   7 *
   8 *  started by Ingo Molnar, Copyright (C) 2001
   9 */
  10
  11#include <linux/mm.h>
  12#include <linux/slab.h>
  13#include <linux/export.h>
  14#include <linux/mempool.h>
  15#include <linux/blkdev.h>
  16#include <linux/writeback.h>
  17
  18static void add_element(mempool_t *pool, void *element)
  19{
  20        BUG_ON(pool->curr_nr >= pool->min_nr);
  21        pool->elements[pool->curr_nr++] = element;
  22}
  23
  24static void *remove_element(mempool_t *pool)
  25{
  26        BUG_ON(pool->curr_nr <= 0);
  27        return pool->elements[--pool->curr_nr];
  28}
  29
  30/**
  31 * mempool_destroy - deallocate a memory pool
  32 * @pool:      pointer to the memory pool which was allocated via
  33 *             mempool_create().
  34 *
  35 * Free all reserved elements in @pool and @pool itself.  This function
  36 * only sleeps if the free_fn() function sleeps.
  37 */
  38void mempool_destroy(mempool_t *pool)
  39{
  40        while (pool->curr_nr) {
  41                void *element = remove_element(pool);
  42                pool->free(element, pool->pool_data);
  43        }
  44        kfree(pool->elements);
  45        kfree(pool);
  46}
  47EXPORT_SYMBOL(mempool_destroy);
  48
  49/**
  50 * mempool_create - create a memory pool
  51 * @min_nr:    the minimum number of elements guaranteed to be
  52 *             allocated for this pool.
  53 * @alloc_fn:  user-defined element-allocation function.
  54 * @free_fn:   user-defined element-freeing function.
  55 * @pool_data: optional private data available to the user-defined functions.
  56 *
  57 * this function creates and allocates a guaranteed size, preallocated
  58 * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
  59 * functions. This function might sleep. Both the alloc_fn() and the free_fn()
  60 * functions might sleep - as long as the mempool_alloc() function is not called
  61 * from IRQ contexts.
  62 */
  63mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
  64                                mempool_free_t *free_fn, void *pool_data)
  65{
  66        return  mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1);
  67}
  68EXPORT_SYMBOL(mempool_create);
  69
  70mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
  71                        mempool_free_t *free_fn, void *pool_data, int node_id)
  72{
  73        mempool_t *pool;
  74        pool = kmalloc_node(sizeof(*pool), GFP_KERNEL | __GFP_ZERO, node_id);
  75        if (!pool)
  76                return NULL;
  77        pool->elements = kmalloc_node(min_nr * sizeof(void *),
  78                                        GFP_KERNEL, node_id);
  79        if (!pool->elements) {
  80                kfree(pool);
  81                return NULL;
  82        }
  83        spin_lock_init(&pool->lock);
  84        pool->min_nr = min_nr;
  85        pool->pool_data = pool_data;
  86        init_waitqueue_head(&pool->wait);
  87        pool->alloc = alloc_fn;
  88        pool->free = free_fn;
  89
  90        /*
  91         * First pre-allocate the guaranteed number of buffers.
  92         */
  93        while (pool->curr_nr < pool->min_nr) {
  94                void *element;
  95
  96                element = pool->alloc(GFP_KERNEL, pool->pool_data);
  97                if (unlikely(!element)) {
  98                        mempool_destroy(pool);
  99                        return NULL;
 100                }
 101                add_element(pool, element);
 102        }
 103        return pool;
 104}
 105EXPORT_SYMBOL(mempool_create_node);
 106
 107/**
 108 * mempool_resize - resize an existing memory pool
 109 * @pool:       pointer to the memory pool which was allocated via
 110 *              mempool_create().
 111 * @new_min_nr: the new minimum number of elements guaranteed to be
 112 *              allocated for this pool.
 113 * @gfp_mask:   the usual allocation bitmask.
 114 *
 115 * This function shrinks/grows the pool. In the case of growing,
 116 * it cannot be guaranteed that the pool will be grown to the new
 117 * size immediately, but new mempool_free() calls will refill it.
 118 *
 119 * Note, the caller must guarantee that no mempool_destroy is called
 120 * while this function is running. mempool_alloc() & mempool_free()
 121 * might be called (eg. from IRQ contexts) while this function executes.
 122 */
 123int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask)
 124{
 125        void *element;
 126        void **new_elements;
 127        unsigned long flags;
 128
 129        BUG_ON(new_min_nr <= 0);
 130
 131        spin_lock_irqsave(&pool->lock, flags);
 132        if (new_min_nr <= pool->min_nr) {
 133                while (new_min_nr < pool->curr_nr) {
 134                        element = remove_element(pool);
 135                        spin_unlock_irqrestore(&pool->lock, flags);
 136                        pool->free(element, pool->pool_data);
 137                        spin_lock_irqsave(&pool->lock, flags);
 138                }
 139                pool->min_nr = new_min_nr;
 140                goto out_unlock;
 141        }
 142        spin_unlock_irqrestore(&pool->lock, flags);
 143
 144        /* Grow the pool */
 145        new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask);
 146        if (!new_elements)
 147                return -ENOMEM;
 148
 149        spin_lock_irqsave(&pool->lock, flags);
 150        if (unlikely(new_min_nr <= pool->min_nr)) {
 151                /* Raced, other resize will do our work */
 152                spin_unlock_irqrestore(&pool->lock, flags);
 153                kfree(new_elements);
 154                goto out;
 155        }
 156        memcpy(new_elements, pool->elements,
 157                        pool->curr_nr * sizeof(*new_elements));
 158        kfree(pool->elements);
 159        pool->elements = new_elements;
 160        pool->min_nr = new_min_nr;
 161
 162        while (pool->curr_nr < pool->min_nr) {
 163                spin_unlock_irqrestore(&pool->lock, flags);
 164                element = pool->alloc(gfp_mask, pool->pool_data);
 165                if (!element)
 166                        goto out;
 167                spin_lock_irqsave(&pool->lock, flags);
 168                if (pool->curr_nr < pool->min_nr) {
 169                        add_element(pool, element);
 170                } else {
 171                        spin_unlock_irqrestore(&pool->lock, flags);
 172                        pool->free(element, pool->pool_data);   /* Raced */
 173                        goto out;
 174                }
 175        }
 176out_unlock:
 177        spin_unlock_irqrestore(&pool->lock, flags);
 178out:
 179        return 0;
 180}
 181EXPORT_SYMBOL(mempool_resize);
 182
 183/**
 184 * mempool_alloc - allocate an element from a specific memory pool
 185 * @pool:      pointer to the memory pool which was allocated via
 186 *             mempool_create().
 187 * @gfp_mask:  the usual allocation bitmask.
 188 *
 189 * this function only sleeps if the alloc_fn() function sleeps or
 190 * returns NULL. Note that due to preallocation, this function
 191 * *never* fails when called from process contexts. (it might
 192 * fail if called from an IRQ context.)
 193 */
 194void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
 195{
 196        void *element;
 197        unsigned long flags;
 198        wait_queue_t wait;
 199        gfp_t gfp_temp;
 200
 201        might_sleep_if(gfp_mask & __GFP_WAIT);
 202
 203        gfp_mask |= __GFP_NOMEMALLOC;   /* don't allocate emergency reserves */
 204        gfp_mask |= __GFP_NORETRY;      /* don't loop in __alloc_pages */
 205        gfp_mask |= __GFP_NOWARN;       /* failures are OK */
 206
 207        gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);
 208
 209repeat_alloc:
 210
 211        element = pool->alloc(gfp_temp, pool->pool_data);
 212        if (likely(element != NULL))
 213                return element;
 214
 215        spin_lock_irqsave(&pool->lock, flags);
 216        if (likely(pool->curr_nr)) {
 217                element = remove_element(pool);
 218                spin_unlock_irqrestore(&pool->lock, flags);
 219                /* paired with rmb in mempool_free(), read comment there */
 220                smp_wmb();
 221                return element;
 222        }
 223
 224        /*
 225         * We use gfp mask w/o __GFP_WAIT or IO for the first round.  If
 226         * alloc failed with that and @pool was empty, retry immediately.
 227         */
 228        if (gfp_temp != gfp_mask) {
 229                spin_unlock_irqrestore(&pool->lock, flags);
 230                gfp_temp = gfp_mask;
 231                goto repeat_alloc;
 232        }
 233
 234        /* We must not sleep if !__GFP_WAIT */
 235        if (!(gfp_mask & __GFP_WAIT)) {
 236                spin_unlock_irqrestore(&pool->lock, flags);
 237                return NULL;
 238        }
 239
 240        /* Let's wait for someone else to return an element to @pool */
 241        init_wait(&wait);
 242        prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
 243
 244        spin_unlock_irqrestore(&pool->lock, flags);
 245
 246        /*
 247         * FIXME: this should be io_schedule().  The timeout is there as a
 248         * workaround for some DM problems in 2.6.18.
 249         */
 250        io_schedule_timeout(5*HZ);
 251
 252        finish_wait(&pool->wait, &wait);
 253        goto repeat_alloc;
 254}
 255EXPORT_SYMBOL(mempool_alloc);
 256
 257/**
 258 * mempool_free - return an element to the pool.
 259 * @element:   pool element pointer.
 260 * @pool:      pointer to the memory pool which was allocated via
 261 *             mempool_create().
 262 *
 263 * this function only sleeps if the free_fn() function sleeps.
 264 */
 265void mempool_free(void *element, mempool_t *pool)
 266{
 267        unsigned long flags;
 268
 269        if (unlikely(element == NULL))
 270                return;
 271
 272        /*
 273         * Paired with the wmb in mempool_alloc().  The preceding read is
 274         * for @element and the following @pool->curr_nr.  This ensures
 275         * that the visible value of @pool->curr_nr is from after the
 276         * allocation of @element.  This is necessary for fringe cases
 277         * where @element was passed to this task without going through
 278         * barriers.
 279         *
 280         * For example, assume @p is %NULL at the beginning and one task
 281         * performs "p = mempool_alloc(...);" while another task is doing
 282         * "while (!p) cpu_relax(); mempool_free(p, ...);".  This function
 283         * may end up using curr_nr value which is from before allocation
 284         * of @p without the following rmb.
 285         */
 286        smp_rmb();
 287
 288        /*
 289         * For correctness, we need a test which is guaranteed to trigger
 290         * if curr_nr + #allocated == min_nr.  Testing curr_nr < min_nr
 291         * without locking achieves that and refilling as soon as possible
 292         * is desirable.
 293         *
 294         * Because curr_nr visible here is always a value after the
 295         * allocation of @element, any task which decremented curr_nr below
 296         * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets
 297         * incremented to min_nr afterwards.  If curr_nr gets incremented
 298         * to min_nr after the allocation of @element, the elements
 299         * allocated after that are subject to the same guarantee.
 300         *
 301         * Waiters happen iff curr_nr is 0 and the above guarantee also
 302         * ensures that there will be frees which return elements to the
 303         * pool waking up the waiters.
 304         */
 305        if (pool->curr_nr < pool->min_nr) {
 306                spin_lock_irqsave(&pool->lock, flags);
 307                if (pool->curr_nr < pool->min_nr) {
 308                        add_element(pool, element);
 309                        spin_unlock_irqrestore(&pool->lock, flags);
 310                        wake_up(&pool->wait);
 311                        return;
 312                }
 313                spin_unlock_irqrestore(&pool->lock, flags);
 314        }
 315        pool->free(element, pool->pool_data);
 316}
 317EXPORT_SYMBOL(mempool_free);
 318
 319/*
 320 * A commonly used alloc and free fn.
 321 */
 322void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
 323{
 324        struct kmem_cache *mem = pool_data;
 325        return kmem_cache_alloc(mem, gfp_mask);
 326}
 327EXPORT_SYMBOL(mempool_alloc_slab);
 328
 329void mempool_free_slab(void *element, void *pool_data)
 330{
 331        struct kmem_cache *mem = pool_data;
 332        kmem_cache_free(mem, element);
 333}
 334EXPORT_SYMBOL(mempool_free_slab);
 335
 336/*
 337 * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory
 338 * specified by pool_data
 339 */
 340void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
 341{
 342        size_t size = (size_t)pool_data;
 343        return kmalloc(size, gfp_mask);
 344}
 345EXPORT_SYMBOL(mempool_kmalloc);
 346
 347void mempool_kfree(void *element, void *pool_data)
 348{
 349        kfree(element);
 350}
 351EXPORT_SYMBOL(mempool_kfree);
 352
 353/*
 354 * A simple mempool-backed page allocator that allocates pages
 355 * of the order specified by pool_data.
 356 */
 357void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
 358{
 359        int order = (int)(long)pool_data;
 360        return alloc_pages(gfp_mask, order);
 361}
 362EXPORT_SYMBOL(mempool_alloc_pages);
 363
 364void mempool_free_pages(void *element, void *pool_data)
 365{
 366        int order = (int)(long)pool_data;
 367        __free_pages(element, order);
 368}
 369EXPORT_SYMBOL(mempool_free_pages);
 370
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.