linux/mm/percpu.c
<<
>>
Prefs
   1/*
   2 * mm/percpu.c - percpu memory allocator
   3 *
   4 * Copyright (C) 2009           SUSE Linux Products GmbH
   5 * Copyright (C) 2009           Tejun Heo <tj@kernel.org>
   6 *
   7 * This file is released under the GPLv2.
   8 *
   9 * This is percpu allocator which can handle both static and dynamic
  10 * areas.  Percpu areas are allocated in chunks.  Each chunk is
  11 * consisted of boot-time determined number of units and the first
  12 * chunk is used for static percpu variables in the kernel image
  13 * (special boot time alloc/init handling necessary as these areas
  14 * need to be brought up before allocation services are running).
  15 * Unit grows as necessary and all units grow or shrink in unison.
  16 * When a chunk is filled up, another chunk is allocated.
  17 *
  18 *  c0                           c1                         c2
  19 *  -------------------          -------------------        ------------
  20 * | u0 | u1 | u2 | u3 |        | u0 | u1 | u2 | u3 |      | u0 | u1 | u
  21 *  -------------------  ......  -------------------  ....  ------------
  22 *
  23 * Allocation is done in offset-size areas of single unit space.  Ie,
  24 * an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0,
  25 * c1:u1, c1:u2 and c1:u3.  On UMA, units corresponds directly to
  26 * cpus.  On NUMA, the mapping can be non-linear and even sparse.
  27 * Percpu access can be done by configuring percpu base registers
  28 * according to cpu to unit mapping and pcpu_unit_size.
  29 *
  30 * There are usually many small percpu allocations many of them being
  31 * as small as 4 bytes.  The allocator organizes chunks into lists
  32 * according to free size and tries to allocate from the fullest one.
  33 * Each chunk keeps the maximum contiguous area size hint which is
  34 * guaranteed to be equal to or larger than the maximum contiguous
  35 * area in the chunk.  This helps the allocator not to iterate the
  36 * chunk maps unnecessarily.
  37 *
  38 * Allocation state in each chunk is kept using an array of integers
  39 * on chunk->map.  A positive value in the map represents a free
  40 * region and negative allocated.  Allocation inside a chunk is done
  41 * by scanning this map sequentially and serving the first matching
  42 * entry.  This is mostly copied from the percpu_modalloc() allocator.
  43 * Chunks can be determined from the address using the index field
  44 * in the page struct. The index field contains a pointer to the chunk.
  45 *
  46 * To use this allocator, arch code should do the followings.
  47 *
  48 * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
  49 *   regular address to percpu pointer and back if they need to be
  50 *   different from the default
  51 *
  52 * - use pcpu_setup_first_chunk() during percpu area initialization to
  53 *   setup the first chunk containing the kernel static percpu area
  54 */
  55
  56#include <linux/bitmap.h>
  57#include <linux/bootmem.h>
  58#include <linux/err.h>
  59#include <linux/list.h>
  60#include <linux/log2.h>
  61#include <linux/mm.h>
  62#include <linux/module.h>
  63#include <linux/mutex.h>
  64#include <linux/percpu.h>
  65#include <linux/pfn.h>
  66#include <linux/slab.h>
  67#include <linux/spinlock.h>
  68#include <linux/vmalloc.h>
  69#include <linux/workqueue.h>
  70#include <linux/kmemleak.h>
  71
  72#include <asm/cacheflush.h>
  73#include <asm/sections.h>
  74#include <asm/tlbflush.h>
  75#include <asm/io.h>
  76
  77#define PCPU_SLOT_BASE_SHIFT            5       /* 1-31 shares the same slot */
  78#define PCPU_DFL_MAP_ALLOC="L78" class="line" naa>
  76
  70f8  76
   1  72fn8  76
  78#define   (efine   )                         c11111111111111\2="L83">  7e   78#define="L83"__addr_to_pc_ALLO="L83"PCPU_*)((unsignks long)dr_to_pc">  (efine   7eef="mm/p8rcpu.c#L75" id="L75" cl8ss="l85PCPU_DFL_MAP_ALLOCCCCCCCCCCCCC(unsignks long)r_to_pc">  (efi="efionfi_ne   7eef="mm/p8rcpu.c#L76" id="L76" cl8ss="l86PCPU_DFL_MAP_ALLOCCCCCCCCCCCCC(unsignks long)r_to_pc">  (efiLO="L_ommenass=__addr_to_pc_ALLO="L_ommenass= clas11111111111\2="L83">  7eef="mm/p8rcpu.c#L77" id="L77" cl8ss="l8ne" naendif11111111111\2="L83">  7eref="mm/ppercpu.c#L8" id="L8" class="l88e" n8me="L82">  72fn8  76
  78#define_pcpu_ptr() and __addr_to_pc_ALLO=pcpu_ptr() and __addr_to_pc">  (efine __addr_to_pc_ALne __adss="ine ">  )                         c111111111111111\2="L83">  79ef="mm/p9rc8u.c#L70"8id="L70" cl9ss="l90PCPU_DFL_MAP(void3">  78#defineforce__addr_to_pc_ALLOforcePCPU_*)((unsignks long)dr_to_pc">  (efine __addr_to_pc_ALne __adss+            c11      c111111111111111\2="L83">  79ef="mm/p9rc8u.c#L71"8id="L71" cl9ss="l91PCPU_DFL_MAP_ALLOCCCCCCCCCCCC(unsignks long)r_to_pc">  (efi="efionfi_ne   79ef="mm/p9rc8u.c#L72"8id="L72" cl9ss="l92PCPU_DFL_MAP_ALLOCCCCCCCCCCCC(unsignks long)r_to_pc">  (efiLO="L_ommenass=__addr_to_pc_ALLO="L_ommenass= clas11111111111\2="L83">  79ef="mm/p9rc8u.c#L73"8id="L73" cl9ss="l93e" naendif11111111111\2="L83">  79 
  45  78#define   (efine   78#define="L83"__addr_to_pc_ALLO="L83"PCPU_*)(r_to_pc">  (efine 
  77#define_pcpu_ptr() and __addr_to_pc_ALLO=pcpu_ptr() and __addr_to_pc">  (efine __addr_to_pc_ALne __adss="ine ">(void3">  78#defineforce__addr_to_pc_ALLOforcePCPU_*)(r_to_pc">  (efine __addr_to_pc_ALne __adsece slot */

  76
 * 3">  78#defi=pcpu  seta href="ine _AL="efi  setPCPU_{7me="L79">  76
 * 3">  78#defilass_    a href="ine _ALlass_    PCPU_DFL_MAP">  78#defilassa href="ine _ALlassPCPU;CCCCCCCCCCClass="line" naa>
  78#defi mapping a href="ine _AL mapping PCPU;CCCCCClass="line" naa> * n c1 o"comment"> * baws as nece slot */
  78#defi than _tigua href="ine _AL than _tiguPCPU;CCCClass="line" naa>
  78#defionfi_ne  *  alloisnt"> * baws as nece slot */
  78#defiun 463
  v2k9:-ex+*" met-span>

  
  
  
  
T,>#original LXR softw"re bygl,>#)tmm/pe4shttp://sourceeptge.net/projects/lxr">LXR a h1/a> ya h/"dthislexlini10efal versstd byg)tmm/pe4ssailto:lxr@taiux.no">lxr@taiux.noa h/.
lxr.taiux.no kindly hos>filbyg)tmm/pe4shttp://www.redpill-taipro.no">Redpill Laipro ASa h/"dprovider of LaiuxLconsultingl"nd ocoma>stds servicesgsince 1995.