linux/lib/cpu_rmap.c
<<
>>
Prefs
   1/*
   2 * cpu_rmap.c: CPU affinity reverse-map support
   3 * Copyright 2011 Solarflare Communications Inc.
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms of the GNU General Public License version 2 as published
   7 * by the Free Software Foundation, incorporated herein by reference.
   8 */
   9
  10#include <linux/cpu_rmap.h>
  11#ifdef CONFIG_GENERIC_HARDIRQS
  12#include <linux/interrupt.h>
  13#endif
  14#include <linux/export.h>
  15
  16/*
  17 * These functions maintain a mapping from CPUs to some ordered set of
  18 * objects with CPU affinities.  This can be seen as a reverse-map of
  19 * CPU affinity.  However, we do not assume that the object affinities
  20 * cover all CPUs in the system.  For those CPUs not directly covered
  21 * by object affinities, we attempt to find a nearest object based on
  22 * CPU topology.
  23 */
  24
  25/**
  26 * alloc_cpu_rmap - allocate CPU affinity reverse-map
  27 * @size: Number of objects to be mapped
  28 * @flags: Allocation flags e.g. %GFP_KERNEL
  29 */
  30struct cpu_rmap *alloc_cpu_rmap(unsigned int size, gfp_t flags)
  31{
  32        struct cpu_rmap *rmap;
  33        unsigned int cpu;
  34        size_t obj_offset;
  35
  36        /* This is a silly number of objects, and we use u16 indices. */
  37        if (size > 0xffff)
  38                return NULL;
  39
  40        /* Offset of object pointer array from base structure */
  41        obj_offset = ALIGN(offsetof(struct cpu_rmap, near[nr_cpu_ids]),
  42                           sizeof(void *));
  43
  44        rmap = kzalloc(obj_offset + size * sizeof(rmap->obj[0]), flags);
  45        if (!rmap)
  46                return NULL;
  47
  48        kref_init(&rmap->refcount);
  49        rmap->obj = (void **)((char *)rmap + obj_offset);
  50
  51        /* Initially assign CPUs to objects on a rota, since we have
  52         * no idea where the objects are.  Use infinite distance, so
  53         * any object with known distance is preferable.  Include the
  54         * CPUs that are not present/online, since we definitely want
  55         * any newly-hotplugged CPUs to have some object assigned.
  56         */
  57        for_each_possible_cpu(cpu) {
  58                rmap->near[cpu].index = cpu % size;
  59                rmap->near[cpu].dist = CPU_RMAP_DIST_INF;
  60        }
  61
  62        rmap->size = size;
  63        return rmap;
  64}
  65EXPORT_SYMBOL(alloc_cpu_rmap);
  66
  67/**
  68 * cpu_rmap_release - internal reclaiming helper called from kref_put
  69 * @ref: kref to struct cpu_rmap
  70 */
  71static void cpu_rmap_release(struct kref *ref)
  72{
  73        struct cpu_rmap *rmap = container_of(ref, struct cpu_rmap, refcount);
  74        kfree(rmap);
  75}
  76
  77/**
  78 * cpu_rmap_get - internal helper to get new ref on a cpu_rmap
  79 * @rmap: reverse-map allocated with alloc_cpu_rmap()
  80 */
  81static inline void cpu_rmap_get(struct cpu_rmap *rmap)
  82{
  83        kref_get(&rmap->refcount);
  84}
  85
  86/**
  87 * cpu_rmap_put - release ref on a cpu_rmap
  88 * @rmap: reverse-map allocated with alloc_cpu_rmap()
  89 */
  90int cpu_rmap_put(struct cpu_rmap *rmap)
  91{
  92        return kref_put(&rmap->refcount, cpu_rmap_release);
  93}
  94EXPORT_SYMBOL(cpu_rmap_put);
  95
  96/* Reevaluate nearest object for given CPU, comparing with the given
  97 * neighbours at the given distance.
  98 */
  99static bool cpu_rmap_copy_neigh(struct cpu_rmap *rmap, unsigned int cpu,
 100                                const struct cpumask *mask, u16 dist)
 101{
 102        int neigh;
 103
 104        for_each_cpu(neigh, mask) {
 105                if (rmap->near[cpu].dist > dist &&
 106                    rmap->near[neigh].dist <= dist) {
 107                        rmap->near[cpu].index = rmap->near[neigh].index;
 108                        rmap->near[cpu].dist = dist;
 109                        return true;
 110                }
 111        }
 112        return false;
 113}
 114
 115#ifdef DEBUG
 116static void debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix)
 117{
 118        unsigned index;
 119        unsigned int cpu;
 120
 121        pr_info("cpu_rmap %p, %s:\n", rmap, prefix);
 122
 123        for_each_possible_cpu(cpu) {
 124                index = rmap->near[  77cpu_rmap_r">  77<5>
  93}
pr_info("cpu_rmap %p, %span>,d e=neaCPU,span>,u (able.  Inpan>,u)a href="+code=rm  77<5>
  26index = rmap, near[cpu].dist =   27  28  29  30struct1 inline void   31{
debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix)
  32      1  str13b/cpu_rmap.c#L83" id="L83" class13"line" namme="L13">  13#endif
<   34      1    35
  36      1  /**
  37      1  if 13comment"> * cpu_rmap_put - release ref addfiniddfPU, comtohave="lib/cpu_rmap.c#L88" id="L88" class1="line" n1ame="L38">  38      1     13comment"> * @rmap: reverse-map allocase-mr alloc_cpu_rmap()
  39
 * @rmap: reverse-map PU,: OU, comtohaddftohe="lib/cpu_rmap.c#L88" id="L88" class1="line" n1ame="L40">  40      1   */
  41      1   * by object affinitiRde=falf="+c array frolib/cpu_rmap.c#L98" id="L98" class1="line" n1ame="L42">  42      1     14comment"> * CPU topology.  43
add_rmap_copy_neigh(straddf="+code=cpu_rmap" class="sref">cpu_rmap *rmap, unsigned int  = (void **)((char_rmap.c#L117" id="L117" ass=="line" n1ame="L44">  44      1    45      1  if 1!u16 cpu_rmap_r">  77<5>
  46      1     14/cpu_rmap.c#L77" id="L77" class1="line" n1ame="L47">  47
(>rmap, rmap->size;
  48      1  index = rmap->  49      1  rmap->obj = (void **)((charu" class="sref">">index =  = (void **)((char_rmap.c#L15" id="L15" class=""line" n1ame="L50">  50
cpu_rmap_r">  77<5>
  51      1    52EXPORT_SYMBOL(add_rmap_copy_neigh(straddf="+u_rmap.c#L122" id="L122" clas="line" n1ame="L53">  53  54         * CPUs th="lib/cpu_rmap.c#L87" id="L87" class1="line" n1ame="L55">  55         * any newlelease ref upd for- upd forse-mr allfopanw  56         */
ap allocase-mr alltohupd foib/cpu_rmap.c#L87" id="L87" class1="line" n1ame="L57">  57      1   * @size: Number of obu_rma: I="+c array fro wctly apde=ngelib/cpu_rmap.c#L28" id="L28" class1="line" n1ame="L58">  58      1     15comment"> * @rmap: reverse-map aparray froib/cpu_rmap.c#L28" id="L28" class1="line" n1ame="L59">  59      1     15comment"> */
  60      1  }
<16+code=cpu_rmap_put" class="sref">upd fo_rmap_copy_neigh(strupd foib"+code=cpu_rmap" class="sref">cpu_rmap *rmap, unsigned int u16 cpu_rmap_r"m  77<5>
  61
cpumask * *  62      1    63      1  ret16ef="+code=for_each_possible_ef="+co_varize_t cpumask *cpup_r">  77<5>
  64}
cpu;
  65  16 *
 *
cpumask *cpup_r"hr" class="sref">  67  68  69/* Initially assign CPUvalid forable.  Inparistem.  Fopariwhic

  70 */
  71static1 void17comment"> * by object affini========*lib/cpu_rmap.c#L90" id="L90" class1="line" n1ame="L72">  72{
(for_each_possible_cpu(fore=cpu" class="sref">cpu) {
  73      1  str1ct rmap, near[cpu].index = index =   74      1  rmap->>near[cpu].dist = CPU_RMAP_DIST_INF;
  75}
for_each_possible_ef="+co_set>fore=cpu" class="sref">cpu) {
upd fo_>cpumask *cpup_r"u_rmap.c#L122" id="L122" clas="line" n1ame="L76">  76
  77  78  79debug_print_rmap(const struct rmap, unsigned int "cpu_rmap %p, &afe stiUvalid f  80  81static1 /* Initially assign CSetrable.  In>
<0nparistem.  For thosemap>cpulib/cpu_rmap.c#L98" id="L98" class1="line" n1ame="L82">  82{
         * no idea where the Markrstem.  Fop()  83      1           * any object with klib/cpu_rmap.c#L90" id="L90" class1="line" n1ame="L84">  84}
for_each_cpu(ap *  85
>near[cpu].index = cpu_rmap_r">  77<5>
  86>near[cpu].dist =   87[upd fo_>cpumask *cpup_r"h  77<5>
  18cpu) {
  89  90int   91{
debug_print_rmap(const struct rmap, unsigned int "cpu_rmap %p, &afe stupd f  92      1  ret19/cpu_rmap.c#L123" id="L123" clas="line" n1ame="L93">  93}
/* Initially assign CUpd forable.  Isa href="l  94for_each_cpu(upd fo_>cpumask *cpup_r"uu_rmap.c#L124" id="L124" clas="line" n1ame="L95">  95
(str"sref">cpu_rmap_copy_neigh(struct rmap, unsigned int cpu) {
  96) {
cpu) {
  97  98(str"sref">cpu_rmap_copy_neigh(struct rmap, unsigned int cpu) {
  19) {
cpu) {
 100     2     2                    const strua hrinue_rmap.c#L122" id="L122" cla2s="line" 2ame="L101"> 101{
(str"sref">cpu_rmap_copy_neigh(struct rmap, unsigned int cpu) {
 102     2  int20              sizeof(void *));
<"+code=rma="+_each_possible_ef="+co_of_nrefr[cpu) {
 103
 104     2  /* Initially assign CWeua uldua hrinuee=cpo NUMA nrefrable.  Is, buomparinowib/cpu_rmap.c#L20" id="L20" class="5line" naame="L95">  95
         * any newly-hotpluuuuuuuuu*ttuplib/cpu_rmap.c#L23" id="L23" class=s="line" 2ame="L106"> 106     2     20comment">         */
 107     2     20            108     2     20commermap.c#L120" id="L120" cla2s="line" 2ame="L109"> 109     2     20ef="+code=rmap" class="sref"class="sref">debug_print_rmap(const struct rmap, unsigned int "cpu_rmap %p, &afe stuct  110     2     21/cpu_rmap.c#L121" id="L121" cla2s="line" 2ame="L111"> 111     2  }
<21ef="+code=pr_info" class="sr+cod
 * 112     2  ret2rn  113}
 114
EXPORT_SYMBOL(upd fo_rmap_copy_neigh(strupd foib"+u_rmap.c#L122" id="L122" cla2s5line" naame="L115"> 115#ifde2  116stati2 void2ode=diode=DEBUG" class="srefCONFIG_GENERIC_HARDIRQS_RMAP_DIST_INF 117{
 118     2  uns21comment"> * @rmap: reverse-n CGlueebetween IRQnotifiestanndase-mr alsu*lib/cpu_rmap.c#L24" id="L24" class=s="line" 2ame="L119"> 119     2  uns21/cpu_rmap.c#L40" id="L40" class2s="line" 2ame="L120"> 120
 121     2  >otifnmask * 122
cpu_rmap *rmap;
 123     2  cpu_rmap_r">  77<5>
 124     2     2      77<5>
  93}
  26/**
  27 * @size: Number of o+cod
irq_="L87" c -o+cod arse-mapted with all/  28 * @rmap: reverse-map allocaRed with alloc_cpu_rmap()
irq_="L8mf="l, aripan>,"lib/c/cpu_rmap.c#L87" id="L87" class2s="line" 2ame="L29">  29 */
  30struct2  */
  31{
 * by object affinitlib/cpu_rmap.c#L24" id="L24" class="="line" 2ame="L32">  32      2  str23b/cpue=debug_print_rmap" +cod
irq_="L87" cmask *cpu_rmap *rmap)
  13#endif2<   34      2  ;
  77<5>
  35
u16 cpu_rmap_r">  77<5>
  36      2    37      2  if 23           rmap)
  38      2     23             39
  40      2  index = index = rmap->index =   41      2  ;
rmap-> = (void **)((charu" class="sref">">index =   42      2     24              sizeof(_rmap" class="sirq_set>ap;
ap;
>otifnmask *;
>NULL;
  43
  44      2    45      2  if 2!h(strcpu_rmap_put(struct rmap)
  46      2     24/cpu__rmap.c#L114" id="L114" cla2="line" n2ame="L47">  47
EXPORT_SYMBOL( *  48      2    49      2   */
  50
 */
upd folib/cpu_rmap.c#L28" id="L28" class2="line" n2ame="L51">  51      2   * by object affiniti@notifn:_ode=cpuirq_ap  52         * no idea ti@>cpu:leas>>cpumparinew SMPmap  53         * any objelib/cpu_rmap.c#L87" id="L87" class2""line" n2ame="L54">  54         * CPUs tha* T is*is*executspain workqueueua hrextlib/cpu_rmap.c#L23" id="L23" class=="line" n2ame="L55">  55         * any newlelib/cpu_rmap.c#L24" id="L24" class=="line" n2ame="L56">  56  57      2  ;
otifnmask *mask) {
  58      2     25comme_rmap.c#L124" id="L124" cla2="line" n2ame="L59">  59      2     2    ;
  60      2  }
<26                   co_each_possible_e="sref">container_of(>otifnmask *>otifnmask *  61
rcp)
  77<5>
  62      2    63      2  ret26ef="+code=for_each_possible_rcp)
upd fo_rmap_copy_neigh(strupd foib"+u_rmap_put" clasglrue;
>rmap, unsigned int glrue;
">index = mask) {
  64}
cp)
  65)
"cpu_rmap %p, &irq_="L87" c_notifn: upd forfailed:span>,da href="+code=rmap" class="sref">cp)
  77<5>
  16  67  68 * cpu_rmap_releas="lib/cpu_rmap.c#L87" id="L87" class2="line" n2ame="L69">  69 * @ref: kref to struirq_="L87" c_pu_rmap< cpu @rim  70 */
  71static2 void27comment"> * by object affinielib/cpu_rmap.c#L24" id="L24" class=="line" n2ame="L72">  72{
;
 he;
  73      2  str27b/cpu_rmap.c#L124" id="L124" cla2="line" n2ame="L74">  74      2  ;
  75}
container_of(> he;
>otifnmask *;
  77<5>
  76
  77;
>rmap, unsigned int   77<5>
  78;
  77<5>
  79  80  81static2  * by object affin="lib/cpu_rmap.c#L87" id="L87" class2="line" n2ame="L82">  82{
         * no idea ruirq_="L87" c_addfiniddfan IRQted with alib/cpu_rmap.c#L87" id="L87" class2="line" n2ame="L83">  83      2           * any objeap allocaThe>ted with alib/cpu_rmap.c#L87" id="L87" class2="line" n2ame="L84">  84}
         * CPUs tha* @irqcaThe>IRQ  85
         * any newlee=/cpu_rmap.c#L87" id="L87" class2="line" n2ame="L86">  86/**
notifies that will upd forthe>ted with alib/cpu_rmap.c#L87" id="L87" class2="line" n2ame="L87">  87 * cpu_rmap_put - relautom"+cot  18 * @rmap: reverse-maib/cpu_rmap.c#L23" id="L23" class=="line" n2ame="L89">  89 */
  90int  */
  91{
 * by object affinielib/cpu_rmap.c#L24" id="L24" class=="line" n2ame="L92">  92      2  ret29/cpu_=cpu" class="sref">irq_="L87" c_adde;
cpu_rmap *rmap, unsigned int irqe;
  93}
  94;
;
  77<5>
  95
rcp)
  77<5>
  96  97;
  98  19;
>otifnmask *otifnmask *;
 100     3     3              p" class="sref"glrue;
>otifnmask *;
;
  91{
>rmap, unsigned int rmap->  92      3  int30ef="+code=rmap" class="sref"="L87" c_geu_rmap_put(struct rmap)
 103
;
">index = add_rmap_copy_neigh(straddf="+c" class="sref">>rmap, unsigned int glrue;
  94)
ap;
ap;
;
>otifnmask *  95
cp)
 106     3     30       ;
>rmap, unsigned int   77<5>
 107     3     30           ;
  77<5>
 108     3     3             109     3     30ef="+code=rmaode=false" class="srefrcp)
  77<5>
 110     3     31/cpu__rmap.c#L114" id="L114" cla3s="line" 3ame="L111"> 111     3  }
<31b/cpuug_print_rmap" ss="sref">EXPORT_SYMBOL(;
 112     3  ret31/cpu_rmap.c#L123" id="L123" cla3s="line" 3ame="L113"> 113}
/* Initially assign C/ONFIG_GENERIC_HARDIRQSielib/cpu_rmap.c#L24" id="L24" class3s="line" 3ame="L114"> 114
footese>
The>original LXR softwareabyuhosemap.c#L24http://sourceparge.net/pro, cos/lxse>LXR lly uspanibar,lxs@ 11ux.nof="+s
O/div>
subfootese>
lxs. 11ux.no kf="ly hostspabyumap.c#L24http://www.redpill- 11pro.no">Redpill L11pro ASibar,arrL11uxef="+ulf