linux/mm/percpu.c
<<
>>
Prefs
   1/*
   2 * mm/percpu.c - percpu memory allocator
   3 *
   4 * Copyright (C) 2009           SUSE Linux Products GmbH
   5 * Copyright (C) 2009           Tejun Heo <tj@kernel.org>
   6 *
   7 * This file is released under the GPLv2.
   8 *
   9 * This is percpu allocator which can handle both static and dynamic
  10 * areas.  Percpu areas are allocated in chunks.  Each chunk is
  11 * consisted of boot-time determined number of units and the first
  12 * chunk is used for static percpu variables in the kernel image
  13 * (special boot time alloc/init handling necessary as these areas
  14 * need to be brought up before allocation services are running).
  15 * Unit grows as necessary and all units grow or shrink in unison.
  16 * When a chunk is filled up, another chunk is allocated.
  17 *
  18 *  c0                           c1                         c2
  19 *  -------------------          -------------------        ------------
  20 * | u0 | u1 | u2 | u3 |        | u0 | u1 | u2 | u3 |      | u0 | u1 | u
  21 *  -------------------  ......  -------------------  ....  ------------
  22 *
  23 * Allocation is done in offset-size areas of single unit space.  Ie,
  24 * an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0,
  25 * c1:u1, c1:u2 and c1:u3.  On UMA, units corresponds directly to
  26 * cpus.  On NUMA, the mapping can be non-linear and even sparse.
  27 * Percpu access can be done by configuring percpu base registers
  28 * according to cpu to unit mapping and pcpu_unit_size.
  29 *
  30 * There are usually many small percpu allocations many of them being
  31 * as small as 4 bytes.  The allocator organizes chunks into lists
  32 * according to free size and tries to allocate from the fullest one.
  33 * Each chunk keeps the maximum contiguous area size hint which is
  34 * guaranteed to be equal to or larger than the maximum contiguous
  35 * area in the chunk.  This helps the allocator not to iterate the
  36 * chunk maps unnecessarily.
  37 *
  38 * Allocation state in each chunk is kept using an array of integers
  39 * on chunk->map.  A positive value in the map represents a free
  40 * region and negative allocated.  Allocation inside a chunk is done
  41 * by scanning this map sequentially and serving the first matching
  42 * entry.  This is mostly copied from the percpu_modalloc() allocator.
  43 * Chunks can be determined from the address using the index field
  44 * in the page struct. The index field contains a pointer to the chunk.
  45 *
  46 * To use this allocator, arch code should do the followings.
  47 *
  48 * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
  49 *   regular address to percpu pointer and back if they need to be
  50 *   different from the default
  51 *
  52 * - use pcpu_setup_first_chunk() during percpu area initialization to
  53 *   setup the first chunk containing the kernel static percpu area
  54 */
  55
  56#include <linux/bitmap.h>
  57#include <linux/bootmem.h>
  58#include <linux/err.h>
  59#include <linux/list.h>
  60#include <linux/log2.h>
  61#include <linux/mm.h>
  62#include <linux/module.h>
  63#include <linux/mutex.h>
  64#include <linux/percpu.h>
  65#include <linux/pfn.h>
  66#include <linux/slab.h>
  67#include <linux/spinlock.h>
  68#include <linux/vmalloc.h>
  69#include <linux/workqueue.h>
  70#include <linux/kmemleak.h>
  71
  72#include <asm/cacheflush.h>
  73#include <asm/sections.h>
  74#include <asm/tlbflush.h>
  75#include <asm/io.h>
  76
  77#define PCPU_SLOT_BASE_SHIFT            5       /* 1-31 shares the same slot */
  78#define PCPU_DFL_MAP_ALLOC              16      /* start a map with 16 ents */
  79
  80#ifdef CONFIG_SMP
  81/* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
  82#ifndef __addr_to_pcpu_ptr
  83#define __addr_to_pcpu_ptr(addr)                                        \
  84        (void __percpu *)((unsigned long)(addr) -                       \
  85                          (unsigned long)pcpu_base_addr +               \
  86                          (unsigned long)__per_cpu_start)
  87#endif
  88#ifndef __pcpu_ptr_to_addr
  89#define __pcpu_ptr_to_addr(ptr)                                         \
  90        (void __force *)((unsigned long)(ptr) +                         \
  91                         (unsigned long)pcpu_base_addr -                \
  92                         (unsigned long)__per_cpu_start)
  93#endif
  94#else   /* CONFIG_SMP */
  95/* on UP, it's always identity mapped */
  96#define __addr_to_pcpu_ptr(addr)        (void __percpu *)(addr)
  97#define __pcpu_ptr_to_addr(ptr)         (void __force *)(ptr)
  98#endif  /* CONFIG_SMP */
  99
 100struct pcpu_chunk {
 101        struct list_head        list;           /* linked to pcpu_slot lists */
 102        int                     free_size;      /* free bytes in the chunk */
 103        int                     contig_hint;    /* max contiguous size hint */
 104        void                    *base_addr;     /* base address of this chunk */
 105        int                     map_used;       /* # of map entries used */
 106        int                     map_alloc;      /* # of map entries allocated */
 107        int                     *map;           /* allocation map */
 108        void                    *data;          /* chunk data */
 109        bool                    immutable;      /* no [de]population allowed */
 110        unsigned long           populated[];    /* populated bitmap */
 111};
 112
 113static int pcpu_unit_pages __read_mostly;
 114static int pcpu_unit_size __read_mostly;
 115static int pcpu_nr_units __read_mostly;
 116static int pcpu_atom_size __read_mostly;
 117static int pcpu_nr_slots __read_mostly;
 118static size_t pcpu_chunk_struct_size __read_mostly;
 119
 120/* cpus with the lowest and highest unit addresses */
 121static unsigned int pcpu_low_unit_cpu __read_mostly;
 122static unsigned int pcpu_high_unit_cpu __read_mostly;
 123
 124/* the address of the first chunk which starts with the kernel static area */
 125void *pcpu_base_addr __read_mostly;
 126EXPORT_SYMBOL_GPL(pcpu_base_addr);
 127
 128static const int *pcpu_unit_map __read_mostly;          /* cpu -> unit */
 129const unsigned long *pcpu_unit_offsets __read_mostly;   /* cpu -> unit offset */
 130
 131/* group information, used for vm allocation */
 132static int pcpu_nr_groups __read_mostly;
 133static const unsigned long *pcpu_group_offsets __read_mostly;
 134static const size_t *pcpu_group_sizes __read_mostly;
 135
 136/*
 137 * The first chunk which always exists.  Note that unlike other
 138 * chunks, this one can be allocated and mapped in several different
 139 * ways and thus often doesn't live in the vmalloc area.
 140 */
 141static struct pcpu_chunk *pcpu_first_chunk;
 142
 143/*
 144 * Optional reserved chunk.  This chunk reserves part of the first
 145 * chunk and serves it for reserved allocations.  The amount of
 146 * reserved offset is in pcpu_reserved_chunk_limit.  When reserved
 147 * area doesn't exist, the following variables contain NULL and 0
 148 * respectively.
 149 */
 150static struct pcpu_chunk *pcpu_reserved_chunk;
 151static int pcpu_reserved_chunk_limit;
 152
 153/*
 154 * Synchronization rules.
 155 *
 156 * There are two locks - pcpu_alloc_mutex and pcpu_lock.  The former
 157 * protects allocation/reclaim paths, chunks, populated bitmap and
 158 * vmalloc mapping.  The latter is a spinlock and protects the index
 159 * data structures - chunk slots, chunks and area maps in chunks.
 160 *
 161 * During allocation, pcpu_alloc_mutex is kept locked all the time and
 162 * pcpu_lock is grabbed and released as necessary.  All actual memory
 163 * allocations are done using GFP_KERNEL with pcpu_lock released.  In
 164 * general, percpu memory can't be allocated with irq off but
 165 * irqsave/restore are still used in alloc path so that it can be used
 166 * from early init path - sched_init() specifically.
 167 *
 168 * Free path accesses and alters only the index data structures, so it
 169 * can be safely called from atomic context.  When memory needs to be
 170 * returned to the system, free path schedules reclaim_work which
 171 * grabs both pcpu_alloc_mutex and pcpu_lock, unlinks chunks to be
 172 * reclaimed, release both locks and frees the chunks.  Note that it's
 173 * necessary to grab both locks to remove a chunk from circulation as
 174 * allocation path might be referencing the chunk with only
 175 * pcpu_alloc_mutex locked.
 176 */
 177static DEFINE_MUTEX(pcpu_alloc_mutex);  /* protects whole alloc and reclaim */
 178static DEFINE_SPINLOCK(pcpu_lock);      /* protects index data structures */
 179
 180static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
 181
 182/* reclaim work to release fully free chunks, scheduled from free path */
 183static void pcpu_reclaim(struct work_struct *work);
 184static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim);
 185
 186static bool pcpu_addr_in_first_chunk(void *addr)
 187{
 188        void *first_start = pcpu_first_chunk->base_addr;
 189
 190        return addr >= first_start && addr < first_start + pcpu_unit_size;
 191}
 192
 193static bool pcpu_addr_in_reserved_chunk(void *addr)
 194{
 195        void *first_start = pcpu_first_chunk->base_addr;
 196
 197        return addr >= first_start &&
 198                addr < first_start + pcpu_reserved_chunk_limit;
 199}
 200
 201static int __pcpu_size_to_slot(int size)
 202{
 203        int highbit = fls(size);        /* size is in bytes */
 204        return max(highbit - PCPU_SLOT_BASE_SHIFT + 2, 1);
 205}
 206
 207static int pcpu_size_to_slot(int size)
 208{
 209        if (size == pcpu_unit_size)
 210                return pcpu_nr_slots - 1;
 211        return __pcpu_size_to_slot(size);
 212}
 213
 214static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
 215{
 216        if (chunk->free_size < sizeof(int) || chunk->contig_hint < sizeof(int))
 217                return 0;
 218
 219        return pcpu_size_to_slot(chunk->free_size);
 220}
 221
 222/* set the pointer to a chunk in a page struct */
 223static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu)
 224{
 225        page->index = (unsigned long)pcpu;
 226}
 227
 228/* obtain pointer to a chunk from a page struct */
 229static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page)
 230{
 231        return (struct pcpu_chunk *)page->index;
 232}
 233
 234static int __maybe_unused pcpu_page_idx(unsigned int cpu, int page_idx)
 235{
 236        return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
 237}
 238
 239static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
 240                                     unsigned int cpu, int page_idx)
 241{
 242        return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] +
 243                (page_idx << PAGE_SHIFT);
 244}
 245
 246static void __maybe_unused pcpu_next_unpop(struct pcpu_chunk *chunk,
 247                                           int *rs, int *re, int end)
 248{
 249        *rs = find_next_zero_bit(chunk->populated, end, *rs);
 250        *re = find_next_bit(chunk->populated, end, *rs + 1);
 251}
 252
 253static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk,
 254                                         int *rs, int *re, int end)
 255{
 256        *rs = find_next_bit(chunk->populated, end, *rs);
 257        *re = find_next_zero_bit(chunk->populated, end, *rs + 1);
 258}
 259
 260/*
 261 * (Un)populated page region iterators.  Iterate over (un)populated
 262 * page regions between @start and @end in @chunk.  @rs and @re should
 263 * be integer variables and will be set to start and end page index of
 264 * the current region.
 265 */
 266#define pcpu_for_each_unpop_region(chunk, rs, re, start, end)               \
 267        for ((rs) = (start), pcpu_next_unpop((chunk), &(rs), &(re), (end)); \
 268             (rs) < (re);                                                   \
 269             (rs) = (re) + 1, pcpu_next_unpop((chunk), &(rs), &(re), (end)))
 270
 271#define pcpu_for_each_pop_region(chunk, rs, re, start, end)                 \
 272        for ((rs) = (start), pcpu_next_pop((chunk), &(rs), &(re), (end));   \
 273             (rs) < (re);                                                   \
 274             (rs) = (re) + 1, pcpu_next_pop((chunk), &(rs), &(re), (end)))
 275
 276/**
 277 * pcpu_mem_zalloc - allocate memory
 278 * @size: bytes to allocate
 279 *
 280 * Allocate @size bytes.  If @size is smaller than PAGE_SIZE,
 281 * kzalloc() is used; otherwise, vzalloc() is used.  The returned
 282 * memory is always zeroed.
 283 *
 284 * CONTEXT:
 285 * Does GFP_KERNEL allocation.
 286 *
 287 * RETURNS:
 288 * Pointer to the allocated area on success, NULL on failure.
 289 */
 290static void *pcpu_mem_zalloc(size_t size)
 291{
 292        if (WARN_ON_ONCE(!slab_is_available()))
 293                return NULL;
 294
 295        if (size <= PAGE_SIZE)
 296                return kzalloc(size, GFP_KERNEL);
 297        else
 298                return vzalloc(size);
 299}
 300
 301/**
 302 * pcpu_mem_free - free memory
 303 * @ptr: memory to free
 304 * @size: size of the area
 305 *
 306 * Free @ptr.  @ptr should have been allocated using pcpu_mem_zalloc().
 307 */
 308static void pcpu_mem_free(void *ptr, size_t size)
 309{
 310        if (size <= PAGE_SIZE)
 311                kfree(ptr);
 312        else
 313                vfree(ptr);
 314}
 315
 316/**
 317 * pcpu_chunk_relocate - put chunk in the appropriate chunk slot
 318 * @chunk: chunk of interest
 319 * @oslot: the previous slot it was on
 320 *
 321 * This function is called after an allocation or free changed @chunk.
 322 * New slot according to the changed state is determined and @chunk is
 323 * moved to the slot.  Note that the reserved chunk is never put on
 324 * chunk slots.
 325 *
 326 * CONTEXT:
 327 * pcpu_lock.
 328 */
 329static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
 330{
 331        int nslot = pcpu_chunk_slot(chunk);
 332
 333        if (chunk != pcpu_reserved_chunk && oslot != nslot) {
 334                if (oslot < nslot)
 335                        list_move(&chunk->list, &pcpu_slot[nslot]);
 336                else
 337                        list_move_tail(&chunk->list, &pcpu_slot[nslot]);
 338        }
 339}
 340
 341/**
 342 * pcpu_need_to_extend - determine whether chunk area map needs to be extended
 343 * @chunk: chunk of interest
 344 *
 345 * Determine whether area map of @chunk needs to be extended to
 346 * accommodate a new allocation.
 347 *
 348 * CONTEXT:
 349 * pcpu_lock.
 350 *
 351 * RETURNS:
 352 * New target map allocation length if extension is necessary, 0
 353 * otherwise.
 354 */
 355static int pcpu_need_to_extend(struct pcpu_chunk *chunk)
 356{
 357        int new_alloc;
 358
 359        if (chunk->map_alloc >= chunk->map_used + 2)
 360                return 0;
 361
 362        new_alloc = PCPU_DFL_MAP_ALLOC;
 363        while (new_alloc < chunk->map_used + 2)
 364                new_alloc *= 2;
 365
 366        return new_alloc;
 367}
 368
 369/**
 370 * pcpu_extend_area_map - extend area map of a chunk
 371 * @chunk: chunk of interest
 372 * @new_alloc: new target allocation length of the area map
 373 *
 374 * Extend area map of @chunk to have @new_alloc entries.
 375 *
 376 * CONTEXT:
 377 * Does GFP_KERNEL allocation.  Grabs and releases pcpu_lock.
 378 *
 379 * RETURNS:
 380 * 0 on success, -errno on failure.
 381 */
 382static int pcpu_extend_area_map(struct pcpu_chunk *chunk, int new_alloc)
 383{
 384        int *old = NULL, *new = NULL;
 385        size_t old_size = 0, new_size = new_alloc * sizeof(new[0]);
 386        unsigned long flags;
 387
 388        new = pcpu_mem_zalloc(new_size);
 389        if (!new)
 390                return -ENOMEM;
 391
 392        /* acquire pcpu_lock and switch to new area map */
 393        spin_lock_irqsave(&pcpu_lock, flags);
 394
 395        if (new_alloc <= chunk->map_alloc)
 396                goto out_unlock;
 397
 398        old_size = chunk->map_alloc * sizeof(chunk->map[0]);
 399        old = chunk->map;
 400
 401        memcpy(new, old, old_size);
 402
 403        chunk->map_alloc = new_alloc;
 404        chunk->map = new;
 405        new = NULL;
 406
 407out_unlock:
 408        spin_unlock_irqrestore(&pcpu_lock, flags);
 409
 410        /*
 411         * pcpu_mem_free() might end up calling vfree() which uses
 412         * IRQ-unsafe lock and thus can't be called under pcpu_lock.
 413         */
 414        pcpu_mem_free(old, old_size);
 415        pcpu_mem_free(new, new_size);
 416
 417        return 0;
 418}
 419
 420/**
 421 * pcpu_split_block - split a map block
 422 * @chunk: chunk of interest
 423 * @i: index of map block to split
 424 * @head: head size in bytes (can be 0)
 425 * @tail: tail size in bytes (can be 0)
 426 *
 427 * Split the @i'th map block into two or three blocks.  If @head is
 428 * non-zero, @head bytes block is inserted before block @i moving it
 429 * to @i+1 and reducing its size by @head bytes.
 430 *
 431 * If @tail is non-zero, the target block, which can be @i or @i+1
 432 * depending on @head, is reduced by @tail bytes and @tail byte block
 433 * is inserted after the target block.
 434 *
 435 * @chunk->map must have enough free slots to accommodate the split.
 436 *
 437 * CONTEXT:
 438 * pcpu_lock.
 439 */
 440static void pcpu_split_block(struct pcpu_chunk *chunk, int i,
 441                             int head, int tail)
 442{
 443        int nr_extra = !!head + !!tail;
 444
 445        BUG_ON(chunk->map_alloc < chunk->map_used + nr_extra);
 446
 447        /* insert new subblocks */
 448        memmove(&chunk->map[i + nr_extra], &chunk->map[i],
 449                sizeof(chunk->map[0]) * (chunk->map_used - i));
 450        chunk->map_used += nr_extra;
 451
 452        if (head) {
 453                chunk->map[i + 1] = chunk->map[i] - head;
 454                chunk->map[i++] = head;
 455        }
 456        if (tail) {
 457                chunk->map[i++] -= tail;
 458                chunk->map[i] = tail;
 459        }
 460}
 461
 462/**
 463 * pcpu_alloc_area - allocate area from a pcpu_chunk
 464 * @chunk: chunk of interest
 465 * @size: wanted size in bytes
 466 * @align: wanted align
 467 *
 468 * Try to allocate @size bytes area aligned at @align from @chunk.
 469 * Note that this function only allocates the offset.  It doesn't
 470 * populate or map the area.
 471 *
 472 * @chunk->map must have at least two free slots.
 473 *
 474 * CONTEXT:
 475 * pcpu_lock.
 476 *
 477 * RETURNS:
 478 * Allocated offset in @chunk on success, -1 if no matching area is
 479 * found.
 480 */
 481static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
 482{
 483        int oslot = pcpu_chunk_slot(chunk);
 484        int max_contig = 0;
 485        int i, off;
 486
 487        for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) {
 488                bool is_last = i + 1 == chunk->map_used;
 489                int head, tail;
 490
 491                /* extra for alignment requirement */
 492                head = ALIGN(off, align) - off;
 493                BUG_ON(i == 0 && head != 0);
 494
 495                if (chunk->map[i] < 0)
 496                        continue;
 497                if (chunk->map[i] < head + size) {
 498                        max_contig = max(chunk->map[i], max_contig);
 499                        continue;
 500                }
 501
 502                /*
 503                 * If head is small or the previous block is free,
 504                 * merge'em.  Note that 'small' is defined as smaller
 505                 * than sizeof(int), which is very small but isn't too
 506                 * uncommon for percpu allocations.
 507                 */
 508                if (head && (head < sizeof(int) || chunk->map[i - 1] > 0)) {
 509                        if (chunk->map[i - 1] > 0)
 510                                chunk->map[i - 1] += head;
 511                        else {
 512                                chunk->map[i - 1] -= head;
 513                                chunk->free_size -= head;
 514                        }
 515                        chunk->map[i] -= head;
 516                        off += head;
 517                        head = 0;
 518                }
 519
 520                /* if tail is small, just keep it around */
 521                tail = chunk->map[i] - head - size;
 522                if (tail < sizeof(int))
 523                        tail = 0;
 524
 525                /* split if warranted */
 526                if (head || tail) {
 527                        pcpu_split_block(chunk, i, head, tail);
 528                        if (head) {
 529                                i++;
 530                                off += head;
 531                                max_contig = max(chunk->map[i - 1], max_contig);
 532                        }
 533                        if (tail)
 534                                max_contig = max(chunk->map[i + 1], max_contig);
 535                }
 536
 537                /* update hint and mark allocated */
 538                if (is_last)
 539                        chunk->contig_hint = max_contig; /* fully scanned */
 540                else
 541                        chunk->contig_hint = max(chunk->contig_hint,
 542                                                 max_contig);
 543
 544                chunk->free_size -= chunk->map[i];
 545                chunk->map[i] = -chunk->map[i];
 546
 547                pcpu_chunk_relocate(chunk, oslot);
 548                return off;
 549        }
 550
 551        chunk->contig_hint = max_contig;        /* fully scanned */
 552        pcpu_chunk_relocate(chunk, oslot);
 553
 554        /* tell the upper layer that this chunk has no matching area */
 555        return -1;
 556}
 557
 558/**
 559 * pcpu_free_area - free area to a pcpu_chunk
 560 * @chunk: chunk of interest
 561 * @freeme: offset of area to free
 562 *
 563 * Free area starting from @freeme to @chunk.  Note that this function
 564 * only modifies the allocation map.  It doesn't depopulate or unmap
 565 * the area.
 566 *
 567 * CONTEXT:
 568 * pcpu_lock.
 569 */
 570static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
 571{
 572        int oslot = pcpu_chunk_slot(chunk);
 573        int i, off;
 574
 575        for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++]))
 576                if (off == freeme)
 577                        break;
 578        BUG_ON(off != freeme);
 579        BUG_ON(chunk->map[i] > 0);
 580
 581        chunk->map[i] = -chunk->map[i];
 582        chunk->free_size += chunk->map[i];
 583
 584        /* merge with previous? */
 585        if (i > 0 && chunk->map[i - 1] >= 0) {
 586                chunk->map[i - 1] += chunk->map[i];
 587                chunk->map_used--;
 588                memmove(&chunk->map[i], &chunk->map[i + 1],
 589                        (chunk->map_used - i) * sizeof(chunk->map[0]));
 590                i--;
 591        }
 592        /* merge with next? */
 593        if (i + 1 < chunk->map_used && chunk->map[i + 1] >= 0) {
 594                chunk->map[i] += chunk->map[i + 1];
 595                chunk->map_used--;
 596                memmove(&chunk->map[i + 1], &chunk->map[i + 2],
 597                        (chunk->map_used - (i + 1)) * sizeof(chunk->map[0]));
 598        }
 599
 600        chunk->contig_hint = max(chunk->map[i], chunk->contig_hint);
 601        pcpu_chunk_relocate(chunk, oslot);
 602}
 603
 604static struct pcpu_chunk *pcpu_alloc_chunk(void)
 605{
 606        struct pcpu_chunk *chunk;
 607
 608        chunk = pcpu_mem_zalloc(pcpu_chunk_struct_size);
 609        if (!chunk)
 610                return NULL;
 611
 612        chunk->map = pcpu_mem_zalloc(PCPU_DFL_MAP_ALLOC *
 613                                                sizeof(chunk->map[0]));
 614        if (!chunk->map) {
 615                kfree(chunk);
 616                return NULL;
 617        }
 618
 619        chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
 620        chunk->map[chunk->map_used++] = pcpu_unit_size;
 621
 622        INIT_LIST_HEAD(&chunk->list);
 623        chunk->free_size = pcpu_unit_size;
 624        chunk->contig_hint = pcpu_unit_size;
 625
 626        return chunk;
 627}
 628
 629static void pcpu_free_chunk(struct pcpu_chunk *chunk)
 630{
 631        if (!chunk)
 632                return;
 633        pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
 634        pcpu_mem_free(chunk, pcpu_chunk_struct_size);
 635}
 636
 637/*
 63" name="L567"> 567 586gname="implname=percp569"> 56963d="L570" class="line" name="L69"> 569 56964n>
 569

piled together.  Th" fohrewf="mmame="L5479"> 479 569ch6nk 569ch6nk
 560
 560/* 560off64.c#L569" id="L569" class="line" namde" noys="lin" name="L6=-las" noyin e="L5,nt;waysine"ced 560
 560 560chunk-6gt;
 593        if (pc6u_chun_area(struct chunk)
 6percpu.c#L571" id="L5n) - align)
 55_chunk(struct chunk)
 6percpu.c#L571" id="L5n) - align)
pcpu_alloc_chunk(void)
pcpu_allocrene"s="lin"L605"> 6035}
chunk)
 6335}
 5a> * 6a>(void)
 6335}
/* 55_area(struct (void)
-> 6335}
 629static void  * pcp6_free_area - free area t6 a pc6u_chun#ifdef+code=pcpu_allocCONFIG_NEED_PER_"linKM>-> 629static void  * @ch6nk: chunk of interest
<6 href=#include &quou.c#L634" ic void chf5" claid static void  *
chf5" claid static void static void  575        for ( * the6area.
 559 *

 560 * CON6EXT:
 569 * pcp6_lock.
 569 * pcp6an>
 568pcp67href="mm/percpu.c#L561" id="L5Th" address5"lith" fome="s="li569"> 569 593        if (oslot<6a> = 672"> 5a> *pcpu_alloc_chunk(void)
 6a>(void)
 633        if (i,  593        if (i = 06  63 615                6ff 593        if ( * CON6;
 63 3        if ( * pcp6" class="sref">BUG_ONhead) {
BUG_ON->
chunk-6gt; 622        chunk-6gt; 593        if ( 569/* merge with previous?6*/ 569i >60 &am68ich is very small but isn't too<* currme="processclabe cle lookf="mit up="nith" vmt; 569ch6nk
 506ch6nk 5696emmov68.c#L569" id="L569" class="liss="linee593"> 593        if ( 6e=map" class="sref"ne" name=" classa>->pcpuraw_smp_processcl_ilpcpu_)595                i-6;
 63 95                
/* merge with next? */<6span>6
i + 1 6lt; <69the previous block is free,"L559"> 559ch6nk 559ch6nk 5606emmov69="L567" class="line" name="L5e@align: aligname=""line" n(max PAGE_SIZE)60"> 560 560 569 569chunk-7gt; 569pc70n>
 568 569i + 1 7e="L670chunk.  Note that this functi69"> 569ch7pcpu_70t doesn't depopulate or unRETURNS568"> 568 5697>pcpu70="L567" class="line" name="L5593"> 593        if ( 55_area(struct (void)
PCP_MAP_ALLOC *
align)
);
booss="line" name="boosnkpcpureserv 633        if (chunk 7 chunk7/a>)
7a href="mm/per_area(struct 
NU7L;
chunk;
 607
 607
chunk-7gt; class="sref">i < i 607
 607
chunk7/a>-&7t;(void)
 607
kf7ee 626        return NU7L;
align)
align)
-&gSIZEign)
)
 63 615                 593        if ( 59truct_size);
f="mign)
                chunk-7gt;-> 617        }
chunk-7gt;
 622        I7IT_LI7T_HEAD(&                chunk-7gt;chunk->spinme="L_irqsaf"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                chunk-7gt; 575        for ( 593        if (chu7k72;
pcpureserv 6ef="+code=map" class="sref"ass="reservmap_used--;
 629static void pc72hunk->align)
contig_hint);
 6615                 6
 59.9static void chunk7/a>)
7a href="mm/perrrrrrrrrrrrrrrrrgotoruct_size);
5ail_unl="Ls="line" name="5ail_unl="Lpcpu.9static void pc7u_mem73"> 604static struct pc7u_mem7free( 6
pcpuu.c#Lneed_to_extendALLOC *
 633 615                chunk->spinmunl="L_irq="L5or"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                 *
 6class="sref">i 6)ode=ma> 594                /*
-> 6
 59.9static void );
5ail_unl="L mutex5" id="L605" cl5ail_unl="L mutexpcpu.9static void 73hunk->;
chunk->spinme="L_irqsaf"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                74 href="mm/perrrrrrrrr36

i < pcpu_line" hunkALLOC *
 6class="sref">i)
                ch7nk(freeme)
 5                ch7nk);
_fome=5" id="L605" cll/a>_fome=pcpu.9static void  637/* 6
 59.9static void off74>head) {
);
5ail_unl="Ls="line" name="5ail_unl="Lpcpu.9static void ->36
 581        chunk-7gt;
pcpures hre>
        pc75a href="mm/percpu.c#L593" id="L593" clsearch=through normal s="lisL5593"> 593        if (off 
chunk);
 *
)
 
de=map_used" class="u.c#Lnrs="sra>-> ++ 594                /* tell the upper layer7that 75> *
 6clde=list" class="srefass="="sref">chunk);
 4p" class="sref"" class="line" name"L602"> =contig_hint" clas class="line" name="L623"> 6594                align)
contig_hint);
 64                static void /* 608        head) {
 6
pcpuu.c#Lneed_to_extendALLOC *
 63.9static void -> 6)o94                 * @ch7nk: chunk of interest
<76;
chunk->spinmunl="L_irq="L5or"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                 *
 6c5                
i 6)ode=ma> 594                chunk-> 6
 59.9static void );
5ail_unl="L mutex5" id="L605" cl5ail_unl="L mutexpcpu.9static void  * the7area.

 *
chunk->spinme="L_irqsaf"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                 * CON7EXT:
 593        if ( * pcp7_lock.
 593        if (
 569pcp77href="mm/percpu.c#L561" id="iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiu593"> 593        if ();
res hreass="sref">pcpures hre>
                oslot<7a> = 77class="line" name="L66666666636
i,  604static struct i < pcpu_line" hunkALLOC *
 6class="sref">i)
         ruct  * the7" class="sref">i = 07 )
 5                7ff);
_fome=5" id="L605" cll/a>_fome=pcpu.9static void  * CON7;

 * pcp7" class="sref">BUG_ON 599
BUG_ON 600         593        if (chunk-7gt;(chunk->spinmunl="L_irq="L5or"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                chunk-7gt;
free_size = pcpu_allocrene"s="lin"L605 95                /* merge with previous?7*/map) {
i >70 &am78cerr5" id="L605" clerr"> 6
 59.9static void ch7nkch7nk 618
7emmov78"> 629static void chunk->spinme="L_irqsaf"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                i-7;
chunk, oslot);
);
res hreass="sref">pcpures hre>
                /* merge with next? */<7span>7
i + 1 7lt; <79the prct_size);
_fome=5" id="L605" cll/a>_fome=pcpu:1        ch7nk(chunk->spinmunl="L_irq="L5or"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                ch7nk 626        return 7emmov79==  593        if ( 6class="sref">i < i)
head) {
chunk->spinme="L_irqsaf"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                pcpu__chunhunk * *
 6class="sref">i < chunk-8gt;;
cerr5" id="L605" clerr"> 6
 59.9static void pc80 href="mm/perrrrrrrrrgotoruct_size);
5ail_unl="Ls="line" name="5ail_unl="Lpcpu.9static void 
i + 1 8e="L680"> 604static struct ch8pcpu_80ree(                 626        return 8>pcpu80==  593        if ( 6
 *
 6 href="mm/percpu.c#Lbas"_lddr5" id="L605" clbas"_lddr"> 6
+lass="sref">i < ( *
 6class="sref">i)
)
8a href="mm/perercpu.c#L627" id="L627atr5" id="L605" cl"tr"> 607
NU81"> 581                chunk-8gt;(&chunk->spinmunl="L_irq="L5or"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                );
5ail_unl="L mutex5" id="L605" cl5ail_unl="L mutexpcpu:1        chunk8/a>-&81ree(                kf8ee                NU81==  593        if ( 59truct_size);
f="mign)
);
err5" id="L605" clerr"> 6 95                                chunk-8gt;->                chunk-8gt;;
 59 95                
I8IT_LI8T_HEAD(&amercpu.c#L627" id="L627class="line" name="L617"> 617        }
chunk-8gt;
chunk-8gt; 575        for ( 559chu8k82="L567" class="line" name="L5e_"line" if ( 559 560 560pc82d="L570" class="line" name="L69"> 569 569chunk8/a>)
83n>
 569
 569pc8u_mem83chunk.  Note that this functioname="L568"> 568pc8u_mem83t doesn't depopulate or unDoes GFP_KERNELf="mm/percp569"> 569 569 568/*
 569 593        if (83hunk(struct (void)
 *
align)
                ;94                84 href="mm/perercpu.c#L627" id="L627ass="e_ALLOC" class="sref">PCP_MAP_ALLOC *
)
);
falsmign)
 6 95                                ch8nk 575        for (ch8nk 559 559/* 560off84.c#L569" id="L569" class="line@align: aligname=""line" n(max PAGE_SIZE)60"> 560 569 569chunk-8gt;
 569pc85o free slots.
 569 569/* tell the upper layer8that 85t doesn't depopulate or u69"> 569 568 569/* 569 568 569 * @ch8nk: chunk of interest
<86href="mm/percpu.c#L561" id="L593"> 593        if ((void)
 *
align)
                
                PCP_MAP_ALLOC *
)
);
reflign)
                
 * the8area.
 626        return  *
 559 * CON8EXT:
 559 559 559pcp87href="mm/percpu.c#L561" id="L5ReL56im _MA fully _chu s="lis exceptn clath" fir4t one569"> 569 569oslot<8a> = 87o free slots.
 568i,  569 593        if ( * the8" class="sref">i = 08 (struct PCPreL56imALLOChref="+code=chunk" clawork_href="5" id="L605" clwork_href="nk(void)
                8ff                 * CON8;
( *
                 * pcp8" class="sref">BUG_ON 5href="+code=chunk" cla"L62_hea=5" id="L605" cl"L62_hea=nk(void)
chunk);
 4p" class="sref"u.c#Lnrs="sra>->                BUG_ON 5href="+code=chunk" class="sref">chunk;
 6,.c#L607" id="L607nextss="line" name=next"> 617        }
 581        chunk-8gt;(                chunk-8gt;(&chunk->spinml="L_irqALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/pe 95                 604static struct /* merge with previous?8*/()
 *
 6class="sref">i 6class="sref">i 6594                i >80 &am88cWARk_Ok-> *
 6 href="mm/percpu.c#Limmutabless="line" name=immutablem/pe 95                ch8nk 637ch8nk 593        if (8emmov88>head) {
free_size =  *
chunk;
 6 5                ->static void i-8;
 581        chunk->"L62_mof"ALLOCde=list" class="sref" class="line" name="L630"> 6 href="mm/percpu.c#L class="line" name="L623"> clde=list" class="sreftodo5" id="L605" cltodo=chu 95                /* merge with next? */<8span>89class="line" 18
i + 1 8lt; <89"> 604static struct ch8nk(chunk->spinmunl="L_irqALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/pe 95                ch8nk 626        return 8emmov89== )
 *
 6class="sref">i 6clde=list" class="sreftodo5" id="L605" cltodo=chucontig_hint" clas class="line" name="L623"> 6594                 6cl0contig_hint" clasclassunit_f="mign)
head) {
 6 95                chunk-9gt; 581        pc90_chunk_relocate(                i + 1 9e="L690"> 604static struct ch9pcpu_90t doesn't depopulate or"L559"> 559 5599>pcpu90="L567" class="line" name="L5 @ptr:fponame=564"ne" name_chu59"> 559 569 569)
90d="L570" class="line" name="L59"> 559NU91href="mm/percpu.c#L561" id="L5name="L568"> 568 569chunk-9gt;
 593        if ((struct (struct (void)
 6 5                chunk9/a>-&91ree                kf9ee(s>(void)
 695                NU91== chunk;
 695                 695                i < chunk-9gt; 600        chunk-9gt;;
 6 5                                I9IT_LI92a hre6
chunk-9gt; *
 6 95                chunk-9gt; 575        for ( 6
 *
 6 95                chu9k92"> 637chunk->spinme="L_irqsaf"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                 629static void pc92ref="+code=map_alloc" class="sref">free_size =  *
 6 95                 <  6
-h(void)
contig_hint);
 695                chunk9/a>)
93n>
                (&pcpu__chunhunk * *
 6class="sref">i < pc9u_mem93"> 604static struct pc9u_mem9free( 593        if (free_size = )
                chunk;
 695                /*
 608        )
 *
 6clde=list" class="srefass="="sref">chunk);
 4p" class="sref"u.c#Lnrs="sra>-> 68        93hunk-> 6 !                ;
94 href="mm/perrrrrrrrrrrrrrrrrrrrrrrrrbreak95                
ch9nk 575        for (ch9nkchunk->spinmunl="L_irq="L5or"ALLOCde=list" class="srefass="e="Ls="line" name="ass="e="Lm/petruct_size);
5laga>-> 6 95                
/*);
EXPORT_SYMBOL_GPss="line" name="EXPORT_SYMBOL_GPsALLOC *
                off94"> 629static void  559 559chunk-9gt;
 559pc95o free slots.
 559 559 559 569 569/* 568 569 593        if ( * @ch9nk: chunk of interest
<96href=""mm/percpu.c#Lbool5" id="L605" clboolamp; ""mm/percpu.c#Lis_kernelnif ();
lddr5" id="L605" cllddr"> 6 3        if ();
CONFIG_SMP5" id="L605" clCONFIG_SMP> 629static void  6
pcpu_"per_( 6
-h(void)
pcpu_"per_( 695                (void)
 6
 *
                 * the9area.
i                
 637 * CON9EXT:
 *
                
pcpus hre"> 6
 *
 6truct_size);
"                
 600        pcp97;
 6
"L594"(void)
pcpus hre"> 6
de=lide=li C/a>(s*) href="mm/percplddr5" id="L605" cllddr"> 6
"e=ma(void)
pcpus hre"> 6
+lass="sref">i)
 6 3        if (oslot<9a> = 97class="line" 18
i,  593        if ( * the9" class="sref">i = 09  695                9ff
 * CON9;
 608         * pcp9" class="sref">BUG_ON 559BUG_ON 559 559chunk-9gt;
 569chunk-9gt;
 569 569/* merge with previous?9*/ 559i >90 &am98.c#L476" id="L476" class="lineuntil this=muncercp"finishes569"> 569ch9nk 569ch9nk 569 * pcp9e=memmove" class="sref">9emmov98.c#L569" id="L569" class="linesupports"ei narrembedding=in clasar address5space5clavm=e" nn4" ping,69"> 569 569i-9;
 569
 569/* merge with next? */<9span>99o free slots.
 569i + 1 9lt; <99chunk.  Note that this functiofir4t "mm/p. But th" current "L62=reflects"betme=5how ass="cf=e" namcl69"> 569ch9nk 569ch9nk 5699emmov99="L567" class="line" name="L5 "L62569"> 569 569 568 56910od="L570" class="line" name="L593"> 593
10on> (s>(void) 6 3 if (10oclass94 i + 1 10ocl>10of="+code=chun/a>(struct (void) 6 * ch10o=c>10oree(map) { map<"> 6 695 ch10o=c>10o 695 10o=m>10o6a href="+codunsignsicnamlass="sref">i 10o"> 608 10o8ee( 569 56910f=="L570" class="line" name="""""""""L necessary but"will apesicppnlookups""linddresses=which69"> 569 569chunk-10 cl>10fo free slots. 593 if (10fref="+code=free_size" class=5ir4t_lowign) * map) { map); ass="e=wsunit_(chunk10hun>10free( 6 * map) { map); ass="highsunit_(kf10e=k>10f); ass="unit_pagea>-> 10f6a href="+codde=mCunsignsiclong) href="mm/percplddr5" id="L605" cllddr"> 6 "L594"(void) ) 10f> 6 "e=ma(void) ; 6 594 * chunk-10 cl>10fhunk->pcpus hre"> 6 * 6truct_size); " chunk-10 cl>10 c> 608 6 "L594"(void) pcpus hre"> 6 de=lide=li href="mm/percplddr5" id="L605" cllddr"> 6 "e=ma(void) pcpus hre"> 6 +lass="sref">i) I10IST>10 class="line" name="L666666666map) { map<"> 6 chunk-10 cl>10 3lass="line" name="L666666666 chunk-10 cl>10 4lass="line" name="L66666666636 chu10=ch>10 6a href="+cod36 10 "> 608 map) { map<"> 6 594 * 6 3 if (10idhref="mm/perrrrrrrrrrrrrrrrrercpu.c#L627" id="L627__pa5" id="L605" cl__ik * * 6 95 chunk10hun>10i href="mm/perrrrrrrrrelse5 10iclass="line" name="L666666666ercpu.c#L627" id="L627aage_to_phys5" id="L605" cl"age_to_phys * * * 6 +5 pc10em_>10i3lass="line" name="L666666666 * * 6 95 pc10em_>10i4lass="line" }relse5 10i * 6 +5 * * 6 95 10i"> 629static void 55910"mL568" id="L568" class="line" acs="line" line" info -"ne" name=ass="cf=e" namrcpninfo59"> 559 559 559 559ch10e=c>10"t doesn't depopulate or urAe" name=ai which is=large enoughn cla@nr_groupsigroups=" naaining59"> 559ch10e=c>10".c#L476" id="L476" class="line@nr_unitsiunits. The ercpu.e="=i'sigroups[0].cs="mapfponams mo= na69"> 569 56910"#L568" id="L568" class="line" NR_CPUS. It'sithe caller'siresp nsibility564"initial="mi"s="map69"> 56910".c#L569" id="L569" class="lineponame=5"lio narrgroups569"> 56910"d="L570" class="line" name="L59"> 559 568chunk-10 cl>10dn> 568 569 593 if (/* tell the upper layer10>/*>10dt doehref="+code=chunk" class="sline" info5" id="L605" class="line" infonkc#L627" id="L627__init5" id="L605" cl__initnki i if (10d== * 6truct_size); ai_f="mign) 695 10nka href="+coddamlass="sref">i 10nn> 6 * i 10n3lass="line" name="L666666666 * ); " 10nree() 6 +lass="sref">i); " 626 return 6 5iLOC" class="srefline" bootmem_no> 5iL * * -> * ) 10n> 6 5 695 6 + 695 10pn> oslot<10lot>10p_HEAD(&); " 695 i, 10p"> 604static struct i = 010" c>10pcli5" id="L605" clli=chu href="mm/percpu.c#Lgroups5" id="L605" clgroupsm/pe[0].uct_size); " 695 10+co>10p"> 63710p7ee( BUG_ON10p8ee(-> * ) BUG_ON10p"> 600 chunk-10 cl>10d1"L5636 chunk-10 cl>10da hre6 559 559i >10 cl>10d.c#L476" id="L476" class="line@ai:"acs="line" info ame_chu59"> 559ch10e=c>10d="L567" class="line" name="L569"> 569ch10e=c>10d#L568" id="L568" class="line" Fchu @ai which w sfne" name="by acs="line" line" info()569"> 56910e=m>10d.c#L569" id="L569" class="lin593"> 593 if (10d"> 60/a>(struct * i-10e=i>10e== 10e_chunk_relocate( * * ); ai5" id="L605" clli=chu href="mm/percpu.c#L_"li_f="mign) i + 1 10 cl>10e"> 604static struct ch10e=c>10et doesn't depopulate or"L559"> 559ch10e=c>10e.c#L476" id="L476" class="lineclassdumpnhine" info -"prdamlout informamrcpnabout ass="line" infonk"> 55910e=m>10e="L567" class="line" name="L5 @lvl: loglevel69"> 56910e#L568" id="L568" class="line" @ai:"=e" namrcpninfo amedump69"> 56910e.c#L569" id="L569" class="lin69"> 569 56911od="L570" class="line" name="L593"> 59311on> (struct * 110class="line" name="L666666666 * i + 1 11ocl>11of="+c94 ch11o=c>11oree(i; ); "; ); widthchunk; ch11o=c>11o"--------"93"> 5995 11o=m>11o6a href="+coddamlass="sref">i); aine" endass="sref">pcpuaine" endnk 1107ee(i; 6truct_size); vchunk; 110>head) { namlass="sref">i); apl5" id="L605" clapl59 6mahrpcpu.c#L593" id="L593" clunitsipar aine", aine"sipar "L59"L593"> 593 600 111u= ; chunk-11 cl>111class="line" name="L6ass="sref">i; 111"> 604static struct chunk11hun>11free( kf11e=k>11f111== ; 1117ee(); "; '\0'93"> 5995 629static void chunk-11 cl>111ref="+code=map_alloc" class=upa5" id="L605" cluik *) chunk-11 cl>112u= ; ; i; * I11IST>112a hre6 chunk-11 cl>11 3lass="line" code=chunk" clasrdam) { "%sass=-aine": s%zu r%zu d%zu u%zu aine"=%zu*%zu"93"> 59t5 chunk-11assbas"_lddrALLO 95 72 "sref">chunk-11assb4="L628" class="line" n10ide">pcpus hre"> 6 +lass="sref">ichunk-11assb4="L628" class="line" n10idereserveds hre"> 6 +lass="srereserveds hrea<_f 72 "sref">chunk-11assb4="L628" class="line" n10idedyns hre"> 6 +lass="sredyns hre> * chunk-11assb4="L628" class="line" n10ide605" clli=chu href="mm/percpu.c#Laine" f="mign) chunk-11assb4="L628" class="line" n10ide6tomclli=chu href="mm/pertomclli=unitwidthchunk; 10+co1f="+cod36<1a> href=unit5" id="L60507ee( href=unit5" id="L605; ++/a>) { ma110 "> 608<1a> ii; ->spinmunl="L_irhref="+code=chunk" class="sref">cli5" id="L605" clli=chu href="mm/percpu.c#unit5" id="L60507ee(]idthchunk; 629 pcpuaine" endndr"> 6 3<1a> if (chunk-10 cl1dr"> 6 95<1a> chne"OC" class="sBUG_O="mm/pei; 6 +lass="sref">i; ); aine"e=chunk"tr5" id="Lg/a>; 6 +lass="sref">i) pc10"sref">11o=m>11o6a href="+coddamlass="sref">ihref=unit5" id="L605); ainef=unit5" id="L605++/a>) { ma1dr"> 6 95<1a> chunk-10 cla hreine" na10d=">10prefliref="+coddamlass="sref">ihdnkwidthchunk">112_chunk_relocate()/a>) { ma1d66666636<1a> pc10"sref">11o=m>11l>11 3lass="line" code=chunk" clasref">11o=m>11KERN_CONTlass="line" codKERN_CONTd="L6asrdam) { ; pc10"sref">11o=m>11l>11 3lass="line" code=chunk" clasrdam) { widthchunk; /*>10i7"L5636<1a> pc10"sref">11o=m>11l>11 3lass="line" code=chunk" clasref">11o=m>11KERN_CONTlass="line" codKERN_CONTd="L6asrdam) { -11assbas"_l_widthm/pei+"1) +lass="sref">i-11assbas"_l_widt/a>(widthchunk; 629static void 559<1a> 559<1a>); "pc10"sref">11o=m>11l>11 3lass="line" code=chunk" clasref">11o=m>11KERN_CONTlass="line" codKERN_CONTd="L6asrdam) { -11assbas"_l clmin_= *); "chunk-10 cl>10 3las110classf">pc10"sref">11o=m>11g/a>; ); "; 559<1a>chunk-10 clllllllll"sref">pc10em_>10i4lass="lin1g59"> 559<1a>11o=m>11l>11 3lass="line" code=chunk" clasref">11o=m>11KERN_CONTlass="line" codKERN_CONTd="L6asrdam) { -11assbas"_lwidthm/pe,es="mof( * ; /*>1n69"> 569map<"> 6 chnt_s""""""""""erge >/*>1n69"> 569map"mm/percp10"8e=chunk10f="+code=off"1class1410p8ee(11o=m>11KERN_CONTlass="line" codKERN_CONTd="L6asrdam) { ; ass="comme1049e=chunk10c#L550" id="L150" c1ass=10>/*>1n69"> 569maass="comme105de=chunk10d="L551" clas1="lin15a>-10 cl1dr"> 6 95<1a> ass="comme10d1e=chunk10 class="sref"1chunk1/a>-10 cl>10dn> 55910do free slots. 559 559 559ch10e=c>10e.c#L476" id="L476ical addressAL"> 55910e=m>10e="L567" class="linIibility564en't+ninfo -"ef=infir4t "mupsigrUS. Ikernelh=m/perical addressAL"> 559 6 ch9nk 559 9emmov98.c#L569" id="L569" as appathoglevel @lvl569"> 56910"d="L570" class="line" name="L59"> 559 559 569 10do free slots. 56910dchunk. Note that" cl clgr72 "sref">chi: h and">chofh=m/per" nfo -"_reaoglevel @lvl569"> 569 569c,/a>;non-zero,e" namfie: h anamou namf byercpts="line" infonk"> 559 10e=m>10e="L567" class="lin 72 "s afc#L56 andm/per"_reafaaren't+na th" unitlin 72 "ssical addressAL"> 559 <1 href16">ch9nk 5591 569 10ed="L570" class="lidm/per"_reas="liart itecturs""lier64en&#h4"shysaboumodelhhasical addressAL"> 559 559 569o17"sr10hun>10do free slots. 56917 na10d=">10dchunk. Note that" cl clgrvl5" id==dec#Lm cls="@nr_units: hrbyercpavail=darerespdyofficglevel @lvl569"> 569ch9nkch+glevel @lvl569"> 569c +" cl clgrvl5" id==amcl cl clgr "sref">cnlinunusitoglevel @lvl569"> 569 56910e#L568" id="L568" clas clgr "sref">cn" namfie: "sr id==amclmut+nbonamef="+chrePAGE_SIZEne" name="L569"> 56910".c#L569" id="L569"amclequalchreespai whL56 an" cl clgr72 "sref">ch+" cl clgr 72 "sref">c +ne" name="L569"> 56910ed="L570" class="li cl clgrvl5" id=oglevel @lvl569"> 569 569 56910do free slots. 569 569 5691>10 cl>10d.c#L476" id="L476" cl clgrnk; < unitlinlinai whL56 an" cl clgrnk; 569ch10e=c>10d="L567" class="lil cl clgr "sref">cnlinai whL56 an" cl clgrnk; 569ch10e=c>10d#L568" id="L568" glevel @lvl569"> 56910e=m>10d.c#L569" id="L569" cl clgrenoughn clamcl cl clgrughn cldescribe virtualcmemorynaiyformofglevel @lvl569"> 569 569i19" na9e="L598href="mm/percpu.c#slasme=5"l_unDyofficsVM"_reas=cessaboname=564"hraccordL Tremapfpsponams mo= na69"> 569 559/* merge wit1 next19"sr10hun>10do free slots. 56919 na10d=">10dchunk. Note th.cglevel @lvl569"> 569ch10e=c>10et doesn't dame="""t'"shouldeha"s mcla Ten't+na th"iat @605" clpllamcglevel @lvl569"> 569ch10e=c>10e.c#L476" id="L476"copi Tdm/per"das=remaL605fid="oglevel @lvl569"> 56910e=m>10e="L567" class="liglevel @lvl569"> 56910e#L568" id="L568" cIf"en't+na th"iendst"wiiand bots= 72 "srlamcldyoffics_reas, i keep currentnk"> 56910e.c#L569" id="L569farrs2 "srlby twona th"s - onesp n72 "s . Itorandm/per"_mcl 72 "srical addressAL"> 559 55911od="L570" class="lfamclaags mcl be="usicldifferlas"_reafpline" @ai:mcl p n7tay awayical addressAL"> 559 55910do free slots. 56920 na10d=">10dchunk. Note th.cglevel @lvl569"> 569 568 56910e=m>10e="L567" class="lisipar "L59"L593"> 59311o=d) * pc10"sref">11o=m>11l>11 33333333333unk->/a>(s>(vo605" clpl* ma1d66666636<1a> 2111udm/per"a href="+codcharauct> s_bufa>); "-11 cl>111cdm/per"a href="+coddamlassde); "2hunk11hun>11free(ichunkdthm/pei+"1) +lass8" class="line" n10idereserveds hre"> 6 +lass="srereserveds hrea<_f 72 "sref">chunkdthm/pei+"1) +lassredyns hre> * kf11e=k>11f11o=d) ->/a>(s>(vosef=ink" clasref">11osef=inERN_,/a>->/a>(s>(vodef=ink" clasref">11odef=inERN_/clunitnk 2 111=unsef="+cay wh 2 1117ee( 2 111=unsef="+cay wh 2 ss="comme21f9e=chunk11 class="sref"2chunk2/a>-11 cl>111runsef="+ca href="+coddamlass> a>); " 2 10nka hr ); " 2 i; 6tructid) ; 6truct" class="line" nd="L605" clli=chu95 2 I11IST>112a hre6 -11 cl>11 3lass="line" cod> 11fa>); "11fERN_CONTlass="line" > s_bufa>); "); "-11thm/i /oue"naine_units5" >ouea hwi do {c10"sref">11o=m>11l>11 33333333333 \thm/i /11o=m>11o6a>; lik=lyERN_CONTlass="line" >oue"naine_units5" >ouea hwi) {c10"sref">11o=m>11l>11 33333333333 33333 \thm/i /(11o=m_emergERN_COprdam\nzu u%zu awidthcPERCPU:in> <"+chresibility56,>">112sassbas"_lwidthm#>oue);33 \thm/i /211o=m_emergERN_COprdam\nzu u%zu awidthcPERCPU:iid=; ); "10nk ns"+fref="+code=chu href="classdumpnhine" info5" id="L605" class="dumpnhin ns"+fref="+cod/a>) EMERGpnhine" info5" /a>) EMERG 6truct> * 6tructBUG* 11l>11 33333333333 33333 \thm/i /2hne"OC" class=}3333333333 33333 o=m>11l>11 33333333333 33333 3333 33333 \thm/i / 604static 2 -10 cla hrelapl59 6mahrpcpu.c#L593" saibiy checkslisipar "L59"L593"> 593p(111"> 604static 2 1117ee(ichunkidthm/i /210p8ee(112_chunk_relocatechunkree-11 cl>11 3lass="line" codPCPU_SETUP_BUG_O= id="L605" clroPCPU_SETUP_BUG_O=ERN_CONTlass="line" "+code=chunkli5" id="L605" clli=chu href="mm/percpu.c#Lunit 72 "sref">chunkreef=" ~ee(11free(chunkreep( 6 2hnt_s2""""""thm/i /10p8eapl59 6mahrpcpu.c#L593" pro infme=5"lrL Prdamlout ind buildroupfig s=darsraccordL Tlylisipar "L59"L593"> 593-11 cl>111ref="+code=map_al1assbid="L6cpu.c#unit5" id="L605bid="L6cL6055" cluik * 2ass="comme205de=chunk10d="L551" clas2="lin25a>-11 cl>112u= 2ass="comme20d1e=chunk10 class="sref"2chunk25 na11d=">112_chunk_relocate< "sree); "); " 2ass="comme20d2e=chunk10hunk_relocate2 clas25ot<10lot>10p_HEAD(&< "sreid="naine_units5" id="Lid=L6055" cluik * 2a 604static 2"ss="comme2054e=chunk10>/* tell the 2pper 25unk11hun>11free a>); "( a>); "( a>); " 559<1a>10d); "( a>); "); " 25ss="comme2056e=chunk10id="L557" cla2s="li25f">10+co1f="+cod36<1a> 6 a>); " 25 10p8ee( 25ss="comme2059e=chunk10_free_area - 2ree a25 na11o=">559"> 559<1a>11free href=unit5" id="L60507eea hgroup"> 6tructid) ; ( href=unit5" id="L605; ++/a>) { 6tructid) ; ii; ->spinmunl="L_irhref="+code=chunk" class="sref">cli5" id="L605" clli=chu href="mm/percpu.c#unit5" id="L60507ee(]idthchunk; 629 2 I11IST>112a hre6 (]idthchunk; 2 ss="comme2064e=chunk10 modifies the2alloc26a>-10 clllllllll"sref/pe,es="mof((]idthchunk; i) ) 2a hre26gt;10it_f="mm/pe95 2 ss="comme20n6e=chunk10n> i) ( 6 95<1a> 2 <2 href26""""""""erge with next? */<10"nk a>); "); " 629 2 2pc10"sref">11o=m>11l>11 3a>; a>); " 10nkd="""""""""s="li666tinuedthm/p11"> 629 2 -10 cl1dr"> 6 95<1a> 2 ss="comme2071e=chunk10id="L572" cla2s="li27 na10d="""""""""s="line" na10( a>); " 2 o27*/<11o/*>110classf">pc10"sref= ); " a>); " 2 27a>-10 cl>10 3las110classf">pc(); "( a>); "); " 2 ss="comme20p4e=chunk10d="L575" clas2="lin27a>-11thm/i /); "( a>); "; 6truct" class="line" nd="L605" clli=chu95 2 eturn pc10"sref">11o=m>/pe,es="mof( 6truct" class="line" nd="L6a*" f="mm/pe /a href="mm/percpli5" id="L605" clli=chu href="mm/percpu.c#Lunit_f="mign) pc10"sref">11o=m>11l>11 3lapl59 6mahrpcpu.c#L593" dec#Lm cl low/high lisipar "L59"L593"> 59310nkd=""""""a>; 593 a>); " a>); " 28 110classf">pc10"srefa>; 593pc10"s/pe,es="mof( a>); "/* merge wit2 prev28a>-10 clllllllll"sref">pc10em_>10i4laee( 28ss="comme2085e=chunk10 class="sref"2i28L5510m/p>10d 569m2p>/*>1n69"> 569m2iERN_/clunitnk; 10it_f="mm/pe95 28 -11 cl>111ref="+code=map_afor_L605 a>); " ERN_CONTlass="line" > a>); " 2 i29" n10id=s="line" na10hgroup"> 6tructPCPU_SETUP_BUG_O= id="L605" clroPCPU_SETUP_BUG_O=ERN_CONTlass="line" "sree); "( a>); "); " 2 ss="comme2091e=chunk10c#L592" id="L292" c29s="10thm/pe,"1))95 2 /* merge wit2 next29ot<10lot>10p_Hapl59 6mahrpcpu.c#L593" weNR_CPre donesparsabouen&#input, 59329 na10# 6tructPCPU_SETUP_BUG_O= id="L605" clroPCPU_SETUP_BUG_O=ERN_59"L593"> 593) DEBUGpnhia hgroup"> 6truct> * 593 2=eturn 2=ss="comme20e7e=chunk10ref="+code=ch2nk" c29ss=11c#L>1117ee( 2ass="comme20e8e=chunk10c#L599" id="L299" c2910p8ee( 2ass="comme20e9e=chunk10d="L600" clas2="lin29a>-11 cl>111ref="+code=map_aed="L "sree); "); " 3ass="comme31ode=chunk11ode=600" clas3="lin30a>-11 cl>112u= 3a1s="comme3191e=chunk10c#L592" id="L392" c30s="10thm/pe,"1))95 3 10p_Hapl59 6mahrpcpu.c#L593" dec#Lm cl baser" arrgec#Lslisipar "L59"L593"> 59330a>-11 cl>11 3lass="line" codid="L "sreaagscpu.c#unit5" id=id="L "sreaagscERN_/clunitnkchunkre" ceg 3a4 3a5 p(; ; ; 3a6 ; 11o=d) 3a7 ( 3a8 10it_f="mm/pe95 3ass="comme31o9e=chunk11o="L600" clas3="lin309t<10lot>10p_Hapl59 6mahrpcpu.c#L593"ipar "L59"L593"> 59311od="L570" class="""""""""* Ame=564"#a th"inloum unite"_ddiloutal d="tinlouiarrforipar "L59"L593"> 59310"n> 56910do free slots. 593-11 cl>11 3lass="line" codid="L="snloumpu.c#unit5" id=id="L="snloumbeN_/clunitnk 3 3hunk11hun>11free( 3 kf11e=k>11f( 3 6 pc10"srefa>(( 3 7 10p8eapl59 6mahrpcpu.c#L593"ipar "L59"L593"> 593c linzero,epfponams mo= na69"> 56911od="L570" class="""""""""* dm/per"a th" cov#Lsldm/per"_reaf+pdyofficfpline" @ai:_reaical addressAL"> 55910"n> c linnouizero,ei keep currentnk"> 56910do free slots. 559 569ch10e=c>10et doesn't da""""""""*sipar "L59"L593"> 593p(11osef=inERN_/clunitnk; ; 3 11osef=inERN_05" clli=chu href="mlisupu.c#unit5" id=lisuERN_idthm/pe,"1))95 3 7 1117ee(11osef=inERN_05" clli=chu href="m605" clpl* 3 310p8ee(11osef=inERN_05" clli=chu href="me); "); " 3 ss="comme30 9e1chunk10pu_free_chunk3 clas32a>-11 cl>111ref="+code=map_aief=ink" clasref">11osef=inERN_05" clli=chu href="me); "); " 3if (-11 cl>112u= 11osef=inERN_05" clli=chu href="mimmus=dar class="line" ndmmus=darERN_/clunitnk 3 33 na11d=">112_chunk_relocate11osef=inERN_05" clli=chu href="mpopul64"hpu.c#unit5" id=iopul64"hpnhia hgroup"> 6tructid="L "sreaagscpu.c#unit5" id=id="L "sreaagscERN_)dthm/pe,"1))95 3 I11IST>112a hre6 6 +lass="srereserveds hrea<_f 72 "sref">chunk4t_>ma1dr"> 6 95<1a> 3 -10 clllllllll"sref/pe,es="mof(11osef=inERN_05" clli=chu href="mfreeeserveds hrea<_f freeeservERN_/clunitnkchunkdthm/pe,"1))95 3 6tructid="L 72 "sreef=ink" clasref">11o=d) < 72 "sreef=inERN_/clunitnk11osef=inERN_dthm/pe,"1))95 3 pc10"srefa>(ichunkdthm/pei+"1) +lass8" class="line" n10idereserveds hre"> 6 +lass="srereserveds hrea<_f 72 "sref">chunkdthm/pe,"1))95 3 7 1117} elset_>ma1dr"> 6 95<1a> 3 311osef=inERN_05" clli=chu href="mfreeeserveds hrea<_f freeeservERN_/clunitnk * 10nk ns"+fref="+codredyns hre> * 593 569m3ass="comme3041e1chunk14d="L622" clas3="lin34 na11d=">112_chunk_relocate11osef=inERN_05" clli=chu href="m666tig_hin 11osef=inERN_05" clli=chu href="mfreeeserveds hrea<_f freeeservERN_dthm/r12a hre6 3ass="comme3042e1chunk14o/* merge wit3 next34">I11IST>112a hre6 -11 cl>11 3lass="line" codief=ink" clasref">11osef=inERN_05" clli=chu href="me); "(11osef=inERN_05" clli=chu href="me); "ichunkdthm/r12a hre6 3a 11fra>; 11osef=inERN_05" clli=chu href="mfreeeserveds hrea<_f freeeservERN_4thm/pe,"1))95 3a 10d 6tructief=ink" clasref">11osef=inERN_05" clli=chu href="me); "(11osef=inERN_05" clli=chu href="me); "11osef=inERN_05" clli=chu href="mfreeeserveds hrea<_f freeeservERN_dthm/r12a hre6 3a 10+co1f="+cod36<1a> 6 3hnt_s34ss=11c#L>1117eapl59 6mahrpcpu.c#L593" sibi dyofficsa th"iaf ne infarylisipar "L59"L593"> 59310p8a>; * 6 95<1a> 3ass="comme3049e=chunk10c#L550" id="L350" c34"nterest10nk ns"+fref="+codref=ink" clasref">11odef=inERN_/clunitnk; ; 3ass="comme305de=chunk10d="L551" clas3="lin35" n10id=s="line" na10hgroup"> 6tructINIT_LIST_HEAD class="line" nINIT_LIST_HEADERN_Cref="+code=chunk" clref=ink" clasref">11odef=inERN_05" clli=chu href="mlisupu.c#unit5" id=lisuERN_idthm/pe,"1))95 3ass="comme30d1e=chunk10 class="sref"3chunk35 na10d="""""""""s="li+code=chunk" clref=ink" clasref">11odef=inERN_05" clli=chu href="m605" clpl* 3ass="comme30d2e=chunk10hunk_relocate3 clas35*/<11o/*>110classf">p+code=chunk" clref=ink" clasref">11odef=inERN_05" clli=chu href="me); "); " 3a 11odef=inERN_05" clli=chu href="me); "); " 3a /* tell the 3pper 35a>-10 clllllllll"sref/pe,es="mof(11odef=inERN_05" clli=chu href="mimmus=dar class="line" ndmmus=darERN_/clunitnk 35ss="comme3055e=chunk10m/percpu.c#L536" id35L5510m/p>10d11odef=inERN_05" clli=chu href="mpopul64"hpu.c#unit5" id=iopul64"hpnhia hgroup"> 6tructid="L "sreaagscpu.c#unit5" id=id="L "sreaagscERN_)dthm/pe,"1))95 35ss="comme3056e=chunk10id="L557" cla3s="li35f">10+co1f="+cod36<1a> 6 (11odef=inERN_05" clli=chu href="m666tig_hin 11odef=inERN_05" clli=chu href="mfreeeserveds hrea<_f freeeservERN_/clunitnk * 11odef=inERN_05" clli=chu href="me); "(11odef=inERN_05" clli=chu href="me); " 35ss="comme3059e=chunk10_free_area - 3ree a35"nterest10nk ns"+fref="+codref=ink" clasref">11odef=inERN_05" clli=chu href="me); "(11odef=inERN_05" clli=chu href="me); "11odef=inERN_05" clli=chu href="mfreeeserveds hrea<_f freeeservERN_dthm/r12a hre6 3 -11 cl>112u>/*>1n69"> 569m3 ss="comme3061e=chunk10eme: offset o3 area36s="10thm/pe,"1))95 3 kren't+na th" "=risipar "L59"L593"> 59311o=d) <39;t+eef=inERN_/clunitnk11odef=inERN_ ?:lunitnk11osef=inERN_dthm/pe,"1))95 3 ss="comme3064e=chunk10 modifies the3alloc36unk11hun>11free(11o=d) <39;t+eef=inERN_, -1)dthm/pe,"1))95 3 ss="comme3065e=chunk10area. 3a hre36gt;10it_f="mm/pe95 3 ss="comme30n6e=chunk10n> 593 6 <3 href36ss=11c#L>1117ee( 3 3pc10"srefreturn 0dthm/pe,"1))95 3 ss="comme3069e=chunk10an> /*>1n69"> 569m3 -10 cl1dr"> 6 95<1a> 3 ss="comme3071e=chunk10id="L572" cla3s="li37 na10#ifdefp(111"> 604static 3 o37">I11IST>112a hre6 37a>-10666(ma1dr"> 6 95<1a> 3 ss="comme30p4e=chunk10d="L575" clas3="lin37unk11hun>11fr[a>(; ma1dr"> 6 95<1a> 3 ss="comme3075e=chunk10" class="sref3>i10d<[a>(; ma1dr"> 6 95<1a> 3 ss="comme3076e=chunk10+code=off" cl3ss="s37 na10d=ref">p[a>(; ma1dr"> 6 95<1a> 3 <"> 6 3a"mm/percp3078e=chunk10" class="sref3>BUG_37f">10it_f="mm/pe95 3a (( 3 6 95<1a> 3(( class=setu/a>); " class=setu/ERN_Cchar"*lli=chu href="m="l* 38 ma1dr"> 6 95<1a> 3< 38ss="comme3084e=chunk10>/* merge wit3 prev38a>-10 clllllllll"srefreturn -unitnk 338gt;10it_f="mm/pe95 3pa>; hvm/pe / 3 3<"mm/percp30d8e=chunk10e=memmove" cl3ss="s3ef">10#ifdefp( 3< -11 cl>111relseta>; a href="de=map" ="lcm/a>); "; 3 i39" n10id=s="line" na10hgroup"> 6truct=d) 3 ss="comme3091e=chunk10c#L592" id="L392" c39s="10#endifthm/i //* merge wit3 next39ot<10#ifdefp( 3="mm/percp30e3e=chunk10 class="sref"3i393>-11 cl>111relseta>; a href="de=map" ="lcm/a>); "; 3 ss="comme3094e=chunk10e=chunk" clas3="sre39a>-10 clllllllll"sref/pe,es="mof( 3 ss="comme30e5e=chunk10e=chunk" clas3="sre39gt;10#endifthm/i /(; 3 "mm/percp30e8e=chunk10c#L599" id="L399" c39f">10it_f="mm/pe95 3ass="comme30e9e=chunk10d="L600" clas3="lin39a>-11 cl>111rreturn 0dthm/pe,"1))95 4ass="comme41ode=chunk11ode=600" clas4="lin40a>-11>/*>1n69"> 569m4a1s="comme4191e=chunk10c#L592" id="L492" c40s="10unitnk classnk; class=setu/a>); " class=setu/ERN_4dthm/pe,"1))95 4 I11IST>112a hre6 40 na10d=">10dchunk. Note th3"ipar "L59"L593"> 593 569 569 56910et doesn't da*sipar "L59"L593"> 59310#iflunitnk); " 593); " 4I11IST>112a hre6 I11IST>112a hre6 5934hunk11#iflunitnk); " 4<5 kf11#defaineONTlass="line" BUILD_PAGE_FIRST_CHUNK id="L605" clroBUILD_PAGE_FIRST_CHUNK>I11IST>112a hre6 10dchunk. Note th3" id="L6uild class="dum() linusitsbyrboth embedlamclaags#39;t+na th" *sipar "L59"L593"> 593); "I11) || unitnk); "I114thm/pe,"1))95 4 11od="L570" class=/*"ipar "L59"L593"> 59310"n> 59310do free slots.c:ren&#f">c of 72 "sr" 593c ee 593ch10e=c>10et doesn't da* @6k; <:fpline" @ai:_k 593 593 59310et doesn't da* Tharrfunc @ai:dec#Lm clingL605;zu of ref">,epfpir map5;zus to d="sERar "L59"L593"> 5934 593 56911od="L570" class="*ERar "L59"L593"> 59343s="10ef=>10"n> 59310do free slots.cerforipar "L59"L593"> 59310dchunk. Note th.* ref"> "=ren&#se" fgL605 unite"returnitsoupfigur" @ai:iinguarrntentERar "L59"L593"> 593ch10e=c>10et doesn't da* to havel="Ls ai:dlffer9;t nne"s ai:dlffer9;t g"L605fpmcl5" c=75%nusagsERar "L59"L593"> 593ceoglevel @lvl569"> 569 59310et doesn't da* RETURNS:ERar "L59"L593"> 5934 59310ed="L570" class="*:failurv,fERR_PTR value iinreturnit ERar "L59"L593"> 593 593( 593110classf">pc10"srefwith nexee(chunk5lunitnk( * 6 95<1a> 4a -10 cl>10 3las110clc10"srefwith nexee(; ma1dr"> 6 95<1a> 4a -10 clllllllll"sref clllllllll"sref/pe,es="mof( 4a ma1dr"> 6 95<1a> 4a (( 4p<"> 6 4hnt_s44ss=11c#L>1117dm/per"in dee(( 4p 10p8co6ichunkdclunitnk); " 4pss="comme4049e=chunk10c#L550" id="L450" c44"nterest(iERN_/cl0dthm/pe,"1))95 4ass="comme405de=chunk10d="L551" clas4="lin45a>-11 cl>112u= 4ass="comme40d1e=chunk10 class="sref"4chunk45 na10d=""""""in dee(_=ed=6lass"*sipar "L59"L593"> 593hunk5lunitnk; (); "); " 4a /* tell the 4pper 45a>-10 cllllllhref=n" ns"+fref="+code=chu="classdumpnhine" info5" id="Lclass="dumpnhia*unitnk 4a 10d10+co1f="+cod36<1a> 6 593 10p8ee(); "); " 45ss="comme4059e=chunk10_free_area - 4ree a45a>-11 cl>111ref="+code=map_amemse 4 -10 cl1dr"> 6 95<1a> 4 ss="comme4061e=chunk10eme: offset o4 area461s=11c#L>1117eapl59 6mahrpcpu.c#L593" cplcul64" =d="Lsugfpmclensure dl5" id= iinenough ee 593 ichunkdthm/pei+"1) +lassss="srereserveds hrea<_f 72 "sref">chunk + clL617nk 4 ss="comme4063e=chunk10 area startin4 from46a>-10 cl>10 3las110clc10"srefwithunitnk * 4 11free( * (ichunkd-lunitnkchunkdthm/pe,"1))95 4 ss="comme4065e=chunk10area. 4a hre46gt;10it_f="mm/pe95 4 ss="comme40n6e=chunk10n> 593 6 <4 href46"""""e=c>10et doesn't daaaaaaaaa* Dec#Lm cl minL "srecpu.,fpline_servlamclmax_upa such tha keep currentnk"> 5694 56910ac0" cod64" 4k plef="+csegclass whichrprefequal"tokeep currentnk"> 56911od="L570" class="""""""""* e11minL "srecpu. ERar "L59"L593"> 59310"n> 593o47ot<10lot>10p_He(( 59347a>-1059"L593"> 593(; ; 593( 593pwhile (( 593 6 ( 59310p8ee( 593 593 593); "ma1dr"> 6 95<1a> 48 110classf">p+code=chunk" cl="L60 4< -11:thm/pe,"1))95 4< /* merge wit4 prev48a>-10 clllllllll"srefchunk_relocate); "); "ma1dr"> 6 95<1a> 48ss="comme4085e=chunk10 class="sref"4i48L5510m/p>10d; ); "); "ma1dr"> 6 95<1a> 48ss="comme40d6e=chunk10e=chunk" clas4="sre48 na10d=ref">pc10"srefffffffffffffffffbreakdthm/pe,"1))95 4<<"> 6 ; ); "(); "( 4< ); "); " 410nkllllllll"srefchunk_relocate); "); "ma1dr"> 6 95<1a> 4 i49" n10id=s="line" na100nkllllllll"srefchunk_relocate<="L60 4 ss="comme4091e=chunk10c#L592" id="L492" c49 na10d="""""""""s="li0nkllllllll"srefchunk_relocateI11 4 /* merge wit4 next49*/<11o/*>110classf">pc10"srefwith nexgoto lass="line" codnext_="L60 4 49a>-10 cl>10 3las110clc10"sref>/*>1n69"> 569m4 ss="comme4094e=chunk10e=chunk" clas4="sre49a>-10 clllllllll"sref>/*>1n69"> 569m4 ss="comme40e5e=chunk10e=chunk" clas4="sre49L5510m/p>10d); "(); " 4 ss="comme40e6e=chunk10e=memmove" cl4ss="s49 na10d=ref">pc10"srefa>(( 4 <"> 6 /*>1n69"> 569m4 10it_f="mm/pe95 4ass="comme40e9e=chunk10d="L600" clas4="lin49a>-11 cl>111rHapl59 6mahrpcpu.c#L593"ipar "L59"L593"> 593cenusagsxgo"s aver 75%ipar "L59"L593"> 593 59310do free slots.ceoglevel @lvl569"> 56950 na10d=">10dchunk. Note ths""""""""*sipar "L59"L593"> 593hunk5" unitnk 5a5 ((ma1dr"> 6 95<1a> 5a6 pc10"srefin dee(ihunk5" 05lunitnk); " 5a7 ((10nkllllllll666tinuedthm/pe,"1))95 5-10 cl1dr"> 6 95<1a> 5<1s="comme5111e=chunk11d="L612" clas5="lin51 na10d="""""""""s="lieema1dr"> 6 95<1a> 5 110classf">pc10"srefin dee(ihunk/clunitnkI11 ( 5< -10 cl>10 3las110clc10"srefee(ihunk5+clunitnkihunkdthm/pe,"1))95 5<4 51a>-10 clllllllll"sref cllllllunitnk); "ihunka*" f="mm/pe /a hrupapu.c#unit5" id=upahunk/-lunitnk( 5<5 /*>1n69"> 569m5 6 10+co1f="+cod36<1a> 59310dchunk. Note thllll"sref cllllll* DonNR_CPt0ac0eptiaf whutagsxis aver 1/3 uniteipar "L59"L593"> 593110" parisonlensures upa==1fplwaysipar "L59"L593"> 59311od="L570" class=""""""""" cllllll* p="Lescen&#folasw;zu checkoglevel @lvl569"> 56910"n> 593pa>; ); "I11 )5/_3i59"L593-10 cl>10 3las110clc10"sref666tinuedthm/pe,"1))95 5 ch10thm/pe,"1))95 5 5 593pc10"srefi>; ihunk55" clunitnkihunki59"L593 5 5hunk5" unitnkihunkdthm/pe,"1))95 5 9 10nk ns"+fref="+codbest_upapu.c#unit5" id=best_upaERN_/clunitnk 593-11 cl>112u>/*>1n69"> 569m5 53 na11d=">112_chunk_relocate 593I11IST>112a hre6 593-10 cllllllee 593iERN_/+clunitnk( 5i6 10+co1f="+cod36<1a> 1117ee(iERN_)dthm/pe,"1))95 5i 5; a href="de=map" cf="mm/percpli5" id="L6459"L593"> 59310nkreturn href="de=map" ERR_PTR="mm/percpli5" ERR_PTRERN_C-unitnk 5ass="comme504de1chunk14id="L631" cla5s="li54a>-11 cl>112u= ); "); " 5a 5a 110ceema1dr"> 6 95<1a> 5a -10 cl>10 3las110clunitnk(); " 5a -10 clllllllll"sref= ); "( 5a 10d<>/*>1n69"> 569m5a 10+co1f="+cod36<1a> 6 5hnt_s54ss=11c#L>1117ee(ichunkdclunitnkichunkdthm/pe,"1))95 5a 10p8ee(chunk clunitnkchunkdthm/pe,"1))95 5pss="comme5049e=chunk10c#L550" id="L550" c54a>-11 cl>111ref="+code=map_acf="mm/percpli5" id="L605" clli=chu href="mredyns hre> * ( * 5ass="comme505de=chunk10d="L551" clas5="lin55a>-11 cl>112u= (( 5a ; (; ; 5a 10p_He(( 5a 593/* tell the 5pper 55a>-10 cllllllee; (ma1dr"> 6 95<1a> 5a 10d( 5a 10+co1f="+cod36<1a> 6 593 10dchunk. Note thllll"sref cllllll* Iibility56 base_offsetrps if _me g"L605fpre e=564"dipar "L59"L593"> 59310ed="L570" class=""""""""" cllllll* back-to-back unite"cpli"r should upd64" tharrtokeep currentnk"> 56911od="L570" class=""""""""" cllllll* nk"lect0actual"_line" @aioglevel @lvl569"> 56910"n> 593 p+code=chunk" cl=f="mm/percpli5" gd="L605" clli=chu href="mbase_offsetpu.c#unit5" id=base_offsetERN_/clunitnk; 5 ss="comme5063e=chunk10 area startin5 from56a>-1059"L593"> 593-10 clllllllll"srefchunk_relocate); "); " 593 5a hre56L5510m/p>10d; ); "(); " 593 pc10"srefffffffffffffffff+code=chunk" cl=f="mm/percpli5" gd="L605" clli=chu href="md="Le); "(iERN_++]5" unitnk); " 5 <"> 6 <5 href56""""""""erge with nexee(iERN_5" He(iERN_5lunitnk 5 5; iERN_dthm/pe,"1))95 5 ss="comme5069e=chunk10an> 569m5 -11 cl>112u= ; iERN_)dthm/pe,"1))95 5 ss="comme5071e=chunk10id="L572" cla5s="li57s="10thm/pe,"1))95 5 o57ot<10lot>10p_return href="de=map" cf="mm/percpli5" id="L6nit_f="mm/pe95 5 ss="comme5073e=chunk10 class="sref"5i57a>-10>/*>1n69"> 569m5 593 5 ss="comme5076e=chunk10+code=off" cl5ss="s57 na10#iflunitnk); "I11)it_f="mm/pe95 5 <"> 6 10et doesn't d/*"ipar "L59"L593"> 59310dchunk. Note tha* id="Lembed<39;t+eef=in - embedlen&#f9;t+n 59310ed="L570" class="*:@ 72 "sref">c:ren&#f">c of 72 "sr" 593c ee 593 ; <:fpline" @ai:_k 59310do free slots. 59310dchunk. Note th.* @6lass=fn: func @ai:to ame=564" 593/* merge wit5 prev58>ch10e=c>10et doesn't da* @free=fn: func @ai:to free# 59358lass=e=c>10et doesn't da*ERar "L59"L593"> 593 593 6 10et doesn't da* c>10bencpli"+cwhere id="Lfet05b39;t+eef=in() ls expectit ERar "L59"L593"> 59310dchunk. Note tha*ERar "L59"L593"> 593 593i59 na11ode>11od="L570" class="* byncpli;zu @6lass=fn amclusitsas-arrwithoutnbe;zu mapp"+cin okeep currentnk"> 569 ; 569/* merge wit5 next59"sr10hun>10do free slots. 59359 na10d=">10dchunk. Note th.*ERar "L59"L593"> 593 593 593 593 6 10et doesn't da* requi7;zu large vm6lass"pdd 7sfun>ceo DonNR_CPt0uLe tharr"le=564e 593cenarrno4 ord#Lslof mag"srudeularger th>11distancesERar "L59"L593"> 59310ed="L570" class="*:between nne"fmemorylpdd 7sesc(ieo 32bsr NUMA machines) ERar "L59"L593"> 593 593 59310do free slots. 59360 na10d=">10dchunk. Note ths*:If the needitsservliscsmpli"r th>11en&#minimum e 593 593 593 59310et doesn't da* 0 ai:suc inf, -errno ai:failurv ERar "L59"L593"> 593 593((chunk5lunitnk( * 6 95<1a> 6(; ma1dr"> 6 95<1a> 6<1s="comme6111e=chunk11d="L612" clas6="lin61 na10d="""""""""s="li0nkllllllll"srefexee( 6 95<1a> 6<2s="comme6112e=chunk11 class="sref"6chunk61*/<11o/*>110classf">pc10"sreflll"srefexee((I115>ma1dr"> 6 95<1a> 6<3s="comme6113e=chunk11 6 61a>-10 cl>10 3las110clc10"sreflll"srefexee((); "I11)it_f="mm/pe95 6<4 61a>-10_>ma1dr"> 6 95<1a> 6<5 10diERN_5" He( 6 7 1117dmef=n" ns"+fref="+code=chu="classdumpnhine" info5" id="Lclass="dumpnhia*unitnk 6 10p8ee(a>; 6 9 ( 6 -10 cl1dr"> 6 95<1a> 6 112_chunk_relocatechunk5lunitnk * ; ma1dr"> 6 95<1a> 6 plc10"sreflll"srefexee( 6 -10 cl>10 3a>; 6 -10 clllllllll"srefreturn href="de=map" PTR_ERR="mm/percpli5" PTR_ERRERN_CONTlass="line" cf="mm/percpli5" id="L64dthm/pe,"1))95 6 5 6 6 pee(ichunkdthm/pei+"1) +lasscf="mm/percpli5" id="L605" clli=chu href="mss="srereserveds hrea<_f 72 "sref">chunk thm/pei+"1) +lasscf="mm/percpli5" id="L605" clli=chu href="mredyns hre> * 6 7 1117ee(a>; a>; 6 610it_f="mm/pe95 6 9 -11 cl>111ref="+code=map_acrea>iERN_5" He(a>; a>; 6if (-11 cl>112ua>; a href="de=map" crea>iERN_4t_>ma1dr"> 6 95<1a> 6 63 na10d="""""""""s="liunitnk 6i 110classf">pgoto lass="line" codout_free="mm/percpli5" out_freeERN_dthm/pe,"1))95 6i 569m6 ch10thm/pe,"1))95 6 5 593peema1dr"> 6 95<1a> 6 7 ( 6 95<1a> 6 6(); "( 6i9 10nkvoida*unitnk 6ass="comme604de1chunk14id="L631" cla6s="li64a>-10 cl1dr"> 6 95<1a> 6a iERN_5ref="ref=" ee(); "( 593110classf">plc10"sreee(); "(); "( 6 95<1a> 6a -10 cl>10 3las110clunitnk); "( 6a ch10thm/pe,"1))95 6a 10d10dcenee 593pc10"srefa>((I11 ); "iERN_5*" f="mm/pe /a hrcf="mm/percpli5" id="L605" clli=chu href="m "srecpu.c#Lunit 72 "srecpu.hunk5lunitnk; ; 593 6 6hnt_s64""""""""erge with nexa>; a href="de=map" ptrpnhine" info5" itrERN_4t_>ma1dr"> 6 95<1a> 6a 6pss="comme6049e=chunk10c#L550" id="L650" c64"nterest10nkkkkkkkkkgoto lass="line" codout_free_crea>iERN_dthm/pe,"1))95 6ass="comme605de=chunk10d="L551" clas6="lin65" n10id=s="line" na10>/*>1n69"> 569m6a 593110classf">p+code=chunk" clkmemleak_free="mm/percpli5" kmemleak_free>I11 6a ERN_[a>( 6a /* tell the 6pper 65>ch10thm/pe,"1))95 6a 10dI11 6a p>/*>1n69"> 569m6a<"> 6 10p8eapl59 6mahrpcpu.c#L593"ipar "L59"L593"> 59310ed="L570" class="""""""""* Copy data amclfree#unusitsparts unitiscshould happ"n afterfpliipar "L59"L593"> 59311od="L570" class="""""""""*fpline" @ai5fpref0" plete; oen&rwise, we may emcluprwithipar "L59"L593"> 593 593 10do free slots. 593ma1dr""> 593-10 clllllllll"srefhref=n" ns"+fref="+code=chu"L605bsdumpnhine" info5" id="L"L605bsdumhunka*unitnk( 593 6a hre66L5510m/p>10d(iERN_[a>( 593 10+co1f="+cod36<1a> 6 <6 href66""""""""erge with nexeeiERN_dlunitnkma1dr""> 5936); "((ma1dr""> 59310nkkkkkkkkk" c>10p8eapl59 6mahrpcpu.c#L593"#unusits "sr,lfree#wholec*sipar "L59"L593"> 593(); "I11 593 6 o67*/<11o/*>110classf">plc10"sre>/*>1n69"> 569m6 ss="comme6073e=chunk10 class="sref"6i67a>-10 cl>10 3las110clc10"srefeapl59 6mahrpcpu.c#L593"#copy amclreturn en&#unusitspartc*sipar "L59"L593"> 593I11 ichunk4dthm1dr""> 593(); "I11 593pc10"sref>/*>1n69"> 569m6 <"> 6 /*>1n69"> 569m6 10it_f="mm/pe95 6 ss="comme6079e=chunk10" class="sref6>BUG_67a>-11 cl>111reapl59 6mahrpcpu.c#L593"#base"pdd 7sfarrnow known,ldec#Lm cl "L605 base"offsetsf*sipar "L59"L593"> 593 6ma1dr""1))95 6< 110classf">p+code=chunk" cl8f="mm/percpli5" id="L605" clli=chu href="mercpu.c#Lnr_gr=unit5"="L60507ee[a>(iERN_[a>( 6I11 ma1dr"> 6 95<1a> 6< /* merge wit6 prev68a>-10 clllllllll"sref cllllll>110classf">p+code=chunk" cl8f="mm/percpli5" id="L605" clli=chu href="mercpu.c#Lnr_gr=unit5"="L60507ee[a>( 59368L5510m/p>10d<>/*>1n69"> 569m6pee( 593 6 10p8eapl59 6mahrpcpu.c#L593" warn if maximum distance arrfuren&r th>1175%lof vm6lass"un>cen*sipar "L59"L593"> 593; ma1dr""1))95 6 i69" n10id=s="line" na10I11 "PERCPU: max_distance=0x%zx too large ee 593"un>cen0x%lx\n"ipar "L5lunitnkma1dr"> 6 95<1a> 6 /* merge wit6 next69*/<11o/*>110classf">plc10"srefll(unsef="+clong) 59369 na10#ifdeflunitnk;we have epliback *sipar "L59"L593"> 593 593pc10"srefgoto lass="line" codout_free="mm/percpli5" out_freeERN_dthm/pe,"1))95 6 <"> 6 6 10d<>/*>1n69"> 569m6 ss="comme60e9e=chunk10d="L600" clas6="lin69 na10/*>1n69"> 569m7ass="comme71ode=chunk11ode=600" clas7="lin70a>-11 cl>112u= "PERCPU: Embeddits%zsfpagss/>1n69"> 569m7a1s="comme7191e=chunk10c#L592" id="L792" c70 na10d="""""""""s="liunitnkI11 ichunk5lunitnkchunk5/*>1n69"> 569m7a2s="comme7192e=chunk10>/* merge wit7 next70*/<11o/*>110classf">p+code=chunk" cl8f="mm/percpli5" id="L605" clli=chu href="mredyns hre> * 59370a>-1059"L593"> 593 7a5s="comme71e5e=chunk10e=chunk" clas7="sre70L5510m/p>10d 7a6 10+co1f="+cod36<1a> iERN_:+co1f="+cod36<1a> 10d10nkHe(); "I11 iERN_[a>(1n69"> 569m7(iERN_5*" f="mm/pe /a hrcf="mm/percpli5" id="L605" clli=chu href="m "srecpu.c#Lunit 72 "srecpu.hunk4dthm/pe,"1))95 7<1s="comme7111e=chunk11d="L612" clas7="lin71 na10lass="line" codout_free="mm/percpli5" out_freeERN_:+co1f="+cod36<1a> 10p_He(I11 7<3s="comme7113e=chunk11 7 71a>-10 cl>10 3i>; iERN_4thm/pe,"1))95 7<4s="comme7114e=chunk11hunk" class="7ref">71a>-10 clllllllll"srefchunk_relocate); "I11 iERN_45lunitnka>; a>; 7<5 7 6 /*>1n69"> 569m7 7 59310it_f="mm/pe95 7 9 10it_f="mm/pe95 7 11od="L570" class=/*"ipar "L59"L593"> 59310"n> 59310do free slots.c:ren&#f">c of 72 "sr" 593 593ch10e=c>10et doesn't da* @free=fn: func @ai:to free# 593 593 59310et doesn't da* Tharrarr"L5elper to ease#fett;zu up pags-remapp"+cf9;t+n 593710bencpli"+cwhere id="Lfet05b39;t+eef=in() ls expectit ERar "L59"L593"> 593 59311od="L570" class="* Tharrarren&#basicfpline" or. S2 "sr" 59373s="10ef=>10"n> 59310do free slots. 59310dchunk. Note th.* RETURNS:ERar "L59"L593"> 593ch10e=c>10et doesn't da* 0 ai:suc inf, -errno ai:failurv ERar "L59"L593"> 593 593(chunk559"L593"> 593(I115>ma1dr"> 6 95<1a> 7 7(); "I115>ma1dr"> 6 95<1a> 7 9 10nkine" na100nklllll+code=chunk" cl=d) ( 7ass="comme704de1chunk14id="L631" cla7s="li74a>-10_>ma1dr""1))95 7a (); " 7a 110cdmef=n" ns"+fref="+code=chu="classdumpnhine" info5" id="Lclass="dumpnhia*unitnk 7a -10 cl>10 3char" ns"+fref="+code=d="Lstrpnhine" info5" i=d="Lstr07ee[16]dthm1dr""> 593-10 cllllllin dee(10d< 7a6 pdmef=n" ns"+fref="+codeagseds hrea<_f pagspnhia**unitnk 7a7 7hnt_s74""""""""erge in dee( 7a 10it_f="mm/pe95 7pss="comme7049e=chunk10c#L550" id="L750" c74"nterestI11 "%luK"ipar "L5lunitnk 7ass="comme705de=chunk10d="L551" clas7="lin75a>-10 cl1dr"> 6 95<1a> 7a chunk5l05lunitnk 7a 110ca>; 7a 7a /* tell the 7pper 754>-11 cl>112u= 7a5 10d<iERN_5!clunitnki 59310+co1f="+cod36<1a> 6 75 10it_f="mm/pe95 75ss="comme7059e=chunk10_free_area - 7ree a75a>-11 cl>111reapl59 6mahrpcpu.c#L593"#unplef="+cpline" @ai5fcanNR_CPt0be freed, "L6mcluprto pags#>; 593-11 cl>112u= i 593 593 ( 593-1059"L593"> 593-10 clllllllapl59 6mahrpcpu.c#L593" ame=564""pagssa*sipar "L59"L593"> 593 7a hre76L5510m/p>10d< 7 peei 6 <7 href76""""""""erge with nexeema1dr""1))95 7 7(); "(); "( 59310nkkkkkkkkkvoida*unitnk 7 -10 cl1dr"> 6 95<1a> 7 ss="comme7071e=chunk10id="L572" cla7s="li77 na10d="""""""""s="li0nklllllunitnk(I11 ); " 593o77*/<11o/*>110classf">plc10"srea>; a href="de=map" ptrpnhine" info5" itrERN_4t_>ma1dr"> 6 95<1a> 7 ss="comme7073e=chunk10 class="sref"7i77a>-10 cl>10 3las110clc10"sref0nklllllunitnkI11 "PERCPU: fail"+cto ame=564" %s pags#"ipar "L59"L593"> 593"ee 593 7 pc10"srefffffffff>/*>1n69"> 569m7 <"> 6 593I11 7 ss="comme7079e=chunk10" class="sref7>BUG_77"nterest10nkkkkkkkkkI11 7/*>1n69"> 569m71n69"> 569m7< 110clapl59 6mahrpcpu.c#L593" ame=564""vm"prea, ma5len&#pagssaand copy 72 "srfdata isipar "L59"L593"> 593( 7< /* merge wit7 prev784>-11 cl>112u= i 59378L5510m/p>10d<); " 59310+co1f="+cod36<1a> 6 i 6 95<1a> 7< 6 95<1a> 710nkkkkkkkkk(unsef="+clong)= 593i79a>-10 cl1dr"> 6 95<1a> 7 ss="comme7091e=chunk10c#L592" id="L792" c79 na10d="""""""""s="liee 6 95<1a> 7 /* merge wit7 next79*/<11o/*>110classf">plc10"sre 59379a>-1059"L593"> 593 5931n69"> 569m7 ss="comme70e6e=chunk10e=memmove" cl7ss="s79 na10d=ref">pc10"sref*>110classf">plc10"sre 593 6 ; unitnk 6 95<1a> 7 I11 "fail"+cto ma5l 593 569m8ass="comme81ode=chunk11ode=600" clas8="lin80" n10id=s="line" na10 593 59310do free slots./*ar ma55;zu here - something>Iar "L59"L593"> 59380 na10d=">10dchunk. Note th.""""""""""""""""* equivalte :to flush_cacn&_vma5C4 on en&#e=56iu> 593Iar "L59"L593"> 593 593 593 59310nkHe(I11 (voida*)= ); "ichunk4dthm1dr""> 593/*>1n69"> 569m8<1s="comme8111e=chunk11d="L612" clas8="lin81 na10/*>1n69"> 569m8<2s="comme8112e=chunk11 class="sref"8chunk81*/<11o/*>110clapl59 6mahrpcpu.c#L593" weNR_CPrefready,f0" mitc*sipar "L59"L593"> 593"PERCPU: %d %s pagss/>1n69"> 569m8<4s="comme8114e=chunk11hunk" class="8ref">81a>-10 clllllllll"srefchunk_relocate< "srepagssc#Lunit 72 "srepagssipL65" ns"+fref="+code=d="Lstrpnhine" info5" i=d="Lstr07ee5" ns"+fref="+codvma>); "ichunk5/*>1n69"> 569m8<5s="comme8115e=chunk11e=kfree" clas8="sre81L5510m/p>10dchunk5p+code=chunk" cl8f="mm/percpli5" id="L605" clli=chu href="mredyns hre> * 59310+co1f="+cod36<1a> 1117ee( 593 5931n69"> 569m8 6 95<1a> 8 2s="comme81 2e=chunk11IST_HEAD" cla8s="sr82*/<11o/*>110classf">p+code=chunk" clfree=fna>); "I11 I11 593-10 cl>10 3ee( 593ch10eass="line" codout_free_crc#Lunit 72out_free_crhunk:+co1f="+cod36<1a> I11 593pee(I11 8 7 1117return href="de=map" rceds hrea<_f c="L6nit_f="mm/pe95 8 8/*>1n69"> 569m8 9 593-10 cl1dr"> 6 95<1a> 8 83s="10#ifndeflunitnk-10 cl1dr"> 6 95<1a> 8 2s="comme80i2e1chunk10ef="mm/percpu8c#L6383"sr10hun>10do free slots. 59310dchunk. Note th.* Generic SMP" 593ch10e=c>10et doesn't da*ERar "L59"L593"> 593 593 59310et doesn't da* importantsbecausi manyfprchschave pdd 7s;zu 7tric @ai5fand mightipar "L59"L593"> 5938 59310ed="L570" class="* ine" @ai A5fan pdditsbonus, insnon-NUMA cases, embedd;zu aripar "L59"L593"> 59310piggy backipar "L59"L593"> 59310"n> 59310do free slots. 593 593-10unsef="+clonglunitnk 8a5 8a6 10+co1f="+cod36<1a> 8hnt_s84"""""72 "srfvoida*dee(((); "1n69"> 569m8a pee(1n69"> 569m8a9 ma1dr"> 6 95<1a> 8ass="comme805de=chunk10d="L551" clas8="lin85" n10id=s="lireturn href="de=map" __6lass=bootmem_nor "iceds hrea<_f __6lass=bootmem_nor "icip11 i(I11 593/*>1n69"> 569m8a 1n69"> 569m8a ((1n69"> 569m8a /* tell the 8pper 854>-11_>ma1dr"> 6 95<1a> 8a5 10d<); "I11 593 569m8a<"> 6 10voidaee((i 569m8a9 ma1dr"> 6 95<1a> 8 -11 cl>112uunsef="+clonglunitnk 8 8 593-10 clllllllapl59 6mahrpcpu.c#L593"ipar "L59"L593"> 593 8a hre86lass=e=c>10et doesn't da""""""""* Always 72 "sfprea ee 593 593 6 <8 href86"""""e=c>10et doesn't da""""""""*sipar "L59"L593"> 5938(1n69"> 569m8 ss="comme8069e=chunk10an> 10nkkkkkkkkkklllllref">pee(1n69"> 569m8 pee( 593; unitnk 6 95<1a> 8 o87*/<11o/*>110classf">punitnkI11 "Fail"+cto sibiiilezi 59387a>-1059"L593"> 593 8 5 ); " 6 95<1a> 8 pc10"srefunitnk); "); " 8 <"> 6 /*>1n69"> 569m8 593 569m8 5931n69"> 569m8< 10do free slots. 59310dchunk. Note th.* UP" 593/* merge wit8 prev88>ch10e=c>10et doesn't da*ERar "L59"L593"> 59388lass=e=c>10et doesn't da* UP"plways usis km-based# 593 593 6 10et doesn't da* vari6blesfpnd donNR_CPt0requirefanyfspeciil# 59310dchunk. Note tha*sipar "L59"L593"> 593((i 569m8 i89a>-10_>ma1dr""1))95 8 ss="comme8091e=chunk10c#L592" id="L892" c89 na10d=""""""cons dee(ma1dr"> 6 95<1a> 8 /* merge wit8 next89*/<11o/*>110classf">punitnkI11 i 569m8 89a>-10 cl>10 3las110clc10"sref0nklllllclassf">punitnk 593 8 ss="comme80e5e=chunk10e=chunk" clas8="sre89L5510m/p>10d 8 ss="comme80e6e=chunk10e=memmove" cl8ss="s89f">10+co1f="+cod36<1a> 6 1117ee(I11 1, 14dthm/pe,"1))95 8 (I11 593; a href="de=map" cf="mm/percpli5" id="L6/|| a href="de=map" fceds hrea<_f fc="L64/*>1n69"> 569m9ass="comme91ode=chunk11ode=600" clas9="lin90" n10id=s="line" na10I11 "Fail"+cto ame=564""memory ee 593 593I11 59390a>-1059"L593"> 593 593 593pee( 5931117ee( 593(iERN_5= 1dthm1dr""> 593(); " 9-10 cl1dr"> 6 95<1a> 9<1s="comme9111e=chunk11d="L612" clas9="lin91 na10d=""""""a>; unitnkI11 6 95<1a> 9<2s="comme9112e=chunk11 class="sref"9chunk91*/<11o/*>110classf">punitnkI11 "Fail"+cto sibiiilezi 593 569m9<4s="comme9114e=chunk11hunk" class="9ref">91a>-10/*>1n69"> 569m9<5s="comme9115e=chunk11e=kfree" clas9="sre91L5510#endif33lapl59 6mahrpcpu.c#L593" CONFIG_SMP"*sipar "L59"L593"> 59310+co1f="+cod36<1a> 10et doesn't d3"ipar "L59"L593"> 59310dchunk. Note tha* F9;t+fpnd 72 "sr ef=insfprefinbiiilezid with temporaryfpline" @aiipar "L59"L593"> 593 59311od="L570" class="* Tharrfunc @ai irrcpliitsafter slab irrb"L6ghtluprpnd placescenoss>Iar "L59"L593"> 59310"n> 59310do free slots. 593-10voidaee((I11 void4/*>1n69"> 569m9 4s="comme91 4e=chunk11 class="sref"9chunk924>-11_>ma1dr"> 6 95<1a> 9 5s="comme90 5e1chunk10d="L626" clas9="lin92L5510m/p>10dma1dr"> 6 95<1a> 9 6 pc10"sref{pee( 5931117dmef=n" ns"+fref="+code=chuef=inpnhine" info5" id="Lef=in>I11a*unitnkI11dthm1dr""> 5939 593( 593-10 cl1dr"> 6 95<1a> 9 93 na10d=""""""eeI115" unitnkma1dr""1))95 9 2s="comme90i2e1chunk10ef="mm/percpu9c#L6393*/<11o/*>110classf">pan d*unitnk); " 593I11a*e_widthm); " 593-10/*>1n69"> 569m9 5s="comme90i5e1chunk10id="L636" cla9s="li93L5510m/p>10di 59310+co1f="+cod36<1a> ); " 5939); " 593 569m9ass="comme904de1chunk14id="L631" cla9s="li94" n10id=s="line" na10 593I11 href="de=map" e); "I1105" clli=chu href="me); " 593punitnkI1105" clli=chu href="me); "); " 593-10 cl>10 3las110cl 593-11 cl>112u>/*>1n69"> 569m9a5 /*>1n69"> 569m9a6 10
Tn&#original LXR softwprefbycen&#*>1n69"> http://sourceeeLXR sloref"y="L65ltharrex1n69"> 5ailto:lxr@4d=ux.no">lxr@4d=ux.no="L6.
lxr.4d=ux.no kindly hos itsbyc*>1n69"> http://www.redpill-4d=pro.no">Redpill Ld=pro AS="L65lprovider of Ld=uxlconsultingrpnd o