linux/mm/memblock.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Procedures for maintaining information about logical memory blocks.
   4 *
   5 * Peter Bergner, IBM Corp.     June 2001.
   6 * Copyright (C) 2001 Peter Bergner.
   7 */
   8
   9#include <linux/kernel.h>
  10#include <linux/slab.h>
  11#include <linux/init.h>
  12#include <linux/bitops.h>
  13#include <linux/poison.h>
  14#include <linux/pfn.h>
  15#include <linux/debugfs.h>
  16#include <linux/kmemleak.h>
  17#include <linux/seq_file.h>
  18#include <linux/memblock.h>
  19
  20#include <asm/sections.h>
  21#include <linux/io.h>
  22
  23#include "internal.h"
  24
  25#define INIT_MEMBLOCK_REGIONS                   128
  26#define INIT_PHYSMEM_REGIONS                    4
  27
  28#ifndef INIT_MEMBLOCK_RESERVED_REGIONS
  29# define INIT_MEMBLOCK_RESERVED_REGIONS         INIT_MEMBLOCK_REGIONS
  30#endif
  31
  32/**
  33 * DOC: memblock overview
  34 *
  35 * Memblock is a method of managing memory regions during the early
  36 * boot period when the usual kernel memory allocators are not up and
  37 * running.
  38 *
  39 * Memblock views the system memory as collections of contiguous
  40 * regions. There are several types of these collections:
  41 *
  42 * * ``memory`` - describes the physical memory available to the
  43 *   kernel; this may differ from the actual physical memory installed
  44 *   in the system, for instance when the memory is restricted with
  45 *   ``mem=`` command line parameter
  46 * * ``reserved`` - describes the regions that were allocated
  47 * * ``physmem`` - describes the actual physical memory available during
  48 *   boot regardless of the possible restrictions and memory hot(un)plug;
  49 *   the ``physmem`` type is only available on some architectures.
  50 *
  51 * Each region is represented by struct memblock_region that
  52 * defines the region extents, its attributes and NUMA node id on NUMA
  53 * systems. Every memory type is described by the struct memblock_type
  54 * which contains an array of memory regions along with
  55 * the allocator metadata. The "memory" and "reserved" types are nicely
  56 * wrapped with struct memblock. This structure is statically
  57 * initialized at build time. The region arrays are initially sized to
  58 * %INIT_MEMBLOCK_REGIONS for "memory" and %INIT_MEMBLOCK_RESERVED_REGIONS
  59 * for "reserved". The region array for "physmem" is initially sized to
  60 * %INIT_PHYSMEM_REGIONS.
  61 * The memblock_allow_resize() enables automatic resizing of the region
  62 * arrays during addition of new regions. This feature should be used
  63 * with care so that memory allocated for the region array will not
  64 * overlap with areas that should be reserved, for example initrd.
  65 *
  66 * The early architecture setup should tell memblock what the physical
  67 * memory layout is by using memblock_add() or memblock_add_node()
  68 * functions. The first function does not assign the region to a NUMA
  69 * node and it is appropriate for UMA systems. Yet, it is possible to
  70 * use it on NUMA systems as well and assign the region to a NUMA node
  71 * later in the setup process using memblock_set_node(). The
  72 * memblock_add_node() performs such an assignment directly.
  73 *
  74 * Once memblock is setup the memory can be allocated using one of the
  75 * API variants:
  76 *
  77 * * memblock_phys_alloc*() - these functions return the **physical**
  78 *   address of the allocated memory
  79 * * memblock_alloc*() - these functions return the **virtual** address
  80 *   of the allocated memory.
  81 *
  82 * Note, that both API variants use implicit assumptions about allowed
  83 * memory ranges and the fallback methods. Consult the documentation
  84 * of memblock_alloc_internal() and memblock_alloc_range_nid()
  85 * functions for more elaborate description.
  86 *
  87 * As the system boot progresses, the architecture specific mem_init()
  88 * function frees all the memory to the buddy page allocator.
  89 *
  90 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
  91 * memblock data structures (except "physmem") will be discarded after the
  92 * system initialization completes.
  93 */
  94
  95#ifndef CONFIG_NEED_MULTIPLE_NODES
  96struct pglist_data __refdata contig_page_data;
  97EXPORT_SYMBOL(contig_page_data);
  98#endif
  99
 100unsigned long max_low_pfn;
 101unsigned long min_low_pfn;
 102unsigned long max_pfn;
 103unsigned long long max_possible_pfn;
 104
 105static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS] __initdata_memblock;
 106static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock;
 107#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
 108static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS];
 109#endif
 110
 111struct memblock memblock __initdata_memblock = {
 112        .memory.regions         = memblock_memory_init_regions,
 113        .memory.cnt             = 1,    /* empty dummy entry */
 114        .memory.max             = INIT_MEMBLOCK_REGIONS,
 115        .memory.name            = "memory",
 116
 117        .reserved.regions       = memblock_reserved_init_regions,
 118        .reserved.cnt           = 1,    /* empty dummy entry */
 119        .reserved.max           = INIT_MEMBLOCK_RESERVED_REGIONS,
 120        .reserved.name          = "reserved",
 121
 122        .bottom_up              = false,
 123        .current_limit          = MEMBLOCK_ALLOC_ANYWHERE,
 124};
 125
 126#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
 127struct memblock_type physmem = {
 128        .regions                = memblock_physmem_init_regions,
 129        .cnt                    = 1,    /* empty dummy entry */
 130        .max                    = INIT_PHYSMEM_REGIONS,
 131        .name                   = "physmem",
 132};
 133#endif
 134
 135/*
 136 * keep a pointer to &memblock.memory in the text section to use it in
 137 * __next_mem_range() and its helpers.
 138 *  For architectures that do not keep memblock data after init, this
 139 * pointer will be reset to NULL at memblock_discard()
 140 */
 141static __refdata struct memblock_type *memblock_memory = &memblock.memory;
 142
 143#define for_each_memblock_type(i, memblock_type, rgn)                   \
 144        for (i = 0, rgn = &memblock_type->regions[0];                   \
 145             i < memblock_type->cnt;                                    \
 146             i++, rgn = &memblock_type->regions[i])
 147
 148#define memblock_dbg(fmt, ...)                                          \
 149        do {                                                            \
 150                if (memblock_debug)                                     \
 151                        pr_info(fmt, ##__VA_ARGS__);                    \
 152        } while (0)
 153
 154static int memblock_debug __initdata_memblock;
 155static bool system_has_some_mirror __initdata_memblock = false;
 156static int memblock_can_resize __initdata_memblock;
 157static int memblock_memory_in_slab __initdata_memblock = 0;
 158static int memblock_reserved_in_slab __initdata_memblock = 0;
 159
 160static enum memblock_flags __init_memblock choose_memblock_flags(void)
 161{
 162        return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
 163}
 164
 165/* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
 166static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size)
 167{
 168        return *size = min(*size, PHYS_ADDR_MAX - base);
 169}
 170
 171/*
 172 * Address comparison utilities
 173 */
 174static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1,
 175                                       phys_addr_t base2, phys_addr_t size2)
 176{
 177        return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
 178}
 179
 180bool __init_memblock memblock_overlaps_region(struct memblock_type *type,
 181                                        phys_addr_t base, phys_addr_t size)
 182{
 183        unsigned long i;
 184
 185        for (i = 0; i < type->cnt; i++)
 186                if (memblock_addrs_overlap(base, size, type->regions[i].base,
 187                                           type->regions[i].size))
 188                        break;
 189        return i < type->cnt;
 190}
 191
 192/**
 193 * __memblock_find_range_bottom_up - find free area utility in bottom-up
 194 * @start: start of candidate range
 195 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
 196 *       %MEMBLOCK_ALLOC_ACCESSIBLE
 197 * @size: size of free area to find
 198 * @align: alignment of free area to find
 199 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
 200 * @flags: pick from blocks based on memory attributes
 201 *
 202 * Utility called from memblock_find_in_range_node(), find free area bottom-up.
 203 *
 204 * Return:
 205 * Found address on success, 0 on failure.
 206 */
 207static phys_addr_t __init_memblock
 208__memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end,
 209                                phys_addr_t size, phys_addr_t align, int nid,
 210                                enum memblock_flags flags)
 211{
 212        phys_addr_t this_start, this_end, cand;
 213        u64 i;
 214
 215        for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) {
 216                this_start = clamp(this_start, start, end);
 217                this_end = clamp(this_end, start, end);
 218
 219                cand = round_up(this_start, align);
 220                if (cand < this_end && this_end - cand >= size)
 221                        return cand;
 222        }
 223
 224        return 0;
 225}
 226
 227/**
 228 * __memblock_find_range_top_down - find free area utility, in top-down
 229 * @start: start of candidate range
 230 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
 231 *       %MEMBLOCK_ALLOC_ACCESSIBLE
 232 * @size: size of free area to find
 233 * @align: alignment of free area to find
 234 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
 235 * @flags: pick from blocks based on memory attributes
 236 *
 237 * Utility called from memblock_find_in_range_node(), find free area top-down.
 238 *
 239 * Return:
 240 * Found address on success, 0 on failure.
 241 */
 242static phys_addr_t __init_memblock
 243__memblock_find_range_top_down(phys_addr_t start, phys_addr_t end,
 244                               phys_addr_t size, phys_addr_t align, int nid,
 245                               enum memblock_flags flags)
 246{
 247        phys_addr_t this_start, this_end, cand;
 248        u64 i;
 249
 250        for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end,
 251                                        NULL) {
 252                this_start = clamp(this_start, start, end);
 253                this_end = clamp(this_end, start, end);
 254
 255                if (this_end < size)
 256                        continue;
 257
 258                cand = round_down(this_end - size, align);
 259                if (cand >= this_start)
 260                        return cand;
 261        }
 262
 263        return 0;
 264}
 265
 266/**
 267 * memblock_find_in_range_node - find free area in given range and node
 268 * @size: size of free area to find
 269 * @align: alignment of free area to find
 270 * @start: start of candidate range
 271 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
 272 *       %MEMBLOCK_ALLOC_ACCESSIBLE
 273 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
 274 * @flags: pick from blocks based on memory attributes
 275 *
 276 * Find @size free area aligned to @align in the specified range and node.
 277 *
 278 * Return:
 279 * Found address on success, 0 on failure.
 280 */
 281static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
 282                                        phys_addr_t align, phys_addr_t start,
 283                                        phys_addr_t end, int nid,
 284                                        enum memblock_flags flags)
 285{
 286        /* pump up @end */
 287        if (end == MEMBLOCK_ALLOC_ACCESSIBLE ||
 288            end == MEMBLOCK_ALLOC_KASAN)
 289                end = memblock.current_limit;
 290
 291        /* avoid allocating the first page */
 292        start = max_t(phys_addr_t, start, PAGE_SIZE);
 293        end = max(start, end);
 294
 295        if (memblock_bottom_up())
 296                return __memblock_find_range_bottom_up(start, end, size, align,
 297                                                       nid, flags);
 298        else
 299                return __memblock_find_range_top_down(start, end, size, align,
 300                                                      nid, flags);
 301}
 302
 303/**
 304 * memblock_find_in_range - find free area in given range
 305 * @start: start of candidate range
 306 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
 307 *       %MEMBLOCK_ALLOC_ACCESSIBLE
 308 * @size: size of free area to find
 309 * @align: alignment of free area to find
 310 *
 311 * Find @size free area aligned to @align in the specified range.
 312 *
 313 * Return:
 314 * Found address on success, 0 on failure.
 315 */
 316phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start,
 317                                        phys_addr_t end, phys_addr_t size,
 318                                        phys_addr_t align)
 319{
 320        phys_addr_t ret;
 321        enum memblock_flags flags = choose_memblock_flags();
 322
 323again:
 324        ret = memblock_find_in_range_node(size, align, start, end,
 325                                            NUMA_NO_NODE, flags);
 326
 327        if (!ret && (flags & MEMBLOCK_MIRROR)) {
 328                pr_warn("Could not allocate %pap bytes of mirrored memory\n",
 329                        &size);
 330                flags &= ~MEMBLOCK_MIRROR;
 331                goto again;
 332        }
 333
 334        return ret;
 335}
 336
 337static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
 338{
 339        type->total_size -= type->regions[r].size;
 340        memmove(&type->regions[r], &type->regions[r + 1],
 341                (type->cnt - (r + 1)) * sizeof(type->regions[r]));
 342        type->cnt--;
 343
 344        /* Special case for empty arrays */
 345        if (type->cnt == 0) {
 346                WARN_ON(type->total_size != 0);
 347                type->cnt = 1;
 348                type->regions[0].base = 0;
 349                type->regions[0].size = 0;
 350                type->regions[0].flags = 0;
 351                memblock_set_region_node(&type->regions[0], MAX_NUMNODES);
 352        }
 353}
 354
 355#ifndef CONFIG_ARCH_KEEP_MEMBLOCK
 356/**
 357 * memblock_discard - discard memory and reserved arrays if they were allocated
 358 */
 359void __init memblock_discard(void)
 360{
 361        phys_addr_t addr, size;
 362
 363        if (memblock.reserved.regions != memblock_reserved_init_regions) {
 364                addr = __pa(memblock.reserved.regions);
 365                size = PAGE_ALIGN(sizeof(struct memblock_region) *
 366                                  memblock.reserved.max);
 367                __memblock_free_late(addr, size);
 368        }
 369
 370        if (memblock.memory.regions != memblock_memory_init_regions) {
 371                addr = __pa(memblock.memory.regions);
 372                size = PAGE_ALIGN(sizeof(struct memblock_region) *
 373                                  memblock.memory.max);
 374                __memblock_free_late(addr, size);
 375        }
 376
 377        memblock_memory = NULL;
 378}
 379#endif
 380
 381/**
 382 * memblock_double_array - double the size of the memblock regions array
 383 * @type: memblock type of the regions array being doubled
 384 * @new_area_start: starting address of memory range to avoid overlap with
 385 * @new_area_size: size of memory range to avoid overlap with
 386 *
 387 * Double the size of the @type regions array. If memblock is being used to
 388 * allocate memory for a new reserved regions array and there is a previously
 389 * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
 390 * waiting to be reserved, ensure the memory used by the new array does
 391 * not overlap.
 392 *
 393 * Return:
 394 * 0 on success, -1 on failure.
 395 */
 396static int __init_memblock memblock_double_array(struct memblock_type *type,
 397                                                phys_addr_t new_area_start,
 398                                                phys_addr_t new_area_size)
 399{
 400        struct memblock_region *new_array, *old_array;
 401        phys_addr_t old_alloc_size, new_alloc_size;
 402        phys_addr_t old_size, new_size, addr, new_end;
 403        int use_slab = slab_is_available();
 404        int *in_slab;
 405
 406        /* We don't allow resizing until we know about the reserved regions
 407         * of memory that aren't suitable for allocation
 408         */
 409        if (!memblock_can_resize)
 410                return -1;
 411
 412        /* Calculate new doubled size */
 413        old_size = type->max * sizeof(struct memblock_region);
 414        new_size = old_size << 1;
 415        /*
 416         * We need to allocated new one align to PAGE_SIZE,
 417         *   so we can free them completely later.
 418         */
 419        old_alloc_size = PAGE_ALIGN(old_size);
 420        new_alloc_size = PAGE_ALIGN(new_size);
 421
 422        /* Retrieve the slab flag */
 423        if (type == &memblock.memory)
 424                in_slab = &memblock_memory_in_slab;
 425        else
 426                in_slab = &memblock_reserved_in_slab;
 427
 428        /* Try to find some space for it */
 429        if (use_slab) {
 430                new_array = kmalloc(new_size, GFP_KERNEL);
 431                addr = new_array ? __pa(new_array) : 0;
 432        } else {
 433                /* only exclude range when trying to double reserved.regions */
 434                if (type != &memblock.reserved)
 435                        new_area_start = new_area_size = 0;
 436
 437                addr = memblock_find_in_range(new_area_start + new_area_size,
 438                                                memblock.current_limit,
 439                                                new_alloc_size, PAGE_SIZE);
 440                if (!addr && new_area_size)
 441                        addr = memblock_find_in_range(0,
 442                                min(new_area_start, memblock.current_limit),
 443                                new_alloc_size, PAGE_SIZE);
 444
 445                new_array = addr ? __va(addr) : NULL;
 446        }
 447        if (!addr) {
 448                pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
 449                       type->name, type->max, type->max * 2);
 450                return -1;
 451        }
 452
 453        new_end = addr + new_size - 1;
 454        memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
 455                        type->name, type->max * 2, &addr, &new_end);
 456
 457        /*
 458         * Found space, we now need to move the array over before we add the
 459         * reserved region since it may be our reserved array itself that is
 460         * full.
 461         */
 462        memcpy(new_array, type->regions, old_size);
 463        memset(new_array + type->max, 0, old_size);
 464        old_array = type->regions;
 465        type->regions = new_array;
 466        type->max <<= 1;
 467
 468        /* Free old array. We needn't free it if the array is the static one */
 469        if (*in_slab)
 470                kfree(old_array);
 471        else if (old_array != memblock_memory_init_regions &&
 472                 old_array != memblock_reserved_init_regions)
 473                memblock_free(__pa(old_array), old_alloc_size);
 474
 475        /*
 476         * Reserve the new array if that comes from the memblock.  Otherwise, we
 477         * needn't do it
 478         */
 479        if (!use_slab)
 480                BUG_ON(memblock_reserve(addr, new_alloc_size));
 481
 482        /* Update slab flag */
 483        *in_slab = use_slab;
 484
 485        return 0;
 486}
 487
 488/**
 489 * memblock_merge_regions - merge neighboring compatible regions
 490 * @type: memblock type to scan
 491 *
 492 * Scan @type and merge neighboring compatible regions.
 493 */
 494static void __init_memblock memblock_merge_regions(struct memblock_type *type)
 495{
 496        int i = 0;
 497
 498        /* cnt never goes below 1 */
 499        while (i < type->cnt - 1) {
 500                struct memblock_region *this = &type->regions[i];
 501                struct memblock_region *next = &type->regions[i + 1];
 502
 503                if (this->base + this->size != next->base ||
 504                    memblock_get_region_node(this) !=
 505                    memblock_get_region_node(next) ||
 506                    this->flags != next->flags) {
 507                        BUG_ON(this->base + this->size > next->base);
 508                        i++;
 509                        continue;
 510                }
 511
 512                this->size += next->size;
 513                /* move forward from next + 1, index of which is i + 2 */
 514                memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next));
 515                type->cnt--;
 516        }
 517}
 518
 519/**
 520 * memblock_insert_region - insert new memblock region
 521 * @type:       memblock type to insert into
 522 * @idx:        index for the insertion point
 523 * @base:       base address of the new region
 524 * @size:       size of the new region
 525 * @nid:        node id of the new region
 526 * @flags:      flags of the new region
 527 *
 528 * Insert new memblock region [@base, @base + @size) into @type at @idx.
 529 * @type must already have extra room to accommodate the new region.
 530 */
 531static void __init_memblock memblock_insert_region(struct memblock_type *type,
 532                                                   int idx, phys_addr_t base,
 533                                                   phys_addr_t size,
 534                                                   int nid,
 535                                                   enum memblock_flags flags)
 536{
 537        struct memblock_region *rgn = &type->regions[idx];
 538
 539        BUG_ON(type->cnt >= type->max);
 540        memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn));
 541        rgn->base = base;
 542        rgn->size = size;
 543        rgn->flags = flags;
 544        memblock_set_region_node(rgn, nid);
 545        type->cnt++;
 546        type->total_size += size;
 547}
 548
 549/**
 550 * memblock_add_range - add new memblock region
 551 * @type: memblock type to add new region into
 552 * @base: base address of the new region
 553 * @size: size of the new region
 554 * @nid: nid of the new region
 555 * @flags: flags of the new region
 556 *
 557 * Add new memblock region [@base, @base + @size) into @type.  The new region
 558 * is allowed to overlap with existing ones - overlaps don't affect already
 559 * existing regions.  @type is guaranteed to be minimal (all neighbouring
 560 * compatible regions are merged) after the addition.
 561 *
 562 * Return:
 563 * 0 on success, -errno on failure.
 564 */
 565static int __init_memblock memblock_add_range(struct memblock_type *type,
 566                                phys_addr_t base, phys_addr_t size,
 567                                int nid, enum memblock_flags flags)
 568{
 569        bool insert = false;
 570        phys_addr_t obase = base;
 571        phys_addr_t end = base + memblock_cap_size(base, &size);
 572        int idx, nr_new;
 573        struct memblock_region *rgn;
 574
 575        if (!size)
 576                return 0;
 577
 578        /* special case for empty array */
 579        if (type->regions[0].size == 0) {
 580                WARN_ON(type->cnt != 1 || type->total_size);
 581                type->regions[0].base = base;
 582                type->regions[0].size = size;
 583                type->regions[0].flags = flags;
 584                memblock_set_region_node(&type->regions[0], nid);
 585                type->total_size = size;
 586                return 0;
 587        }
 588repeat:
 589        /*
 590         * The following is executed twice.  Once with %false @insert and
 591         * then with %true.  The first counts the number of regions needed
 592         * to accommodate the new area.  The second actually inserts them.
 593         */
 594        base = obase;
 595        nr_new = 0;
 596
 597        for_each_memblock_type(idx, type, rgn) {
 598                phys_addr_t rbase = rgn->base;
 599                phys_addr_t rend = rbase + rgn->size;
 600
 601                if (rbase >= end)
 602                        break;
 603                if (rend <= base)
 604                        continue;
 605                /*
 606                 * @rgn overlaps.  If it separates the lower part of new
 607                 * area, insert that portion.
 608                 */
 609                if (rbase > base) {
 610#ifdef CONFIG_NEED_MULTIPLE_NODES
 611                        WARN_ON(nid != memblock_get_region_node(rgn));
 612#endif
 613                        WARN_ON(flags != rgn->flags);
 614                        nr_new++;
 615                        if (insert)
 616                                memblock_insert_region(type, idx++, base,
 617                                                       rbase - base, nid,
 618                                                       flags);
 619                }
 620                /* area below @rend is dealt with, forget about it */
 621                base = min(rend, end);
 622        }
 623
 624        /* insert the remaining portion */
 625        if (base < end) {
 626                nr_new++;
 627                if (insert)
 628                        memblock_insert_region(type, idx, base, end - base,
 629                                               nid, flags);
 630        }
 631
 632        if (!nr_new)
 633                return 0;
 634
 635        /*
 636         * If this was the first round, resize array and repeat for actual
 637         * insertions; otherwise, merge and return.
 638         */
 639        if (!insert) {
 640                while (type->cnt + nr_new > type->max)
 641                        if (memblock_double_array(type, obase, size) < 0)
 642                                return -ENOMEM;
 643                insert = true;
 644                goto repeat;
 645        } else {
 646                memblock_merge_regions(type);
 647                return 0;
 648        }
 649}
 650
 651/**
 652 * memblock_add_node - add new memblock region within a NUMA node
 653 * @base: base address of the new region
 654 * @size: size of the new region
 655 * @nid: nid of the new region
 656 *
 657 * Add new memblock region [@base, @base + @size) to the "memory"
 658 * type. See memblock_add_range() description for mode details
 659 *
 660 * Return:
 661 * 0 on success, -errno on failure.
 662 */
 663int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size,
 664                                       int nid)
 665{
 666        return memblock_add_range(&memblock.memory, base, size, nid, 0);
 667}
 668
 669/**
 670 * memblock_add - add new memblock region
 671 * @base: base address of the new region
 672 * @size: size of the new region
 673 *
 674 * Add new memblock region [@base, @base + @size) to the "memory"
 675 * type. See memblock_add_range() description for mode details
 676 *
 677 * Return:
 678 * 0 on success, -errno on failure.
 679 */
 680int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
 681{
 682        phys_addr_t end = base + size - 1;
 683
 684        memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
 685                     &base, &end, (void *)_RET_IP_);
 686
 687        return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
 688}
 689
 690/**
 691 * memblock_isolate_range - isolate given range into disjoint memblocks
 692 * @type: memblock type to isolate range for
 693 * @base: base of range to isolate
 694 * @size: size of range to isolate
 695 * @start_rgn: out parameter for the start of isolated region
 696 * @end_rgn: out parameter for the end of isolated region
 697 *
 698 * Walk @type and ensure that regions don't cross the boundaries defined by
 699 * [@base, @base + @size).  Crossing regions are split at the boundaries,
 700 * which may create at most two more regions.  The index of the first
 701 * region inside the range is returned in *@start_rgn and end in *@end_rgn.
 702 *
 703 * Return:
 704 * 0 on success, -errno on failure.
 705 */
 706static int __init_memblock memblock_isolate_range(struct memblock_type *type,
 707                                        phys_addr_t base, phys_addr_t size,
 708                                        int *start_rgn, int *end_rgn)
 709{
 710        phys_addr_t end = base + memblock_cap_size(base, &size);
 711        int idx;
 712        struct memblock_region *rgn;
 713
 714        *start_rgn = *end_rgn = 0;
 715
 716        if (!size)
 717                return 0;
 718
 719        /* we'll create at most two more regions */
 720        while (type->cnt + 2 > type->max)
 721                if (memblock_double_array(type, base, size) < 0)
 722                        return -ENOMEM;
 723
 724        for_each_memblock_type(idx, type, rgn) {
 725                phys_addr_t rbase = rgn->base;
 726                phys_addr_t rend = rbase + rgn->size;
 727
 728                if (rbase >= end)
 729                        break;
 730                if (rend <= base)
 731                        continue;
 732
 733                if (rbase < base) {
 734                        /*
 735                         * @rgn intersects from below.  Split and continue
 736                         * to process the next region - the new top half.
 737                         */
 738                        rgn->base = base;
 739                        rgn->size -= base - rbase;
 740                        type->total_size -= base - rbase;
 741                        memblock_insert_region(type, idx, rbase, base - rbase,
 742                                               memblock_get_region_node(rgn),
 743                                               rgn->flags);
 744                } else if (rend > end) {
 745                        /*
 746                         * @rgn intersects from above.  Split and redo the
 747                         * current region - the new bottom half.
 748                         */
 749                        rgn->base = end;
 750                        rgn->size -= end - rbase;
 751                        type->total_size -= end - rbase;
 752                        memblock_insert_region(type, idx--, rbase, end - rbase,
 753                                               memblock_get_region_node(rgn),
 754                                               rgn->flags);
 755                } else {
 756                        /* @rgn is fully contained, record it */
 757                        if (!*end_rgn)
 758                                *start_rgn = idx;
 759                        *end_rgn = idx + 1;
 760                }
 761        }
 762
 763        return 0;
 764}
 765
 766static int __init_memblock memblock_remove_range(struct memblock_type *type,
 767                                          phys_addr_t base, phys_addr_t size)
 768{
 769        int start_rgn, end_rgn;
 770        int i, ret;
 771
 772        ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
 773        if (ret)
 774                return ret;
 775
 776        for (i = end_rgn - 1; i >= start_rgn; i--)
 777                memblock_remove_region(type, i);
 778        return 0;
 779}
 780
 781int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
 782{
 783        phys_addr_t end = base + size - 1;
 784
 785        memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
 786                     &base, &end, (void *)_RET_IP_);
 787
 788        return memblock_remove_range(&memblock.memory, base, size);
 789}
 790
 791/**
 792 * memblock_free - free boot memory block
 793 * @base: phys starting address of the  boot memory block
 794 * @size: size of the boot memory block in bytes
 795 *
 796 * Free boot memory block previously allocated by memblock_alloc_xx() API.
 797 * The freeing memory will not be released to the buddy allocator.
 798 */
 799int __init_memblock memblock_free(phys_addr_t base, phys_addr_t size)
 800{
 801        phys_addr_t end = base + size - 1;
 802
 803        memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
 804                     &base, &end, (void *)_RET_IP_);
 805
 806        kmemleak_free_part_phys(base, size);
 807        return memblock_remove_range(&memblock.reserved, base, size);
 808}
 809
 810int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
 811{
 812        phys_addr_t end = base + size - 1;
 813
 814        memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
 815                     &base, &end, (void *)_RET_IP_);
 816
 817        return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
 818}
 819
 820#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
 821int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size)
 822{
 823        phys_addr_t end = base + size - 1;
 824
 825        memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
 826                     &base, &end, (void *)_RET_IP_);
 827
 828        return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0);
 829}
 830#endif
 831
 832/**
 833 * memblock_setclr_flag - set or clear flag for a memory region
 834 * @base: base address of the region
 835 * @size: size of the region
 836 * @set: set or clear the flag
 837 * @flag: the flag to update
 838 *
 839 * This function isolates region [@base, @base + @size), and sets/clears flag
 840 *
 841 * Return: 0 on success, -errno on failure.
 842 */
 843static int __init_memblock memblock_setclr_flag(phys_addr_t base,
 844                                phys_addr_t size, int set, int flag)
 845{
 846        struct memblock_type *type = &memblock.memory;
 847        int i, ret, start_rgn, end_rgn;
 848
 849        ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
 850        if (ret)
 851                return ret;
 852
 853        for (i = start_rgn; i < end_rgn; i++) {
 854                struct memblock_region *r = &type->regions[i];
 855
 856                if (set)
 857                        r->flags |= flag;
 858                else
 859                        r->flags &= ~flag;
 860        }
 861
 862        memblock_merge_regions(type);
 863        return 0;
 864}
 865
 866/**
 867 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
 868 * @base: the base phys addr of the region
 869 * @size: the size of the region
 870 *
 871 * Return: 0 on success, -errno on failure.
 872 */
 873int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size)
 874{
 875        return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG);
 876}
 877
 878/**
 879 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
 880 * @base: the base phys addr of the region
 881 * @size: the size of the region
 882 *
 883 * Return: 0 on success, -errno on failure.
 884 */
 885int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size)
 886{
 887        return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG);
 888}
 889
 890/**
 891 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
 892 * @base: the base phys addr of the region
 893 * @size: the size of the region
 894 *
 895 * Return: 0 on success, -errno on failure.
 896 */
 897int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size)
 898{
 899        system_has_some_mirror = true;
 900
 901        return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR);
 902}
 903
 904/**
 905 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
 906 * @base: the base phys addr of the region
 907 * @size: the size of the region
 908 *
 909 * Return: 0 on success, -errno on failure.
 910 */
 911int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
 912{
 913        return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP);
 914}
 915
 916/**
 917 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
 918 * @base: the base phys addr of the region
 919 * @size: the size of the region
 920 *
 921 * Return: 0 on success, -errno on failure.
 922 */
 923int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
 924{
 925        return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
 926}
 927
 928static bool should_skip_region(struct memblock_type *type,
 929                               struct memblock_region *m,
 930                               int nid, int flags)
 931{
 932        int m_nid = memblock_get_region_node(m);
 933
 934        /* we never skip regions when iterating memblock.reserved or physmem */
 935        if (type != memblock_memory)
 936                return false;
 937
 938        /* only memory regions are associated with nodes, check it */
 939        if (nid != NUMA_NO_NODE && nid != m_nid)
 940                return true;
 941
 942        /* skip hotpluggable memory regions if needed */
 943        if (movable_node_is_enabled() && memblock_is_hotpluggable(m) &&
 944            !(flags & MEMBLOCK_HOTPLUG))
 945                return true;
 946
 947        /* if we want mirror memory skip non-mirror memory regions */
 948        if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
 949                return true;
 950
 951        /* skip nomap memory unless we were asked for it explicitly */
 952        if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
 953                return true;
 954
 955        return false;
 956}
 957
 958/**
 959 * __next_mem_range - next function for for_each_free_mem_range() etc.
 960 * @idx: pointer to u64 loop variable
 961 * @nid: node selector, %NUMA_NO_NODE for all nodes
 962 * @flags: pick from blocks based on memory attributes
 963 * @type_a: pointer to memblock_type from where the range is taken
 964 * @type_b: pointer to memblock_type which excludes memory from being taken
 965 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
 966 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
 967 * @out_nid: ptr to int for nid of the range, can be %NULL
 968 *
 969 * Find the first area from *@idx which matches @nid, fill the out
 970 * parameters, and update *@idx for the next iteration.  The lower 32bit of
 971 * *@idx contains index into type_a and the upper 32bit indexes the
 972 * areas before each region in type_b.  For example, if type_b regions
 973 * look like the following,
 974 *
 975 *      0:[0-16), 1:[32-48), 2:[128-130)
 976 *
 977 * The upper 32bit indexes the following regions.
 978 *
 979 *      0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
 980 *
 981 * As both region arrays are sorted, the function advances the two indices
 982 * in lockstep and returns each intersection.
 983 */
 984void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags,
 985                      struct memblock_type *type_a,
 986                      struct memblock_type *type_b, phys_addr_t *out_start,
 987                      phys_addr_t *out_end, int *out_nid)
 988{
 989        int idx_a = *idx & 0xffffffff;
 990        int idx_b = *idx >> 32;
 991
 992        if (WARN_ONCE(nid == MAX_NUMNODES,
 993        "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
 994                nid = NUMA_NO_NODE;
 995
 996        for (; idx_a < type_a->cnt; idx_a++) {
 997                struct memblock_region *m = &type_a->regions[idx_a];
 998
 999                phys_addr_t m_start = m->base;
1000                phys_addr_t m_end = m->base + m->size;
1001                int         m_nid = memblock_get_region_node(m);
1002
1003                if (should_skip_region(type_a, m, nid, flags))
1004                        continue;
1005
1006                if (!type_b) {
1007                        if (out_start)
1008                                *out_start = m_start;
1009                        if (out_end)
1010                                *out_end = m_end;
1011                        if (out_nid)
1012                                *out_nid = m_nid;
1013                        idx_a++;
1014                        *idx = (u32)idx_a | (u64)idx_b << 32;
1015                        return;
1016                }
1017
1018                /* scan areas before each reservation */
1019                for (; idx_b < type_b->cnt + 1; idx_b++) {
1020                        struct memblock_region *r;
1021                        phys_addr_t r_start;
1022                        phys_addr_t r_end;
1023
1024                        r = &type_b->regions[idx_b];
1025                        r_start = idx_b ? r[-1].base + r[-1].size : 0;
1026                        r_end = idx_b < type_b->cnt ?
1027                                r->base : PHYS_ADDR_MAX;
1028
1029                        /*
1030                         * if idx_b advanced past idx_a,
1031                         * break out to advance idx_a
1032                         */
1033                        if (r_start >= m_end)
1034                                break;
1035                        /* if the two regions intersect, we're done */
1036                        if (m_start < r_end) {
1037                                if (out_start)
1038                                        *out_start =
1039                                                max(m_start, r_start);
1040                                if (out_end)
1041                                        *out_end = min(m_end, r_end);
1042                                if (out_nid)
1043                                        *out_nid = m_nid;
1044                                /*
1045                                 * The region which ends first is
1046                                 * advanced for the next iteration.
1047                                 */
1048                                if (m_end <= r_end)
1049                                        idx_a++;
1050                                else
1051                                        idx_b++;
1052                                *idx = (u32)idx_a | (u64)idx_b << 32;
1053                                return;
1054                        }
1055                }
1056        }
1057
1058        /* signal end of iteration */
1059        *idx = ULLONG_MAX;
1060}
1061
1062/**
1063 * __next_mem_range_rev - generic next function for for_each_*_range_rev()
1064 *
1065 * @idx: pointer to u64 loop variable
1066 * @nid: node selector, %NUMA_NO_NODE for all nodes
1067 * @flags: pick from blocks based on memory attributes
1068 * @type_a: pointer to memblock_type from where the range is taken
1069 * @type_b: pointer to memblock_type which excludes memory from being taken
1070 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1071 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1072 * @out_nid: ptr to int for nid of the range, can be %NULL
1073 *
1074 * Finds the next range from type_a which is not marked as unsuitable
1075 * in type_b.
1076 *
1077 * Reverse of __next_mem_range().
1078 */
1079void __init_memblock __next_mem_range_rev(u64 *idx, int nid,
1080                                          enum memblock_flags flags,
1081                                          struct memblock_type *type_a,
1082                                          struct memblock_type *type_b,
1083                                          phys_addr_t *out_start,
1084                                          phys_addr_t *out_end, int *out_nid)
1085{
1086        int idx_a = *idx & 0xffffffff;
1087        int idx_b = *idx >> 32;
1088
1089        if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1090                nid = NUMA_NO_NODE;
1091
1092        if (*idx == (u64)ULLONG_MAX) {
1093                idx_a = type_a->cnt - 1;
1094                if (type_b != NULL)
1095                        idx_b = type_b->cnt;
1096                else
1097                        idx_b = 0;
1098        }
1099
1100        for (; idx_a >= 0; idx_a--) {
1101                struct memblock_region *m = &type_a->regions[idx_a];
1102
1103                phys_addr_t m_start = m->base;
1104                phys_addr_t m_end = m->base + m->size;
1105                int m_nid = memblock_get_region_node(m);
1106
1107                if (should_skip_region(type_a, m, nid, flags))
1108                        continue;
1109
1110                if (!type_b) {
1111                        if (out_start)
1112                                *out_start = m_start;
1113                        if (out_end)
1114                                *out_end = m_end;
1115                        if (out_nid)
1116                                *out_nid = m_nid;
1117                        idx_a--;
1118                        *idx = (u32)idx_a | (u64)idx_b << 32;
1119                        return;
1120                }
1121
1122                /* scan areas before each reservation */
1123                for (; idx_b >= 0; idx_b--) {
1124                        struct memblock_region *r;
1125                        phys_addr_t r_start;
1126                        phys_addr_t r_end;
1127
1128                        r = &type_b->regions[idx_b];
1129                        r_start = idx_b ? r[-1].base + r[-1].size : 0;
1130                        r_end = idx_b < type_b->cnt ?
1131                                r->base : PHYS_ADDR_MAX;
1132                        /*
1133                         * if idx_b advanced past idx_a,
1134                         * break out to advance idx_a
1135                         */
1136
1137                        if (r_end <= m_start)
1138                                break;
1139                        /* if the two regions intersect, we're done */
1140                        if (m_end > r_start) {
1141                                if (out_start)
1142                                        *out_start = max(m_start, r_start);
1143                                if (out_end)
1144                                        *out_end = min(m_end, r_end);
1145                                if (out_nid)
1146                                        *out_nid = m_nid;
1147                                if (m_start >= r_start)
1148                                        idx_a--;
1149                                else
1150                                        idx_b--;
1151                                *idx = (u32)idx_a | (u64)idx_b << 32;
1152                                return;
1153                        }
1154                }
1155        }
1156        /* signal end of iteration */
1157        *idx = ULLONG_MAX;
1158}
1159
1160/*
1161 * Common iterator interface used to define for_each_mem_pfn_range().
1162 */
1163void __init_memblock __next_mem_pfn_range(int *idx, int nid,
1164                                unsigned long *out_start_pfn,
1165                                unsigned long *out_end_pfn, int *out_nid)
1166{
1167        struct memblock_type *type = &memblock.memory;
1168        struct memblock_region *r;
1169        int r_nid;
1170
1171        while (++*idx < type->cnt) {
1172                r = &type->regions[*idx];
1173                r_nid = memblock_get_region_node(r);
1174
1175                if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
1176                        continue;
1177                if (nid == MAX_NUMNODES || nid == r_nid)
1178                        break;
1179        }
1180        if (*idx >= type->cnt) {
1181                *idx = -1;
1182                return;
1183        }
1184
1185        if (out_start_pfn)
1186                *out_start_pfn = PFN_UP(r->base);
1187        if (out_end_pfn)
1188                *out_end_pfn = PFN_DOWN(r->base + r->size);
1189        if (out_nid)
1190                *out_nid = r_nid;
1191}
1192
1193/**
1194 * memblock_set_node - set node ID on memblock regions
1195 * @base: base of area to set node ID for
1196 * @size: size of area to set node ID for
1197 * @type: memblock type to set node ID for
1198 * @nid: node ID to set
1199 *
1200 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
1201 * Regions which cross the area boundaries are split as necessary.
1202 *
1203 * Return:
1204 * 0 on success, -errno on failure.
1205 */
1206int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
1207                                      struct memblock_type *type, int nid)
1208{
1209#ifdef CONFIG_NEED_MULTIPLE_NODES
1210        int start_rgn, end_rgn;
1211        int i, ret;
1212
1213        ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
1214        if (ret)
1215                return ret;
1216
1217        for (i = start_rgn; i < end_rgn; i++)
1218                memblock_set_region_node(&type->regions[i], nid);
1219
1220        memblock_merge_regions(type);
1221#endif
1222        return 0;
1223}
1224
1225#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1226/**
1227 * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
1228 *
1229 * @idx: pointer to u64 loop variable
1230 * @zone: zone in which all of the memory blocks reside
1231 * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
1232 * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
1233 *
1234 * This function is meant to be a zone/pfn specific wrapper for the
1235 * for_each_mem_range type iterators. Specifically they are used in the
1236 * deferred memory init routines and as such we were duplicating much of
1237 * this logic throughout the code. So instead of having it in multiple
1238 * locations it seemed like it would make more sense to centralize this to
1239 * one new iterator that does everything they need.
1240 */
1241void __init_memblock
1242__next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
1243                             unsigned long *out_spfn, unsigned long *out_epfn)
1244{
1245        int zone_nid = zone_to_nid(zone);
1246        phys_addr_t spa, epa;
1247        int nid;
1248
1249        __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1250                         &memblock.memory, &memblock.reserved,
1251                         &spa, &epa, &nid);
1252
1253        while (*idx != U64_MAX) {
1254                unsigned long epfn = PFN_DOWN(epa);
1255                unsigned long spfn = PFN_UP(spa);
1256
1257                /*
1258                 * Verify the end is at least past the start of the zone and
1259                 * that we have at least one PFN to initialize.
1260                 */
1261                if (zone->zone_start_pfn < epfn && spfn < epfn) {
1262                        /* if we went too far just stop searching */
1263                        if (zone_end_pfn(zone) <= spfn) {
1264                                *idx = U64_MAX;
1265                                break;
1266                        }
1267
1268                        if (out_spfn)
1269                                *out_spfn = max(zone->zone_start_pfn, spfn);
1270                        if (out_epfn)
1271                                *out_epfn = min(zone_end_pfn(zone), epfn);
1272
1273                        return;
1274                }
1275
1276                __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
1277                                 &memblock.memory, &memblock.reserved,
1278                                 &spa, &epa, &nid);
1279        }
1280
1281        /* signal end of iteration */
1282        if (out_spfn)
1283                *out_spfn = ULONG_MAX;
1284        if (out_epfn)
1285                *out_epfn = 0;
1286}
1287
1288#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1289
1290/**
1291 * memblock_alloc_range_nid - allocate boot memory block
1292 * @size: size of memory block to be allocated in bytes
1293 * @align: alignment of the region and block's size
1294 * @start: the lower bound of the memory region to allocate (phys address)
1295 * @end: the upper bound of the memory region to allocate (phys address)
1296 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1297 * @exact_nid: control the allocation fall back to other nodes
1298 *
1299 * The allocation is performed from memory region limited by
1300 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
1301 *
1302 * If the specified node can not hold the requested memory and @exact_nid
1303 * is false, the allocation falls back to any node in the system.
1304 *
1305 * For systems with memory mirroring, the allocation is attempted first
1306 * from the regions with mirroring enabled and then retried from any
1307 * memory region.
1308 *
1309 * In addition, function sets the min_count to 0 using kmemleak_alloc_phys for
1310 * allocated boot memory block, so that it is never reported as leaks.
1311 *
1312 * Return:
1313 * Physical address of allocated memory block on success, %0 on failure.
1314 */
1315phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
1316                                        phys_addr_t align, phys_addr_t start,
1317                                        phys_addr_t end, int nid,
1318                                        bool exact_nid)
1319{
1320        enum memblock_flags flags = choose_memblock_flags();
1321        phys_addr_t found;
1322
1323        if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1324                nid = NUMA_NO_NODE;
1325
1326        if (!align) {
1327                /* Can't use WARNs this early in boot on powerpc */
1328                dump_stack();
1329                align = SMP_CACHE_BYTES;
1330        }
1331
1332again:
1333        found = memblock_find_in_range_node(size, align, start, end, nid,
1334                                            flags);
1335        if (found && !memblock_reserve(found, size))
1336                goto done;
1337
1338        if (nid != NUMA_NO_NODE && !exact_nid) {
1339                found = memblock_find_in_range_node(size, align, start,
1340                                                    end, NUMA_NO_NODE,
1341                                                    flags);
1342                if (found && !memblock_reserve(found, size))
1343                        goto done;
1344        }
1345
1346        if (flags & MEMBLOCK_MIRROR) {
1347                flags &= ~MEMBLOCK_MIRROR;
1348                pr_warn("Could not allocate %pap bytes of mirrored memory\n",
1349                        &size);
1350                goto again;
1351        }
1352
1353        return 0;
1354
1355done:
1356        /* Skip kmemleak for kasan_init() due to high volume. */
1357        if (end != MEMBLOCK_ALLOC_KASAN)
1358                /*
1359                 * The min_count is set to 0 so that memblock allocated
1360                 * blocks are never reported as leaks. This is because many
1361                 * of these blocks are only referred via the physical
1362                 * address which is not looked up by kmemleak.
1363                 */
1364                kmemleak_alloc_phys(found, size, 0, 0);
1365
1366        return found;
1367}
1368
1369/**
1370 * memblock_phys_alloc_range - allocate a memory block inside specified range
1371 * @size: size of memory block to be allocated in bytes
1372 * @align: alignment of the region and block's size
1373 * @start: the lower bound of the memory region to allocate (physical address)
1374 * @end: the upper bound of the memory region to allocate (physical address)
1375 *
1376 * Allocate @size bytes in the between @start and @end.
1377 *
1378 * Return: physical address of the allocated memory block on success,
1379 * %0 on failure.
1380 */
1381phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size,
1382                                             phys_addr_t align,
1383                                             phys_addr_t start,
1384                                             phys_addr_t end)
1385{
1386        memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n",
1387                     __func__, (u64)size, (u64)align, &start, &end,
1388                     (void *)_RET_IP_);
1389        return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
1390                                        false);
1391}
1392
1393/**
1394 * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node
1395 * @size: size of memory block to be allocated in bytes
1396 * @align: alignment of the region and block's size
1397 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1398 *
1399 * Allocates memory block from the specified NUMA node. If the node
1400 * has no available memory, attempts to allocated from any node in the
1401 * system.
1402 *
1403 * Return: physical address of the allocated memory block on success,
1404 * %0 on failure.
1405 */
1406phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
1407{
1408        return memblock_alloc_range_nid(size, align, 0,
1409                                        MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
1410}
1411
1412/**
1413 * memblock_alloc_internal - allocate boot memory block
1414 * @size: size of memory block to be allocated in bytes
1415 * @align: alignment of the region and block's size
1416 * @min_addr: the lower bound of the memory region to allocate (phys address)
1417 * @max_addr: the upper bound of the memory region to allocate (phys address)
1418 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1419 * @exact_nid: control the allocation fall back to other nodes
1420 *
1421 * Allocates memory block using memblock_alloc_range_nid() and
1422 * converts the returned physical address to virtual.
1423 *
1424 * The @min_addr limit is dropped if it can not be satisfied and the allocation
1425 * will fall back to memory below @min_addr. Other constraints, such
1426 * as node and mirrored memory will be handled again in
1427 * memblock_alloc_range_nid().
1428 *
1429 * Return:
1430 * Virtual address of allocated memory block on success, NULL on failure.
1431 */
1432static void * __init memblock_alloc_internal(
1433                                phys_addr_t size, phys_addr_t align,
1434                                phys_addr_t min_addr, phys_addr_t max_addr,
1435                                int nid, bool exact_nid)
1436{
1437        phys_addr_t alloc;
1438
1439        /*
1440         * Detect any accidental use of these APIs after slab is ready, as at
1441         * this moment memblock may be deinitialized already and its
1442         * internal data may be destroyed (after execution of memblock_free_all)
1443         */
1444        if (WARN_ON_ONCE(slab_is_available()))
1445                return kzalloc_node(size, GFP_NOWAIT, nid);
1446
1447        if (max_addr > memblock.current_limit)
1448                max_addr = memblock.current_limit;
1449
1450        alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
1451                                        exact_nid);
1452
1453        /* retry allocation without lower limit */
1454        if (!alloc && min_addr)
1455                alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
1456                                                exact_nid);
1457
1458        if (!alloc)
1459                return NULL;
1460
1461        return phys_to_virt(alloc);
1462}
1463
1464/**
1465 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
1466 * without zeroing memory
1467 * @size: size of memory block to be allocated in bytes
1468 * @align: alignment of the region and block's size
1469 * @min_addr: the lower bound of the memory region from where the allocation
1470 *        is preferred (phys address)
1471 * @max_addr: the upper bound of the memory region from where the allocation
1472 *            is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1473 *            allocate only from memory limited by memblock.current_limit value
1474 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1475 *
1476 * Public function, provides additional debug information (including caller
1477 * info), if enabled. Does not zero allocated memory.
1478 *
1479 * Return:
1480 * Virtual address of allocated memory block on success, NULL on failure.
1481 */
1482void * __init memblock_alloc_exact_nid_raw(
1483                        phys_addr_t size, phys_addr_t align,
1484                        phys_addr_t min_addr, phys_addr_t max_addr,
1485                        int nid)
1486{
1487        void *ptr;
1488
1489        memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1490                     __func__, (u64)size, (u64)align, nid, &min_addr,
1491                     &max_addr, (void *)_RET_IP_);
1492
1493        ptr = memblock_alloc_internal(size, align,
1494                                           min_addr, max_addr, nid, true);
1495        if (ptr && size > 0)
1496                page_init_poison(ptr, size);
1497
1498        return ptr;
1499}
1500
1501/**
1502 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
1503 * memory and without panicking
1504 * @size: size of memory block to be allocated in bytes
1505 * @align: alignment of the region and block's size
1506 * @min_addr: the lower bound of the memory region from where the allocation
1507 *        is preferred (phys address)
1508 * @max_addr: the upper bound of the memory region from where the allocation
1509 *            is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1510 *            allocate only from memory limited by memblock.current_limit value
1511 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1512 *
1513 * Public function, provides additional debug information (including caller
1514 * info), if enabled. Does not zero allocated memory, does not panic if request
1515 * cannot be satisfied.
1516 *
1517 * Return:
1518 * Virtual address of allocated memory block on success, NULL on failure.
1519 */
1520void * __init memblock_alloc_try_nid_raw(
1521                        phys_addr_t size, phys_addr_t align,
1522                        phys_addr_t min_addr, phys_addr_t max_addr,
1523                        int nid)
1524{
1525        void *ptr;
1526
1527        memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1528                     __func__, (u64)size, (u64)align, nid, &min_addr,
1529                     &max_addr, (void *)_RET_IP_);
1530
1531        ptr = memblock_alloc_internal(size, align,
1532                                           min_addr, max_addr, nid, false);
1533        if (ptr && size > 0)
1534                page_init_poison(ptr, size);
1535
1536        return ptr;
1537}
1538
1539/**
1540 * memblock_alloc_try_nid - allocate boot memory block
1541 * @size: size of memory block to be allocated in bytes
1542 * @align: alignment of the region and block's size
1543 * @min_addr: the lower bound of the memory region from where the allocation
1544 *        is preferred (phys address)
1545 * @max_addr: the upper bound of the memory region from where the allocation
1546 *            is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1547 *            allocate only from memory limited by memblock.current_limit value
1548 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1549 *
1550 * Public function, provides additional debug information (including caller
1551 * info), if enabled. This function zeroes the allocated memory.
1552 *
1553 * Return:
1554 * Virtual address of allocated memory block on success, NULL on failure.
1555 */
1556void * __init memblock_alloc_try_nid(
1557                        phys_addr_t size, phys_addr_t align,
1558                        phys_addr_t min_addr, phys_addr_t max_addr,
1559                        int nid)
1560{
1561        void *ptr;
1562
1563        memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1564                     __func__, (u64)size, (u64)align, nid, &min_addr,
1565                     &max_addr, (void *)_RET_IP_);
1566        ptr = memblock_alloc_internal(size, align,
1567                                           min_addr, max_addr, nid, false);
1568        if (ptr)
1569                memset(ptr, 0, size);
1570
1571        return ptr;
1572}
1573
1574/**
1575 * __memblock_free_late - free pages directly to buddy allocator
1576 * @base: phys starting address of the  boot memory block
1577 * @size: size of the boot memory block in bytes
1578 *
1579 * This is only useful when the memblock allocator has already been torn
1580 * down, but we are still initializing the system.  Pages are released directly
1581 * to the buddy allocator.
1582 */
1583void __init __memblock_free_late(phys_addr_t base, phys_addr_t size)
1584{
1585        phys_addr_t cursor, end;
1586
1587        end = base + size - 1;
1588        memblock_dbg("%s: [%pa-%pa] %pS\n",
1589                     __func__, &base, &end, (void *)_RET_IP_);
1590        kmemleak_free_part_phys(base, size);
1591        cursor = PFN_UP(base);
1592        end = PFN_DOWN(base + size);
1593
1594        for (; cursor < end; cursor++) {
1595                memblock_free_pages(pfn_to_page(cursor), cursor, 0);
1596                totalram_pages_inc();
1597        }
1598}
1599
1600/*
1601 * Remaining API functions
1602 */
1603
1604phys_addr_t __init_memblock memblock_phys_mem_size(void)
1605{
1606        return memblock.memory.total_size;
1607}
1608
1609phys_addr_t __init_memblock memblock_reserved_size(void)
1610{
1611        return memblock.reserved.total_size;
1612}
1613
1614/* lowest address */
1615phys_addr_t __init_memblock memblock_start_of_DRAM(void)
1616{
1617        return memblock.memory.regions[0].base;
1618}
1619
1620phys_addr_t __init_memblock memblock_end_of_DRAM(void)
1621{
1622        int idx = memblock.memory.cnt - 1;
1623
1624        return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
1625}
1626
1627static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
1628{
1629        phys_addr_t max_addr = PHYS_ADDR_MAX;
1630        struct memblock_region *r;
1631
1632        /*
1633         * translate the memory @limit size into the max address within one of
1634         * the memory memblock regions, if the @limit exceeds the total size
1635         * of those regions, max_addr will keep original value PHYS_ADDR_MAX
1636         */
1637        for_each_mem_region(r) {
1638                if (limit <= r->size) {
1639                        max_addr = r->base + limit;
1640                        break;
1641                }
1642                limit -= r->size;
1643        }
1644
1645        return max_addr;
1646}
1647
1648void __init memblock_enforce_memory_limit(phys_addr_t limit)
1649{
1650        phys_addr_t max_addr;
1651
1652        if (!limit)
1653                return;
1654
1655        max_addr = __find_max_addr(limit);
1656
1657        /* @limit exceeds the total size of the memory, do nothing */
1658        if (max_addr == PHYS_ADDR_MAX)
1659                return;
1660
1661        /* truncate both memory and reserved regions */
1662        memblock_remove_range(&memblock.memory, max_addr,
1663                              PHYS_ADDR_MAX);
1664        memblock_remove_range(&memblock.reserved, max_addr,
1665                              PHYS_ADDR_MAX);
1666}
1667
1668void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
1669{
1670        int start_rgn, end_rgn;
1671        int i, ret;
1672
1673        if (!size)
1674                return;
1675
1676        ret = memblock_isolate_range(&memblock.memory, base, size,
1677                                                &start_rgn, &end_rgn);
1678        if (ret)
1679                return;
1680
1681        /* remove all the MAP regions */
1682        for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
1683                if (!memblock_is_nomap(&memblock.memory.regions[i]))
1684                        memblock_remove_region(&memblock.memory, i);
1685
1686        for (i = start_rgn - 1; i >= 0; i--)
1687                if (!memblock_is_nomap(&memblock.memory.regions[i]))
1688                        memblock_remove_region(&memblock.memory, i);
1689
1690        /* truncate the reserved regions */
1691        memblock_remove_range(&memblock.reserved, 0, base);
1692        memblock_remove_range(&memblock.reserved,
1693                        base + size, PHYS_ADDR_MAX);
1694}
1695
1696void __init memblock_mem_limit_remove_map(phys_addr_t limit)
1697{
1698        phys_addr_t max_addr;
1699
1700        if (!limit)
1701                return;
1702
1703        max_addr = __find_max_addr(limit);
1704
1705        /* @limit exceeds the total size of the memory, do nothing */
1706        if (max_addr == PHYS_ADDR_MAX)
1707                return;
1708
1709        memblock_cap_memory_range(0, max_addr);
1710}
1711
1712static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
1713{
1714        unsigned int left = 0, right = type->cnt;
1715
1716        do {
1717                unsigned int mid = (right + left) / 2;
1718
1719                if (addr < type->regions[mid].base)
1720                        right = mid;
1721                else if (addr >= (type->regions[mid].base +
1722                                  type->regions[mid].size))
1723                        left = mid + 1;
1724                else
1725                        return mid;
1726        } while (left < right);
1727        return -1;
1728}
1729
1730bool __init_memblock memblock_is_reserved(phys_addr_t addr)
1731{
1732        return memblock_search(&memblock.reserved, addr) != -1;
1733}
1734
1735bool __init_memblock memblock_is_memory(phys_addr_t addr)
1736{
1737        return memblock_search(&memblock.memory, addr) != -1;
1738}
1739
1740bool __init_memblock memblock_is_map_memory(phys_addr_t addr)
1741{
1742        int i = memblock_search(&memblock.memory, addr);
1743
1744        if (i == -1)
1745                return false;
1746        return !memblock_is_nomap(&memblock.memory.regions[i]);
1747}
1748
1749int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
1750                         unsigned long *start_pfn, unsigned long *end_pfn)
1751{
1752        struct memblock_type *type = &memblock.memory;
1753        int mid = memblock_search(type, PFN_PHYS(pfn));
1754
1755        if (mid == -1)
1756                return -1;
1757
1758        *start_pfn = PFN_DOWN(type->regions[mid].base);
1759        *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
1760
1761        return memblock_get_region_node(&type->regions[mid]);
1762}
1763
1764/**
1765 * memblock_is_region_memory - check if a region is a subset of memory
1766 * @base: base of region to check
1767 * @size: size of region to check
1768 *
1769 * Check if the region [@base, @base + @size) is a subset of a memory block.
1770 *
1771 * Return:
1772 * 0 if false, non-zero if true
1773 */
1774bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
1775{
1776        int idx = memblock_search(&memblock.memory, base);
1777        phys_addr_t end = base + memblock_cap_size(base, &size);
1778
1779        if (idx == -1)
1780                return false;
1781        return (memblock.memory.regions[idx].base +
1782                 memblock.memory.regions[idx].size) >= end;
1783}
1784
1785/**
1786 * memblock_is_region_reserved - check if a region intersects reserved memory
1787 * @base: base of region to check
1788 * @size: size of region to check
1789 *
1790 * Check if the region [@base, @base + @size) intersects a reserved
1791 * memory block.
1792 *
1793 * Return:
1794 * True if they intersect, false if not.
1795 */
1796bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
1797{
1798        memblock_cap_size(base, &size);
1799        return memblock_overlaps_region(&memblock.reserved, base, size);
1800}
1801
1802void __init_memblock memblock_trim_memory(phys_addr_t align)
1803{
1804        phys_addr_t start, end, orig_start, orig_end;
1805        struct memblock_region *r;
1806
1807        for_each_mem_region(r) {
1808                orig_start = r->base;
1809                orig_end = r->base + r->size;
1810                start = round_up(orig_start, align);
1811                end = round_down(orig_end, align);
1812
1813                if (start == orig_start && end == orig_end)
1814                        continue;
1815
1816                if (start < end) {
1817                        r->base = start;
1818                        r->size = end - start;
1819                } else {
1820                        memblock_remove_region(&memblock.memory,
1821                                               r - memblock.memory.regions);
1822                        r--;
1823                }
1824        }
1825}
1826
1827void __init_memblock memblock_set_current_limit(phys_addr_t limit)
1828{
1829        memblock.current_limit = limit;
1830}
1831
1832phys_addr_t __init_memblock memblock_get_current_limit(void)
1833{
1834        return memblock.current_limit;
1835}
1836
1837static void __init_memblock memblock_dump(struct memblock_type *type)
1838{
1839        phys_addr_t base, end, size;
1840        enum memblock_flags flags;
1841        int idx;
1842        struct memblock_region *rgn;
1843
1844        pr_info(" %s.cnt  = 0x%lx\n", type->name, type->cnt);
1845
1846        for_each_memblock_type(idx, type, rgn) {
1847                char nid_buf[32] = "";
1848
1849                base = rgn->base;
1850                size = rgn->size;
1851                end = base + size - 1;
1852                flags = rgn->flags;
1853#ifdef CONFIG_NEED_MULTIPLE_NODES
1854                if (memblock_get_region_node(rgn) != MAX_NUMNODES)
1855                        snprintf(nid_buf, sizeof(nid_buf), " on node %d",
1856                                 memblock_get_region_node(rgn));
1857#endif
1858                pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
1859                        type->name, idx, &base, &end, &size, nid_buf, flags);
1860        }
1861}
1862
1863static void __init_memblock __memblock_dump_all(void)
1864{
1865        pr_info("MEMBLOCK configuration:\n");
1866        pr_info(" memory size = %pa reserved size = %pa\n",
1867                &memblock.memory.total_size,
1868                &memblock.reserved.total_size);
1869
1870        memblock_dump(&memblock.memory);
1871        memblock_dump(&memblock.reserved);
1872#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
1873        memblock_dump(&physmem);
1874#endif
1875}
1876
1877void __init_memblock memblock_dump_all(void)
1878{
1879        if (memblock_debug)
1880                __memblock_dump_all();
1881}
1882
1883void __init memblock_allow_resize(void)
1884{
1885        memblock_can_resize = 1;
1886}
1887
1888static int __init early_memblock(char *p)
1889{
1890        if (p && strstr(p, "debug"))
1891                memblock_debug = 1;
1892        return 0;
1893}
1894early_param("memblock", early_memblock);
1895
1896static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn)
1897{
1898        struct page *start_pg, *end_pg;
1899        phys_addr_t pg, pgend;
1900
1901        /*
1902         * Convert start_pfn/end_pfn to a struct page pointer.
1903         */
1904        start_pg = pfn_to_page(start_pfn - 1) + 1;
1905        end_pg = pfn_to_page(end_pfn - 1) + 1;
1906
1907        /*
1908         * Convert to physical addresses, and round start upwards and end
1909         * downwards.
1910         */
1911        pg = PAGE_ALIGN(__pa(start_pg));
1912        pgend = __pa(end_pg) & PAGE_MASK;
1913
1914        /*
1915         * If there are free pages between these, free the section of the
1916         * memmap array.
1917         */
1918        if (pg < pgend)
1919                memblock_free(pg, pgend - pg);
1920}
1921
1922/*
1923 * The mem_map array can get very big.  Free the unused area of the memory map.
1924 */
1925static void __init free_unused_memmap(void)
1926{
1927        unsigned long start, end, prev_end = 0;
1928        int i;
1929
1930        if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) ||
1931            IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))
1932                return;
1933
1934        /*
1935         * This relies on each bank being in address order.
1936         * The banks are sorted previously in bootmem_init().
1937         */
1938        for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) {
1939#ifdef CONFIG_SPARSEMEM
1940                /*
1941                 * Take care not to free memmap entries that don't exist
1942                 * due to SPARSEMEM sections which aren't present.
1943                 */
1944                start = min(start, ALIGN(prev_end, PAGES_PER_SECTION));
1945#else
1946                /*
1947                 * Align down here since the VM subsystem insists that the
1948                 * memmap entries are valid from the bank start aligned to
1949                 * MAX_ORDER_NR_PAGES.
1950                 */
1951                start = round_down(start, MAX_ORDER_NR_PAGES);
1952#endif
1953
1954                /*
1955                 * If we had a previous bank, and there is a space
1956                 * between the current bank and the previous, free it.
1957                 */
1958                if (prev_end && prev_end < start)
1959                        free_memmap(prev_end, start);
1960
1961                /*
1962                 * Align up here since the VM subsystem insists that the
1963                 * memmap entries are valid from the bank end aligned to
1964                 * MAX_ORDER_NR_PAGES.
1965                 */
1966                prev_end = ALIGN(end, MAX_ORDER_NR_PAGES);
1967        }
1968
1969#ifdef CONFIG_SPARSEMEM
1970        if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION))
1971                free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
1972#endif
1973}
1974
1975static void __init __free_pages_memory(unsigned long start, unsigned long end)
1976{
1977        int order;
1978
1979        while (start < end) {
1980                order = min(MAX_ORDER - 1UL, __ffs(start));
1981
1982                while (start + (1UL << order) > end)
1983                        order--;
1984
1985                memblock_free_pages(pfn_to_page(start), start, order);
1986
1987                start += (1UL << order);
1988        }
1989}
1990
1991static unsigned long __init __free_memory_core(phys_addr_t start,
1992                                 phys_addr_t end)
1993{
1994        unsigned long start_pfn = PFN_UP(start);
1995        unsigned long end_pfn = min_t(unsigned long,
1996                                      PFN_DOWN(end), max_low_pfn);
1997
1998        if (start_pfn >= end_pfn)
1999                return 0;
2000
2001        __free_pages_memory(start_pfn, end_pfn);
2002
2003        return end_pfn - start_pfn;
2004}
2005
2006static unsigned long __init free_low_memory_core_early(void)
2007{
2008        unsigned long count = 0;
2009        phys_addr_t start, end;
2010        u64 i;
2011
2012        memblock_clear_hotplug(0, -1);
2013
2014        for_each_reserved_mem_range(i, &start, &end)
2015                reserve_bootmem_region(start, end);
2016
2017        /*
2018         * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
2019         *  because in some case like Node0 doesn't have RAM installed
2020         *  low ram will be on Node1
2021         */
2022        for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
2023                                NULL)
2024                count += __free_memory_core(start, end);
2025
2026        return count;
2027}
2028
2029static int reset_managed_pages_done __initdata;
2030
2031void reset_node_managed_pages(pg_data_t *pgdat)
2032{
2033        struct zone *z;
2034
2035        for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
2036                atomic_long_set(&z->managed_pages, 0);
2037}
2038
2039void __init reset_all_zones_managed_pages(void)
2040{
2041        struct pglist_data *pgdat;
2042
2043        if (reset_managed_pages_done)
2044                return;
2045
2046        for_each_online_pgdat(pgdat)
2047                reset_node_managed_pages(pgdat);
2048
2049        reset_managed_pages_done = 1;
2050}
2051
2052/**
2053 * memblock_free_all - release free pages to the buddy allocator
2054 */
2055void __init memblock_free_all(void)
2056{
2057        unsigned long pages;
2058
2059        free_unused_memmap();
2060        reset_all_zones_managed_pages();
2061
2062        pages = free_low_memory_core_early();
2063        totalram_pages_add(pages);
2064}
2065
2066#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
2067
2068static int memblock_debug_show(struct seq_file *m, void *private)
2069{
2070        struct memblock_type *type = m->private;
2071        struct memblock_region *reg;
2072        int i;
2073        phys_addr_t end;
2074
2075        for (i = 0; i < type->cnt; i++) {
2076                reg = &type->regions[i];
2077                end = reg->base + reg->size - 1;
2078
2079                seq_printf(m, "%4d: ", i);
2080                seq_printf(m, "%pa..%pa\n", &reg->base, &end);
2081        }
2082        return 0;
2083}
2084DEFINE_SHOW_ATTRIBUTE(memblock_debug);
2085
2086static int __init memblock_init_debugfs(void)
2087{
2088        struct dentry *root = debugfs_create_dir("memblock", NULL);
2089
2090        debugfs_create_file("memory", 0444, root,
2091                            &memblock.memory, &memblock_debug_fops);
2092        debugfs_create_file("reserved", 0444, root,
2093                            &memblock.reserved, &memblock_debug_fops);
2094#ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
2095        debugfs_create_file("physmem", 0444, root, &physmem,
2096                            &memblock_debug_fops);
2097#endif
2098
2099        return 0;
2100}
2101__initcall(memblock_init_debugfs);
2102
2103#endif /* CONFIG_DEBUG_FS */
2104