linux/mm/nobootmem.c
<<
>>
Prefs
   1/*
   2 *  bootmem - A boot-time physical memory allocator and configurator
   3 *
   4 *  Copyright (C) 1999 Ingo Molnar
   5 *                1999 Kanoj Sarcar, SGI
   6 *                2008 Johannes Weiner
   7 *
   8 * Access to this subsystem has to be serialized externally (which is true
   9 * for the boot process anyway).
  10 */
  11#include <linux/init.h>
  12#include <linux/pfn.h>
  13#include <linux/slab.h>
  14#include <linux/bootmem.h>
  15#include <linux/export.h>
  16#include <linux/kmemleak.h>
  17#include <linux/range.h>
  18#include <linux/memblock.h>
  19
  20#include <asm/bug.h>
  21#include <asm/io.h>
  22#include <asm/processor.h>
  23
  24#include "internal.h"
  25
  26#ifndef CONFIG_NEED_MULTIPLE_NODES
  27struct pglist_data __refdata contig_page_data;
  28EXPORT_SYMBOL(contig_page_data);
  29#endif
  30
  31unsigned long max_low_pfn;
  32unsigned long min_low_pfn;
  33unsigned long max_pfn;
  34
  35static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
  36                                        u64 goal, u64 limit)
  37{
  38        void *ptr;
  39        u64 addr;
  40
  41        if (limit > memblock.current_limit)
  42                limit = memblock.current_limit;
  43
  44        addr = memblock_find_in_range_node(goal, limit, size, align, nid);
  45        if (!addr)
  46                return NULL;
  47
  48        ptr = phys_to_virt(addr);
  49        memset(ptr, 0, size);
  50        memblock_reserve(addr, size);
  51        /*
  52         * The min_count is set to 0 so that bootmem allocated blocks
  53         * are never reported as leaks.
  54         */
  55        kmemleak_alloc(ptr, size, 0, 0);
  56        return ptr;
  57}
  58
  59/*
  60 * free_bootmem_late - free bootmem pages directly to page allocator
  61 * @addr: starting address of the range
  62 * @size: size of the range in bytes
  63 *
  64 * This is only useful when the bootmem allocator has already been torn
  65 * down, but we are still initializing the system.  Pages are given directly
  66 * to the page allocator, no bootmem metadata is updated because it is gone.
  67 */
  68void __init free_bootmem_late(unsigned long addr, unsigned long size)
  69{
  70        unsigned long cursor, end;
  71
  72        kmemleak_free_part(__va(addr), size);
  73
  74        cursor = PFN_UP(addr);
  75        end = PFN_DOWN(addr + size);
  76
  77        for (; cursor < end; cursor++) {
  78                __free_pages_bootmem(pfn_to_page(cursor), 0);
  79                totalram_pages++;
  80        }
  81}
  82
  83static void __init __free_pages_memory(unsigned long start, unsigned long end)
  84{
  85        unsigned long i, start_aligned, end_aligned;
  86        int order = ilog2(BITS_PER_LONG);
  87
  88        start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
  89        end_aligned = end & ~(BITS_PER_LONG - 1);
  90
  91        if (end_aligned <= start_aligned) {
  92                for (i = start; i < end; i++)
  93                        __free_pages_bootmem(pfn_to_page(i), 0);
  94
  95                return;
  96        }
  97
  98        for (i = start; i < start_aligned; i++)
  99                __free_pages_bootmem(pfn_to_page(i), 0);
 100
 101        for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
 102                __free_pages_bootmem(pfn_to_page(i), order);
 103
 104        for (i = end_aligned; i < end; i++)
 105                __free_pages_bootmem(pfn_to_page(i), 0);
 106}
 107
 108static unsigned long __init __free_memory_core(phys_addr_t start,
 109                                 phys_addr_t end)
 110{
 111        unsigned long start_pfn = PFN_UP(start);
 112        unsigned long end_pfn = min_t(unsigned long,
 113                                      PFN_DOWN(end), max_low_pfn);
 114
 115        if (start_pfn > end_pfn)
 116                return 0;
 117
 118        __free_pages_memory(start_pfn, end_pfn);
 119
 120        return end_pfn - start_pfn;
 121}
 122
 123unsigned long __init free_low_memory_core_early(int nodeid)
 124{
 125        unsigned long count = 0;
 126        phys_addr_t start, end, size;
 127        u64 i;
 128
 129        for_each_free_mem_range(i, MAX_NUMNODES, &start, &end, NULL)
 130                count += __free_memory_core(start, end);
 131
 132        /* free range that is used for reserved array if we allocate it */
 133        size = get_allocated_memblock_reserved_regions_info(&start);
 134        if (size)
 135                count += __free_memory_core(start, start + size);
 136
 137        return count;
 138}
 139
 140static void reset_node_lowmem_managed_pages(pg_data_t *pgdat)
 141{
 142        struct zone *z;
 143
 144        /*
 145         * In free_area_init_core(), highmem zone's managed_pages is set to
 146         * present_pages, and bootmem allocator doesn't allocate from highmem
 147         * zones. So there's no need to recalculate managed_pages because all
 148         * highmem pages will be managed by the buddy system. Here highmem
 149         * zone also includes highmem movable zone.
 150         */
 151        for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
 152                if (!is_highmem(z))
 153                        z->managed_pages = 0;
 154}
 155
 156/**
 157 * free_all_bootmem - release free pages to the buddy allocator
 158 *
 159 * Returns the number of pages actually released.
 160 */
 161unsigned long __init free_all_bootmem(void)
 162{
 163        struct pglist_data *pgdat;
 164
 165        for_each_online_pgdat(pgdat)
 166                reset_node_lowmem_managed_pages(pgdat);
 167
 168        /*
 169         * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
 170         *  because in some case like Node0 doesn't have RAM installed
 171         *  low ram will be on Node1
 172         */
 173        return free_low_memory_core_early(MAX_NUMNODES);
 174}
 175
 176/**
 177 * free_bootmem_node - mark a page range as usable
 178 * @pgdat: node the range resides on
 179 * @physaddr: starting address of the range
 180 * @size: size of the range in bytes
 181 *
 182 * Partial pages will be considered reserved and left as they are.
 183 *
 184 * The range must reside completely on the specified node.
 185 */
 186void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 187                              unsigned long size)
 188{
 189        kmemleak_free_part(__va(physaddr), size);
 190        memblock_free(physaddr, size);
 191}
 192
 193/**
 194 * free_bootmem - mark a page range as usable
 195 * @addr: starting address of the range
 196 * @size: size of the range in bytes
 197 *
 198 * Partial pages will be considered reserved and left as they are.
 199 *
 200 * The range must be contiguous but may span node boundaries.
 201 */
 202void __init free_bootmem(unsigned long addr, unsigned long size)
 203{
 204        kmemleak_free_part(__va(addr), size);
 205        memblock_free(addr, size);
 206}
 207
 208static void * __init ___alloc_bootmem_nopanic(unsigned long size,
 209                                        unsigned long align,
 210                                        unsigned long goal,
 211                                        unsigned long limit)
 212{
 213        void *ptr;
 214
 215        if (WARN_ON_ONCE(slab_is_available()))
 216                return kzalloc(size, GFP_NOWAIT);
 217
 218restart:
 219
 220        ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
 221
 222        if (ptr)
 223                return ptr;
 224
 225        if (goal != 0) {
 226                goal = 0;
 227                goto restart;
 228        }
 229
 230        return NULL;
 231}
 232
 233/**
 234 * __alloc_bootmem_nopanic - allocate boot memory without panicking
 235 * @size: size of the request in bytes
 236 * @align: alignment of the region
 237 * @goal: preferred starting address of the region
 238 *
 239 * The goal is dropped if it can not be satisfied and the allocation will
 240 * fall back to memory below @goal.
 241 *
 242 * Allocation may happen on any node in the system.
 243 *
 244 * Returns NULL on failure.
 245 */
 246void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
 247                                        unsigned long goal)
 248{
 249        unsigned long limit = -1UL;
 250
 251        return ___alloc_bootmem_nopanic(size, align, goal, limit);
 252}
 253
 254static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
 255                                        unsigned long goal, unsigned long limit)
 256{
 257        void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
 258
 259        if (mem)
 260                return mem;
 261        /*
 262         * Whoops, we cannot satisfy the allocation request.
 263         */
 264        printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
 265        panic("Out of memory");
 266        return NULL;
 267}
 268
 269/**
 270 * __alloc_bootmem - allocate boot memory
 271 * @size: size of the request in bytes
 272 * @align: alignment of the region
 273 * @goal: preferred starting address of the region
 274 *
 275 * The goal is dropped if it can not be satisfied and the allocation will
 276 * fall back to memory below @goal.
 277 *
 278 * Allocation may happen on any node in the system.
 279 *
 280 * The function panics if the request can not be satisfied.
 281 */
 282void * __init __alloc_bootmem(unsigned long size, unsigned long align,
 283                              unsigned long goal)
 284{
 285        unsigned long limit = -1UL;
 286
 287        return ___alloc_bootmem(size, align, goal, limit);
 288}
 289
 290void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat,
 291                                                   unsigned long size,
 292                                                   unsigned long align,
 293                                                   unsigned long goal,
 294                                                   unsigned long limit)
 295{
 296        void *ptr;
 297
 298again:
 299        ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
 300                                        goal, limit);
 301        if (ptr)
 302                return ptr;
 303
 304        ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
 305                                        goal, limit);
 306        if (ptr)
 307                return ptr;
 308
 309        if (goal) {
 310                goal = 0;
 311                goto again;
 312        }
 313
 314        return NULL;
 315}
 316
 317void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
 318                                   unsigned long align, unsigned long goal)
 319{
 320        if (WARN_ON_ONCE(slab_is_available()))
 321                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 322
 323        return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0);
 324}
 325
 326void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
 327                                    unsigned long align, unsigned long goal,
 328                                    unsigned long limit)
 329{
 330        void *ptr;
 331
 332        ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, limit);
 333        if (ptr)
 334                return ptr;
 335
 336        printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
 337        panic("Out of memory");
 338        return NULL;
 339}
 340
 341/**
 342 * __alloc_bootmem_node - allocate boot memory from a specific node
 343 * @pgdat: node to allocate from
 344 * @size: size of the request in bytes
 345 * @align: alignment of the region
 346 * @goal: preferred starting address of the region
 347 *
 348 * The goal is dropped if it can not be satisfied and the allocation will
 349 * fall back to memory below @goal.
 350 *
 351 * Allocation may fall back to any node in the system if the specified node
 352 * can not hold the requested memory.
 353 *
 354 * The function panics if the request can not be satisfied.
 355 */
 356void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
 357                                   unsigned long align, unsigned long goal)
 358{
 359        if (WARN_ON_ONCE(slab_is_available()))
 360                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 361
 362        return ___alloc_bootmem_node(pgdat, size, align, goal, 0);
 363}
 364
 365void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
 366                                   unsigned long align, unsigned long goal)
 367{
 368        return __alloc_bootmem_node(pgdat, size, align, goal);
 369}
 370
 371#ifndef ARCH_LOW_ADDRESS_LIMIT
 372#define ARCH_LOW_ADDRESS_LIMIT  0xffffffffUL
 373#endif
 374
 375/**
 376 * __alloc_bootmem_low - allocate low boot memory
 377 * @size: size of the request in bytes
 378 * @align: alignment of the region
 379 * @goal: preferred starting address of the region
 380 *
 381 * The goal is dropped if it can not be satisfied and the allocation will
 382 * fall back to memory below @goal.
 383 *
 384 * Allocation may happen on any node in the system.
 385 *
 386 * The function panics if the request can not be satisfied.
 387 */
 388void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
 389                                  unsigned long goal)
 390{
 391        return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
 392}
 393
 394void * __init __alloc_bootmem_low_nopanic(unsigned long size,
 395                                          unsigned long align,
 396                                          unsigned long goal)
 397{
 398        return ___alloc_bootmem_nopanic(size, align, goal,
 399                                        ARCH_LOW_ADDRESS_LIMIT);
 400}
 401
 402/**
 403 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
 404 * @pgdat: node to allocate from
 405 * @size: size of the request in bytes
 406 * @align: alignment of the region
 407 * @goal: preferred starting address of the region
 408 *
 409 * The goal is dropped if it can not be satisfied and the allocation will
 410 * fall back to memory below @goal.
 411 *
 412 * Allocation may fall back to any node in the system if the specified node
 413 * can not hold the requested memory.
 414 *
 415 * The function panics if the request can not be satisfied.
 416 */
 417void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
 418                                       unsigned long align, unsigned long goal)
 419{
 420        if (WARN_ON_ONCE(slab_is_available()))
 421                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 422
 423        return ___alloc_bootmem_node(pgdat, size, align, goal,
 424                                     ARCH_LOW_ADDRESS_LIMIT);
 425}
 426
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.