linux/mm/nobootmem.c
<<
>>
Prefs
   1/*
   2 *  bootmem - A boot-time physical memory allocator and configurator
   3 *
   4 *  Copyright (C) 1999 Ingo Molnar
   5 *                1999 Kanoj Sarcar, SGI
   6 *                2008 Johannes Weiner
   7 *
   8 * Access to this subsystem has to be serialized externally (which is true
   9 * for the boot process anyway).
  10 */
  11#include <linux/init.h>
  12#include <linux/pfn.h>
  13#include <linux/slab.h>
  14#include <linux/bootmem.h>
  15#include <linux/export.h>
  16#include <linux/kmemleak.h>
  17#include <linux/range.h>
  18#include <linux/memblock.h>
  19
  20#include <asm/bug.h>
  21#include <asm/io.h>
  22#include <asm/processor.h>
  23
  24#include "internal.h"
  25
  26#ifndef CONFIG_NEED_MULTIPLE_NODES
  27struct pglist_data __refdata contig_page_data;
  28EXPORT_SYMBOL(contig_page_data);
  29#endif
  30
  31unsigned long max_low_pfn;
  32unsigned long min_low_pfn;
  33unsigned long max_pfn;
  34
  35static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
  36                                        u64 goal, u64 limit)
  37{
  38        void *ptr;
  39        u64 addr;
  40
  41        if (limit > memblock.current_limit)
  42                limit = memblock.current_limit;
  43
  44        addr = memblock_find_in_range_node(goal, limit, size, align, nid);
  45        if (!addr)
  46                return NULL;
  47
  48        memblock_reserve(addr, size);
  49        ptr = phys_to_virt(addr);
  50        memset(ptr, 0, size);
  51        /*
  52         * The min_count is set to 0 so that bootmem allocated blocks
  53         * are never reported as leaks.
  54         */
  55        kmemleak_alloc(ptr, size, 0, 0);
  56        return ptr;
  57}
  58
  59/*
  60 * free_bootmem_late - free bootmem pages directly to page allocator
  61 * @addr: starting address of the range
  62 * @size: size of the range in bytes
  63 *
  64 * This is only useful when the bootmem allocator has already been torn
  65 * down, but we are still initializing the system.  Pages are given directly
  66 * to the page allocator, no bootmem metadata is updated because it is gone.
  67 */
  68void __init free_bootmem_late(unsigned long addr, unsigned long size)
  69{
  70        unsigned long cursor, end;
  71
  72        kmemleak_free_part(__va(addr), size);
  73
  74        cursor = PFN_UP(addr);
  75        end = PFN_DOWN(addr + size);
  76
  77        for (; cursor < end; cursor++) {
  78                __free_pages_bootmem(pfn_to_page(cursor), 0);
  79                totalram_pages++;
  80        }
  81}
  82
  83static void __init __free_pages_memory(unsigned long start, unsigned long end)
  84{
  85        unsigned long i, start_aligned, end_aligned;
  86        int order = ilog2(BITS_PER_LONG);
  87
  88        start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
  89        end_aligned = end & ~(BITS_PER_LONG - 1);
  90
  91        if (end_aligned <= start_aligned) {
  92                for (i = start; i < end; i++)
  93                        __free_pages_bootmem(pfn_to_page(i), 0);
  94
  95                return;
  96        }
  97
  98        for (i = start; i < start_aligned; i++)
  99                __free_pages_bootmem(pfn_to_page(i), 0);
 100
 101        for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
 102                __free_pages_bootmem(pfn_to_page(i), order);
 103
 104        for (i = end_aligned; i < end; i++)
 105                __free_pages_bootmem(pfn_to_page(i), 0);
 106}
 107
 108static unsigned long __init __free_memory_core(phys_addr_t start,
 109                                 phys_addr_t end)
 110{
 111        unsigned long start_pfn = PFN_UP(start);
 112        unsigned long end_pfn = min_t(unsigned long,
 113                                      PFN_DOWN(end), max_low_pfn);
 114
 115        if (start_pfn > end_pfn)
 116                return 0;
 117
 118        __free_pages_memory(start_pfn, end_pfn);
 119
 120        return end_pfn - start_pfn;
 121}
 122
 123static unsigned long __init free_low_memory_core_early(void)
 124{
 125        unsigned long count = 0;
 126        phys_addr_t start, end, size;
 127        u64 i;
 128
 129        for_each_free_mem_range(i, MAX_NUMNODES, &start, &end, NULL)
 130                count += __free_memory_core(start, end);
 131
 132        /* free range that is used for reserved array if we allocate it */
 133        size = get_allocated_memblock_reserved_regions_info(&start);
 134        if (size)
 135                count += __free_memory_core(start, start + size);
 136
 137        return count;
 138}
 139
 140static int reset_managed_pages_done __initdata;
 141
 142static inline void __init reset_node_managed_pages(pg_data_t *pgdat)
 143{
 144        struct zone *z;
 145
 146        if (reset_managed_pages_done)
 147                return;
 148        for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
 149                z->managed_pages = 0;
 150}
 151
 152void __init reset_all_zones_managed_pages(void)
 153{
 154        struct pglist_data *pgdat;
 155
 156        for_each_online_pgdat(pgdat)
 157                reset_node_managed_pages(pgdat);
 158        reset_managed_pages_done = 1;
 159}
 160
 161/**
 162 * free_all_bootmem - release free pages to the buddy allocator
 163 *
 164 * Returns the number of pages actually released.
 165 */
 166unsigned long __init free_all_bootmem(void)
 167{
 168        unsigned long pages;
 169
 170        reset_all_zones_managed_pages();
 171
 172        /*
 173         * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
 174         *  because in some case like Node0 doesn't have RAM installed
 175         *  low ram will be on Node1
 176         */
 177        pages = free_low_memory_core_early();
 178        totalram_pages += pages;
 179
 180        return pages;
 181}
 182
 183/**
 184 * free_bootmem_node - mark a page range as usable
 185 * @pgdat: node the range resides on
 186 * @physaddr: starting address of the range
 187 * @size: size of the range in bytes
 188 *
 189 * Partial pages will be considered reserved and left as they are.
 190 *
 191 * The range must reside completely on the specified node.
 192 */
 193void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 194                              unsigned long size)
 195{
 196        kmemleak_free_part(__va(physaddr), size);
 197        memblock_free(physaddr, size);
 198}
 199
 200/**
 201 * free_bootmem - mark a page range as usable
 202 * @addr: starting address of the range
 203 * @size: size of the range in bytes
 204 *
 205 * Partial pages will be considered reserved and left as they are.
 206 *
 207 * The range must be contiguous but may span node boundaries.
 208 */
 209void __init free_bootmem(unsigned long addr, unsigned long size)
 210{
 211        kmemleak_free_part(__va(addr), size);
 212        memblock_free(addr, size);
 213}
 214
 215static void * __init ___alloc_bootmem_nopanic(unsigned long size,
 216                                        unsigned long align,
 217                                        unsigned long goal,
 218                                        unsigned long limit)
 219{
 220        void *ptr;
 221
 222        if (WARN_ON_ONCE(slab_is_available()))
 223                return kzalloc(size, GFP_NOWAIT);
 224
 225restart:
 226
 227        ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
 228
 229        if (ptr)
 230                return ptr;
 231
 232        if (goal != 0) {
 233                goal = 0;
 234                goto restart;
 235        }
 236
 237        return NULL;
 238}
 239
 240/**
 241 * __alloc_bootmem_nopanic - allocate boot memory without panicking
 242 * @size: size of the request in bytes
 243 * @align: alignment of the region
 244 * @goal: preferred starting address of the region
 245 *
 246 * The goal is dropped if it can not be satisfied and the allocation will
 247 * fall back to memory below @goal.
 248 *
 249 * Allocation may happen on any node in the system.
 250 *
 251 * Returns NULL on failure.
 252 */
 253void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
 254                                        unsigned long goal)
 255{
 256        unsigned long limit = -1UL;
 257
 258        return ___alloc_bootmem_nopanic(size, align, goal, limit);
 259}
 260
 261static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
 262                                        unsigned long goal, unsigned long limit)
 263{
 264        void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit);
 265
 266        if (mem)
 267                return mem;
 268        /*
 269         * Whoops, we cannot satisfy the allocation request.
 270         */
 271        printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
 272        panic("Out of memory");
 273        return NULL;
 274}
 275
 276/**
 277 * __alloc_bootmem - allocate boot memory
 278 * @size: size of the request in bytes
 279 * @align: alignment of the region
 280 * @goal: preferred starting address of the region
 281 *
 282 * The goal is dropped if it can not be satisfied and the allocation will
 283 * fall back to memory below @goal.
 284 *
 285 * Allocation may happen on any node in the system.
 286 *
 287 * The function panics if the request can not be satisfied.
 288 */
 289void * __init __alloc_bootmem(unsigned long size, unsigned long align,
 290                              unsigned long goal)
 291{
 292        unsigned long limit = -1UL;
 293
 294        return ___alloc_bootmem(size, align, goal, limit);
 295}
 296
 297void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat,
 298                                                   unsigned long size,
 299                                                   unsigned long align,
 300                                                   unsigned long goal,
 301                                                   unsigned long limit)
 302{
 303        void *ptr;
 304
 305again:
 306        ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
 307                                        goal, limit);
 308        if (ptr)
 309                return ptr;
 310
 311        ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
 312                                        goal, limit);
 313        if (ptr)
 314                return ptr;
 315
 316        if (goal) {
 317                goal = 0;
 318                goto again;
 319        }
 320
 321        return NULL;
 322}
 323
 324void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
 325                                   unsigned long align, unsigned long goal)
 326{
 327        if (WARN_ON_ONCE(slab_is_available()))
 328                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 329
 330        return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0);
 331}
 332
 333void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
 334                                    unsigned long align, unsigned long goal,
 335                                    unsigned long limit)
 336{
 337        void *ptr;
 338
 339        ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, limit);
 340        if (ptr)
 341                return ptr;
 342
 343        printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
 344        panic("Out of memory");
 345        return NULL;
 346}
 347
 348/**
 349 * __alloc_bootmem_node - allocate boot memory from a specific node
 350 * @pgdat: node to allocate from
 351 * @size: size of the request in bytes
 352 * @align: alignment of the region
 353 * @goal: preferred starting address of the region
 354 *
 355 * The goal is dropped if it can not be satisfied and the allocation will
 356 * fall back to memory below @goal.
 357 *
 358 * Allocation may fall back to any node in the system if the specified node
 359 * can not hold the requested memory.
 360 *
 361 * The function panics if the request can not be satisfied.
 362 */
 363void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
 364                                   unsigned long align, unsigned long goal)
 365{
 366        if (WARN_ON_ONCE(slab_is_available()))
 367                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 368
 369        return ___alloc_bootmem_node(pgdat, size, align, goal, 0);
 370}
 371
 372void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
 373                                   unsigned long align, unsigned long goal)
 374{
 375        return __alloc_bootmem_node(pgdat, size, align, goal);
 376}
 377
 378#ifndef ARCH_LOW_ADDRESS_LIMIT
 379#define ARCH_LOW_ADDRESS_LIMIT  0xffffffffUL
 380#endif
 381
 382/**
 383 * __alloc_bootmem_low - allocate low boot memory
 384 * @size: size of the request in bytes
 385 * @align: alignment of the region
 386 * @goal: preferred starting address of the region
 387 *
 388 * The goal is dropped if it can not be satisfied and the allocation will
 389 * fall back to memory below @goal.
 390 *
 391 * Allocation may happen on any node in the system.
 392 *
 393 * The function panics if the request can not be satisfied.
 394 */
 395void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
 396                                  unsigned long goal)
 397{
 398        return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT);
 399}
 400
 401void * __init __alloc_bootmem_low_nopanic(unsigned long size,
 402                                          unsigned long align,
 403                                          unsigned long goal)
 404{
 405        return ___alloc_bootmem_nopanic(size, align, goal,
 406                                        ARCH_LOW_ADDRESS_LIMIT);
 407}
 408
 409/**
 410 * __alloc_bootmem_low_node - allocate low boot memory from a specific node
 411 * @pgdat: node to allocate from
 412 * @size: size of the request in bytes
 413 * @align: alignment of the region
 414 * @goal: preferred starting address of the region
 415 *
 416 * The goal is dropped if it can not be satisfied and the allocation will
 417 * fall back to memory below @goal.
 418 *
 419 * Allocation may fall back to any node in the system if the specified node
 420 * can not hold the requested memory.
 421 *
 422 * The function panics if the request can not be satisfied.
 423 */
 424void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
 425                                       unsigned long align, unsigned long goal)
 426{
 427        if (WARN_ON_ONCE(slab_is_available()))
 428                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 429
 430        return ___alloc_bootmem_node(pgdat, size, align, goal,
 431                                     ARCH_LOW_ADDRESS_LIMIT);
 432}
 433
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.