linux/mm/page_alloc.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/mm/page_alloc.c
   4 *
   5 *  Manages the free list, the system allocates free pages here.
   6 *  Note that kmalloc() lives in slab.c
   7 *
   8 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   9 *  Swap reorganised 29.12.95, Stephen Tweedie
  10 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  11 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  12 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  13 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  14 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  15 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  16 */
  17
  18#include <linux/stddef.h>
  19#include <linux/mm.h>
  20#include <linux/highmem.h>
  21#include <linux/swap.h>
  22#include <linux/interrupt.h>
  23#include <linux/pagemap.h>
  24#include <linux/jiffies.h>
  25#include <linux/memblock.h>
  26#include <linux/compiler.h>
  27#include <linux/kernel.h>
  28#include <linux/kasan.h>
  29#include <linux/module.h>
  30#include <linux/suspend.h>
  31#include <linux/pagevec.h>
  32#include <linux/blkdev.h>
  33#include <linux/slab.h>
  34#include <linux/ratelimit.h>
  35#include <linux/oom.h>
  36#include <linux/topology.h>
  37#include <linux/sysctl.h>
  38#include <linux/cpu.h>
  39#include <linux/cpuset.h>
  40#include <linux/memory_hotplug.h>
  41#include <linux/nodemask.h>
  42#include <linux/vmalloc.h>
  43#include <linux/vmstat.h>
  44#include <linux/mempolicy.h>
  45#include <linux/memremap.h>
  46#include <linux/stop_machine.h>
  47#include <linux/random.h>
  48#include <linux/sort.h>
  49#include <linux/pfn.h>
  50#include <linux/backing-dev.h>
  51#include <linux/fault-inject.h>
  52#include <linux/page-isolation.h>
  53#include <linux/debugobjects.h>
  54#include <linux/kmemleak.h>
  55#include <linux/compaction.h>
  56#include <trace/events/kmem.h>
  57#include <trace/events/oom.h>
  58#include <linux/prefetch.h>
  59#include <linux/mm_inline.h>
  60#include <linux/mmu_notifier.h>
  61#include <linux/migrate.h>
  62#include <linux/hugetlb.h>
  63#include <linux/sched/rt.h>
  64#include <linux/sched/mm.h>
  65#include <linux/page_owner.h>
  66#include <linux/kthread.h>
  67#include <linux/memcontrol.h>
  68#include <linux/ftrace.h>
  69#include <linux/lockdep.h>
  70#include <linux/nmi.h>
  71#include <linux/psi.h>
  72#include <linux/padata.h>
  73#include <linux/khugepaged.h>
  74#include <linux/buffer_head.h>
  75#include <asm/sections.h>
  76#include <asm/tlbflush.h>
  77#include <asm/div64.h>
  78#include "internal.h"
  79#include "shuffle.h"
  80#include "page_reporting.h"
  81
  82/* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
  83typedef int __bitwise fpi_t;
  84
  85/* No special request */
  86#define FPI_NONE                ((__force fpi_t)0)
  87
  88/*
  89 * Skip free page reporting notification for the (possibly merged) page.
  90 * This does not hinder free page reporting from grabbing the page,
  91 * reporting it and marking it "reported" -  it only skips notifying
  92 * the free page reporting infrastructure about a newly freed page. For
  93 * example, used when temporarily pulling a page from a freelist and
  94 * putting it back unmodified.
  95 */
  96#define FPI_SKIP_REPORT_NOTIFY  ((__force fpi_t)BIT(0))
  97
  98/*
  99 * Place the (possibly merged) page to the tail of the freelist. Will ignore
 100 * page shuffling (relevant code - e.g., memory onlining - is expected to
 101 * shuffle the whole zone).
 102 *
 103 * Note: No code should rely on this flag for correctness - it's purely
 104 *       to allow for optimizations when handing back either fresh pages
 105 *       (memory onlining) or untouched pages (page isolation, free page
 106 *       reporting).
 107 */
 108#define FPI_TO_TAIL             ((__force fpi_t)BIT(1))
 109
 110/*
 111 * Don't poison memory with KASAN (only for the tag-based modes).
 112 * During boot, all non-reserved memblock memory is exposed to page_alloc.
 113 * Poisoning all that memory lengthens boot time, especially on systems with
 114 * large amount of RAM. This flag is used to skip that poisoning.
 115 * This is only done for the tag-based KASAN modes, as those are able to
 116 * detect memory corruptions with the memory tags assigned by default.
 117 * All memory allocated normally after boot gets poisoned as usual.
 118 */
 119#define FPI_SKIP_KASAN_POISON   ((__force fpi_t)BIT(2))
 120
 121/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
 122static DEFINE_MUTEX(pcp_batch_high_lock);
 123#define MIN_PERCPU_PAGELIST_FRACTION    (8)
 124
 125#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
 126DEFINE_PER_CPU(int, numa_node);
 127EXPORT_PER_CPU_SYMBOL(numa_node);
 128#endif
 129
 130DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
 131
 132#ifdef CONFIG_HAVE_MEMORYLESS_NODES
 133/*
 134 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 135 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 136 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 137 * defined in <linux/topology.h>.
 138 */
 139DEFINE_PER_CPU(int, _numa_mem_);                /* Kernel "local memory" node */
 140EXPORT_PER_CPU_SYMBOL(_numa_mem_);
 141#endif
 142
 143/* work_structs for global per-cpu drains */
 144struct pcpu_drain {
 145        struct zone *zone;
 146        struct work_struct work;
 147};
 148static DEFINE_MUTEX(pcpu_drain_mutex);
 149static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
 150
 151#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
 152volatile unsigned long latent_entropy __latent_entropy;
 153EXPORT_SYMBOL(latent_entropy);
 154#endif
 155
 156/*
 157 * Array of node states.
 158 */
 159nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
 160        [N_POSSIBLE] = NODE_MASK_ALL,
 161        [N_ONLINE] = { { [0] = 1UL } },
 162#ifndef CONFIG_NUMA
 163        [N_NORMAL_MEMORY] = { { [0] = 1UL } },
 164#ifdef CONFIG_HIGHMEM
 165        [N_HIGH_MEMORY] = { { [0] = 1UL } },
 166#endif
 167        [N_MEMORY] = { { [0] = 1UL } },
 168        [N_CPU] = { { [0] = 1UL } },
 169#endif  /* NUMA */
 170};
 171EXPORT_SYMBOL(node_states);
 172
 173atomic_long_t _totalram_pages __read_mostly;
 174EXPORT_SYMBOL(_totalram_pages);
 175unsigned long totalreserve_pages __read_mostly;
 176unsigned long totalcma_pages __read_mostly;
 177
 178int percpu_pagelist_fraction;
 179gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
 180DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
 181EXPORT_SYMBOL(init_on_alloc);
 182
 183DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
 184EXPORT_SYMBOL(init_on_free);
 185
 186static bool _init_on_alloc_enabled_early __read_mostly
 187                                = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
 188static int __init early_init_on_alloc(char *buf)
 189{
 190
 191        return kstrtobool(buf, &_init_on_alloc_enabled_early);
 192}
 193early_param("init_on_alloc", early_init_on_alloc);
 194
 195static bool _init_on_free_enabled_early __read_mostly
 196                                = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);
 197static int __init early_init_on_free(char *buf)
 198{
 199        return kstrtobool(buf, &_init_on_free_enabled_early);
 200}
 201early_param("init_on_free", early_init_on_free);
 202
 203/*
 204 * A cached value of the page's pageblock's migratetype, used when the page is
 205 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
 206 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
 207 * Also the migratetype set in the page does not necessarily match the pcplist
 208 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
 209 * other index - this ensures that it will be put on the correct CMA freelist.
 210 */
 211static inline int get_pcppage_migratetype(struct page *page)
 212{
 213        return page->index;
 214}
 215
 216static inline void set_pcppage_migratetype(struct page *page, int migratetype)
 217{
 218        page->index = migratetype;
 219}
 220
 221#ifdef CONFIG_PM_SLEEP
 222/*
 223 * The following functions are used by the suspend/hibernate code to temporarily
 224 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
 225 * while devices are suspended.  To avoid races with the suspend/hibernate code,
 226 * they should always be called with system_transition_mutex held
 227 * (gfp_allowed_mask also should only be modified with system_transition_mutex
 228 * held, unless the suspend/hibernate code is guaranteed not to run in parallel
 229 * with that modification).
 230 */
 231
 232static gfp_t saved_gfp_mask;
 233
 234void pm_restore_gfp_mask(void)
 235{
 236        WARN_ON(!mutex_is_locked(&system_transition_mutex));
 237        if (saved_gfp_mask) {
 238                gfp_allowed_mask = saved_gfp_mask;
 239                saved_gfp_mask = 0;
 240        }
 241}
 242
 243void pm_restrict_gfp_mask(void)
 244{
 245        WARN_ON(!mutex_is_locked(&system_transition_mutex));
 246        WARN_ON(saved_gfp_mask);
 247        saved_gfp_mask = gfp_allowed_mask;
 248        gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
 249}
 250
 251bool pm_suspended_storage(void)
 252{
 253        if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
 254                return false;
 255        return true;
 256}
 257#endif /* CONFIG_PM_SLEEP */
 258
 259#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 260unsigned int pageblock_order __read_mostly;
 261#endif
 262
 263static void __free_pages_ok(struct page *page, unsigned int order,
 264                            fpi_t fpi_flags);
 265
 266/*
 267 * results with 256, 32 in the lowmem_reserve sysctl:
 268 *      1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 269 *      1G machine -> (16M dma, 784M normal, 224M high)
 270 *      NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 271 *      HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
 272 *      HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
 273 *
 274 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 275 * don't need any ZONE_NORMAL reservation
 276 */
 277int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
 278#ifdef CONFIG_ZONE_DMA
 279        [ZONE_DMA] = 256,
 280#endif
 281#ifdef CONFIG_ZONE_DMA32
 282        [ZONE_DMA32] = 256,
 283#endif
 284        [ZONE_NORMAL] = 32,
 285#ifdef CONFIG_HIGHMEM
 286        [ZONE_HIGHMEM] = 0,
 287#endif
 288        [ZONE_MOVABLE] = 0,
 289};
 290
 291static char * const zone_names[MAX_NR_ZONES] = {
 292#ifdef CONFIG_ZONE_DMA
 293         "DMA",
 294#endif
 295#ifdef CONFIG_ZONE_DMA32
 296         "DMA32",
 297#endif
 298         "Normal",
 299#ifdef CONFIG_HIGHMEM
 300         "HighMem",
 301#endif
 302         "Movable",
 303#ifdef CONFIG_ZONE_DEVICE
 304         "Device",
 305#endif
 306};
 307
 308const char * const migratetype_names[MIGRATE_TYPES] = {
 309        "Unmovable",
 310        "Movable",
 311        "Reclaimable",
 312        "HighAtomic",
 313#ifdef CONFIG_CMA
 314        "CMA",
 315#endif
 316#ifdef CONFIG_MEMORY_ISOLATION
 317        "Isolate",
 318#endif
 319};
 320
 321compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {
 322        [NULL_COMPOUND_DTOR] = NULL,
 323        [COMPOUND_PAGE_DTOR] = free_compound_page,
 324#ifdef CONFIG_HUGETLB_PAGE
 325        [HUGETLB_PAGE_DTOR] = free_huge_page,
 326#endif
 327#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 328        [TRANSHUGE_PAGE_DTOR] = free_transhuge_page,
 329#endif
 330};
 331
 332int min_free_kbytes = 1024;
 333int user_min_free_kbytes = -1;
 334#ifdef CONFIG_DISCONTIGMEM
 335/*
 336 * DiscontigMem defines memory ranges as separate pg_data_t even if the ranges
 337 * are not on separate NUMA nodes. Functionally this works but with
 338 * watermark_boost_factor, it can reclaim prematurely as the ranges can be
 339 * quite small. By default, do not boost watermarks on discontigmem as in
 340 * many cases very high-order allocations like THP are likely to be
 341 * unsupported and the premature reclaim offsets the advantage of long-term
 342 * fragmentation avoidance.
 343 */
 344int watermark_boost_factor __read_mostly;
 345#else
 346int watermark_boost_factor __read_mostly = 15000;
 347#endif
 348int watermark_scale_factor = 10;
 349
 350static unsigned long nr_kernel_pages __initdata;
 351static unsigned long nr_all_pages __initdata;
 352static unsigned long dma_reserve __initdata;
 353
 354static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
 355static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
 356static unsigned long required_kernelcore __initdata;
 357static unsigned long required_kernelcore_percent __initdata;
 358static unsigned long required_movablecore __initdata;
 359static unsigned long required_movablecore_percent __initdata;
 360static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
 361static bool mirrored_kernelcore __meminitdata;
 362
 363/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
 364int movable_zone;
 365EXPORT_SYMBOL(movable_zone);
 366
 367#if MAX_NUMNODES > 1
 368unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
 369unsigned int nr_online_nodes __read_mostly = 1;
 370EXPORT_SYMBOL(nr_node_ids);
 371EXPORT_SYMBOL(nr_online_nodes);
 372#endif
 373
 374int page_group_by_mobility_disabled __read_mostly;
 375
 376#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
 377/*
 378 * During boot we initialize deferred pages on-demand, as needed, but once
 379 * page_alloc_init_late() has finished, the deferred pages are all initialized,
 380 * and we can permanently disable that path.
 381 */
 382static DEFINE_STATIC_KEY_TRUE(deferred_pages);
 383
 384/*
 385 * Calling kasan_free_pages() only after deferred memory initialization
 386 * has completed. Poisoning pages during deferred memory init will greatly
 387 * lengthen the process and cause problem in large memory systems as the
 388 * deferred pages initialization is done with interrupt disabled.
 389 *
 390 * Assuming that there will be no reference to those newly initialized
 391 * pages before they are ever allocated, this should have no effect on
 392 * KASAN memory tracking as the poison will be properly inserted at page
 393 * allocation time. The only corner case is when pages are allocated by
 394 * on-demand allocation and then freed again before the deferred pages
 395 * initialization is done, but this is not likely to happen.
 396 */
 397static inline void kasan_free_nondeferred_pages(struct page *page, int order,
 398                                                bool init, fpi_t fpi_flags)
 399{
 400        if (static_branch_unlikely(&deferred_pages))
 401                return;
 402        if (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
 403                        (fpi_flags & FPI_SKIP_KASAN_POISON))
 404                return;
 405        kasan_free_pages(page, order, init);
 406}
 407
 408/* Returns true if the struct page for the pfn is uninitialised */
 409static inline bool __meminit early_page_uninitialised(unsigned long pfn)
 410{
 411        int nid = early_pfn_to_nid(pfn);
 412
 413        if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
 414                return true;
 415
 416        return false;
 417}
 418
 419/*
 420 * Returns true when the remaining initialisation should be deferred until
 421 * later in the boot cycle when it can be parallelised.
 422 */
 423static bool __meminit
 424defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
 425{
 426        static unsigned long prev_end_pfn, nr_initialised;
 427
 428        /*
 429         * prev_end_pfn static that contains the end of previous zone
 430         * No need to protect because called very early in boot before smp_init.
 431         */
 432        if (prev_end_pfn != end_pfn) {
 433                prev_end_pfn = end_pfn;
 434                nr_initialised = 0;
 435        }
 436
 437        /* Always populate low zones for address-constrained allocations */
 438        if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
 439                return false;
 440
 441        if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
 442                return true;
 443        /*
 444         * We start only with one section of pages, more pages are added as
 445         * needed until the rest of deferred pages are initialized.
 446         */
 447        nr_initialised++;
 448        if ((nr_initialised > PAGES_PER_SECTION) &&
 449            (pfn & (PAGES_PER_SECTION - 1)) == 0) {
 450                NODE_DATA(nid)->first_deferred_pfn = pfn;
 451                return true;
 452        }
 453        return false;
 454}
 455#else
 456static inline void kasan_free_nondeferred_pages(struct page *page, int order,
 457                                                bool init, fpi_t fpi_flags)
 458{
 459        if (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
 460                        (fpi_flags & FPI_SKIP_KASAN_POISON))
 461                return;
 462        kasan_free_pages(page, order, init);
 463}
 464
 465static inline bool early_page_uninitialised(unsigned long pfn)
 466{
 467        return false;
 468}
 469
 470static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
 471{
 472        return false;
 473}
 474#endif
 475
 476/* Return a pointer to the bitmap storing bits affecting a block of pages */
 477static inline unsigned long *get_pageblock_bitmap(struct page *page,
 478                                                        unsigned long pfn)
 479{
 480#ifdef CONFIG_SPARSEMEM
 481        return section_to_usemap(__pfn_to_section(pfn));
 482#else
 483        return page_zone(page)->pageblock_flags;
 484#endif /* CONFIG_SPARSEMEM */
 485}
 486
 487static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
 488{
 489#ifdef CONFIG_SPARSEMEM
 490        pfn &= (PAGES_PER_SECTION-1);
 491#else
 492        pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
 493#endif /* CONFIG_SPARSEMEM */
 494        return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
 495}
 496
 497static __always_inline
 498unsigned long __get_pfnblock_flags_mask(struct page *page,
 499                                        unsigned long pfn,
 500                                        unsigned long mask)
 501{
 502        unsigned long *bitmap;
 503        unsigned long bitidx, word_bitidx;
 504        unsigned long word;
 505
 506        bitmap = get_pageblock_bitmap(page, pfn);
 507        bitidx = pfn_to_bitidx(page, pfn);
 508        word_bitidx = bitidx / BITS_PER_LONG;
 509        bitidx &= (BITS_PER_LONG-1);
 510
 511        word = bitmap[word_bitidx];
 512        return (word >> bitidx) & mask;
 513}
 514
 515/**
 516 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
 517 * @page: The page within the block of interest
 518 * @pfn: The target page frame number
 519 * @mask: mask of bits that the caller is interested in
 520 *
 521 * Return: pageblock_bits flags
 522 */
 523unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
 524                                        unsigned long mask)
 525{
 526        return __get_pfnblock_flags_mask(page, pfn, mask);
 527}
 528
 529static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
 530{
 531        return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
 532}
 533
 534/**
 535 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
 536 * @page: The page within the block of interest
 537 * @flags: The flags to set
 538 * @pfn: The target page frame number
 539 * @mask: mask of bits that the caller is interested in
 540 */
 541void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
 542                                        unsigned long pfn,
 543                                        unsigned long mask)
 544{
 545        unsigned long *bitmap;
 546        unsigned long bitidx, word_bitidx;
 547        unsigned long old_word, word;
 548
 549        BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
 550        BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
 551
 552        bitmap = get_pageblock_bitmap(page, pfn);
 553        bitidx = pfn_to_bitidx(page, pfn);
 554        word_bitidx = bitidx / BITS_PER_LONG;
 555        bitidx &= (BITS_PER_LONG-1);
 556
 557        VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
 558
 559        mask <<= bitidx;
 560        flags <<= bitidx;
 561
 562        word = READ_ONCE(bitmap[word_bitidx]);
 563        for (;;) {
 564                old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
 565                if (word == old_word)
 566                        break;
 567                word = old_word;
 568        }
 569}
 570
 571void set_pageblock_migratetype(struct page *page, int migratetype)
 572{
 573        if (unlikely(page_group_by_mobility_disabled &&
 574                     migratetype < MIGRATE_PCPTYPES))
 575                migratetype = MIGRATE_UNMOVABLE;
 576
 577        set_pfnblock_flags_mask(page, (unsigned long)migratetype,
 578                                page_to_pfn(page), MIGRATETYPE_MASK);
 579}
 580
 581#ifdef CONFIG_DEBUG_VM
 582static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 583{
 584        int ret = 0;
 585        unsigned seq;
 586        unsigned long pfn = page_to_pfn(page);
 587        unsigned long sp, start_pfn;
 588
 589        do {
 590                seq = zone_span_seqbegin(zone);
 591                start_pfn = zone->zone_start_pfn;
 592                sp = zone->spanned_pages;
 593                if (!zone_spans_pfn(zone, pfn))
 594                        ret = 1;
 595        } while (zone_span_seqretry(zone, seq));
 596
 597        if (ret)
 598                pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
 599                        pfn, zone_to_nid(zone), zone->name,
 600                        start_pfn, start_pfn + sp);
 601
 602        return ret;
 603}
 604
 605static int page_is_consistent(struct zone *zone, struct page *page)
 606{
 607        if (!pfn_valid_within(page_to_pfn(page)))
 608                return 0;
 609        if (zone != page_zone(page))
 610                return 0;
 611
 612        return 1;
 613}
 614/*
 615 * Temporary debugging check for pages not lying within a given zone.
 616 */
 617static int __maybe_unused bad_range(struct zone *zone, struct page *page)
 618{
 619        if (page_outside_zone_boundaries(zone, page))
 620                return 1;
 621        if (!page_is_consistent(zone, page))
 622                return 1;
 623
 624        return 0;
 625}
 626#else
 627static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
 628{
 629        return 0;
 630}
 631#endif
 632
 633static void bad_page(struct page *page, const char *reason)
 634{
 635        static unsigned long resume;
 636        static unsigned long nr_shown;
 637        static unsigned long nr_unshown;
 638
 639        /*
 640         * Allow a burst of 60 reports, then keep quiet for that minute;
 641         * or allow a steady drip of one report per second.
 642         */
 643        if (nr_shown == 60) {
 644                if (time_before(jiffies, resume)) {
 645                        nr_unshown++;
 646                        goto out;
 647                }
 648                if (nr_unshown) {
 649                        pr_alert(
 650                              "BUG: Bad page state: %lu messages suppressed\n",
 651                                nr_unshown);
 652                        nr_unshown = 0;
 653                }
 654                nr_shown = 0;
 655        }
 656        if (nr_shown++ == 0)
 657                resume = jiffies + 60 * HZ;
 658
 659        pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
 660                current->comm, page_to_pfn(page));
 661        __dump_page(page, reason);
 662        dump_page_owner(page);
 663
 664        print_modules();
 665        dump_stack();
 666out:
 667        /* Leave bad fields for debug, except PageBuddy could make trouble */
 668        page_mapcount_reset(page); /* remove PageBuddy */
 669        add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 670}
 671
 672/*
 673 * Higher-order pages are called "compound pages".  They are structured thusly:
 674 *
 675 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
 676 *
 677 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
 678 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
 679 *
 680 * The first tail page's ->compound_dtor holds the offset in array of compound
 681 * page destructors. See compound_page_dtors.
 682 *
 683 * The first tail page's ->compound_order holds the order of allocation.
 684 * This usage means that zero-order pages may not be compound.
 685 */
 686
 687void free_compound_page(struct page *page)
 688{
 689        mem_cgroup_uncharge(page);
 690        __free_pages_ok(page, compound_order(page), FPI_NONE);
 691}
 692
 693void prep_compound_page(struct page *page, unsigned int order)
 694{
 695        int i;
 696        int nr_pages = 1 << order;
 697
 698        __SetPageHead(page);
 699        for (i = 1; i < nr_pages; i++) {
 700                struct page *p = page + i;
 701                set_page_count(p, 0);
 702                p->mapping = TAIL_MAPPING;
 703                set_compound_head(p, page);
 704        }
 705
 706        set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
 707        set_compound_order(page, order);
 708        atomic_set(compound_mapcount_ptr(page), -1);
 709        if (hpage_pincount_available(page))
 710                atomic_set(compound_pincount_ptr(page), 0);
 711}
 712
 713#ifdef CONFIG_DEBUG_PAGEALLOC
 714unsigned int _debug_guardpage_minorder;
 715
 716bool _debug_pagealloc_enabled_early __read_mostly
 717                        = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
 718EXPORT_SYMBOL(_debug_pagealloc_enabled_early);
 719DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
 720EXPORT_SYMBOL(_debug_pagealloc_enabled);
 721
 722DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
 723
 724static int __init early_debug_pagealloc(char *buf)
 725{
 726        return kstrtobool(buf, &_debug_pagealloc_enabled_early);
 727}
 728early_param("debug_pagealloc", early_debug_pagealloc);
 729
 730static int __init debug_guardpage_minorder_setup(char *buf)
 731{
 732        unsigned long res;
 733
 734        if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
 735                pr_err("Bad debug_guardpage_minorder value\n");
 736                return 0;
 737        }
 738        _debug_guardpage_minorder = res;
 739        pr_info("Setting debug_guardpage_minorder to %lu\n", res);
 740        return 0;
 741}
 742early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
 743
 744static inline bool set_page_guard(struct zone *zone, struct page *page,
 745                                unsigned int order, int migratetype)
 746{
 747        if (!debug_guardpage_enabled())
 748                return false;
 749
 750        if (order >= debug_guardpage_minorder())
 751                return false;
 752
 753        __SetPageGuard(page);
 754        INIT_LIST_HEAD(&page->lru);
 755        set_page_private(page, order);
 756        /* Guard pages are not available for any usage */
 757        __mod_zone_freepage_state(zone, -(1 << order), migratetype);
 758
 759        return true;
 760}
 761
 762static inline void clear_page_guard(struct zone *zone, struct page *page,
 763                                unsigned int order, int migratetype)
 764{
 765        if (!debug_guardpage_enabled())
 766                return;
 767
 768        __ClearPageGuard(page);
 769
 770        set_page_private(page, 0);
 771        if (!is_migrate_isolate(migratetype))
 772                __mod_zone_freepage_state(zone, (1 << order), migratetype);
 773}
 774#else
 775static inline bool set_page_guard(struct zone *zone, struct page *page,
 776                        unsigned int order, int migratetype) { return false; }
 777static inline void clear_page_guard(struct zone *zone, struct page *page,
 778                                unsigned int order, int migratetype) {}
 779#endif
 780
 781/*
 782 * Enable static keys related to various memory debugging and hardening options.
 783 * Some override others, and depend on early params that are evaluated in the
 784 * order of appearance. So we need to first gather the full picture of what was
 785 * enabled, and then make decisions.
 786 */
 787void init_mem_debugging_and_hardening(void)
 788{
 789        bool page_poisoning_requested = false;
 790
 791#ifdef CONFIG_PAGE_POISONING
 792        /*
 793         * Page poisoning is debug page alloc for some arches. If
 794         * either of those options are enabled, enable poisoning.
 795         */
 796        if (page_poisoning_enabled() ||
 797             (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
 798              debug_pagealloc_enabled())) {
 799                static_branch_enable(&_page_poisoning_enabled);
 800                page_poisoning_requested = true;
 801        }
 802#endif
 803
 804        if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
 805            page_poisoning_requested) {
 806                pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
 807                        "will take precedence over init_on_alloc and init_on_free\n");
 808                _init_on_alloc_enabled_early = false;
 809                _init_on_free_enabled_early = false;
 810        }
 811
 812        if (_init_on_alloc_enabled_early)
 813                static_branch_enable(&init_on_alloc);
 814        else
 815                static_branch_disable(&init_on_alloc);
 816
 817        if (_init_on_free_enabled_early)
 818                static_branch_enable(&init_on_free);
 819        else
 820                static_branch_disable(&init_on_free);
 821
 822#ifdef CONFIG_DEBUG_PAGEALLOC
 823        if (!debug_pagealloc_enabled())
 824                return;
 825
 826        static_branch_enable(&_debug_pagealloc_enabled);
 827
 828        if (!debug_guardpage_minorder())
 829                return;
 830
 831        static_branch_enable(&_debug_guardpage_enabled);
 832#endif
 833}
 834
 835static inline void set_buddy_order(struct page *page, unsigned int order)
 836{
 837        set_page_private(page, order);
 838        __SetPageBuddy(page);
 839}
 840
 841/*
 842 * This function checks whether a page is free && is the buddy
 843 * we can coalesce a page and its buddy if
 844 * (a) the buddy is not in a hole (check before calling!) &&
 845 * (b) the buddy is in the buddy system &&
 846 * (c) a page and its buddy have the same order &&
 847 * (d) a page and its buddy are in the same zone.
 848 *
 849 * For recording whether a page is in the buddy system, we set PageBuddy.
 850 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
 851 *
 852 * For recording page's order, we use page_private(page).
 853 */
 854static inline bool page_is_buddy(struct page *page, struct page *buddy,
 855                                                        unsigned int order)
 856{
 857        if (!page_is_guard(buddy) && !PageBuddy(buddy))
 858                return false;
 859
 860        if (buddy_order(buddy) != order)
 861                return false;
 862
 863        /*
 864         * zone check is done late to avoid uselessly calculating
 865         * zone/node ids for pages that could never merge.
 866         */
 867        if (page_zone_id(page) != page_zone_id(buddy))
 868                return false;
 869
 870        VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
 871
 872        return true;
 873}
 874
 875#ifdef CONFIG_COMPACTION
 876static inline struct capture_control *task_capc(struct zone *zone)
 877{
 878        struct capture_control *capc = current->capture_control;
 879
 880        return unlikely(capc) &&
 881                !(current->flags & PF_KTHREAD) &&
 882                !capc->page &&
 883                capc->cc->zone == zone ? capc : NULL;
 884}
 885
 886static inline bool
 887compaction_capture(struct capture_control *capc, struct page *page,
 888                   int order, int migratetype)
 889{
 890        if (!capc || order != capc->cc->order)
 891                return false;
 892
 893        /* Do not accidentally pollute CMA or isolated regions*/
 894        if (is_migrate_cma(migratetype) ||
 895            is_migrate_isolate(migratetype))
 896                return false;
 897
 898        /*
 899         * Do not let lower order allocations pollute a movable pageblock.
 900         * This might let an unmovable request use a reclaimable pageblock
 901         * and vice-versa but no more than normal fallback logic which can
 902         * have trouble finding a high-order free page.
 903         */
 904        if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
 905                return false;
 906
 907        capc->page = page;
 908        return true;
 909}
 910
 911#else
 912static inline struct capture_control *task_capc(struct zone *zone)
 913{
 914        return NULL;
 915}
 916
 917static inline bool
 918compaction_capture(struct capture_control *capc, struct page *page,
 919                   int order, int migratetype)
 920{
 921        return false;
 922}
 923#endif /* CONFIG_COMPACTION */
 924
 925/* Used for pages not on another list */
 926static inline void add_to_free_list(struct page *page, struct zone *zone,
 927                                    unsigned int order, int migratetype)
 928{
 929        struct free_area *area = &zone->free_area[order];
 930
 931        list_add(&page->lru, &area->free_list[migratetype]);
 932        area->nr_free++;
 933}
 934
 935/* Used for pages not on another list */
 936static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
 937                                         unsigned int order, int migratetype)
 938{
 939        struct free_area *area = &zone->free_area[order];
 940
 941        list_add_tail(&page->lru, &area->free_list[migratetype]);
 942        area->nr_free++;
 943}
 944
 945/*
 946 * Used for pages which are on another list. Move the pages to the tail
 947 * of the list - so the moved pages won't immediately be considered for
 948 * allocation again (e.g., optimization for memory onlining).
 949 */
 950static inline void move_to_free_list(struct page *page, struct zone *zone,
 951                                     unsigned int order, int migratetype)
 952{
 953        struct free_area *area = &zone->free_area[order];
 954
 955        list_move_tail(&page->lru, &area->free_list[migratetype]);
 956}
 957
 958static inline void del_page_from_free_list(struct page *page, struct zone *zone,
 959                                           unsigned int order)
 960{
 961        /* clear reported state and update reported page count */
 962        if (page_reported(page))
 963                __ClearPageReported(page);
 964
 965        list_del(&page->lru);
 966        __ClearPageBuddy(page);
 967        set_page_private(page, 0);
 968        zone->free_area[order].nr_free--;
 969}
 970
 971/*
 972 * If this is not the largest possible page, check if the buddy
 973 * of the next-highest order is free. If it is, it's possible
 974 * that pages are being freed that will coalesce soon. In case,
 975 * that is happening, add the free page to the tail of the list
 976 * so it's less likely to be used soon and more likely to be merged
 977 * as a higher order page
 978 */
 979static inline bool
 980buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
 981                   struct page *page, unsigned int order)
 982{
 983        struct page *higher_page, *higher_buddy;
 984        unsigned long combined_pfn;
 985
 986        if (order >= MAX_ORDER - 2)
 987                return false;
 988
 989        if (!pfn_valid_within(buddy_pfn))
 990                return false;
 991
 992        combined_pfn = buddy_pfn & pfn;
 993        higher_page = page + (combined_pfn - pfn);
 994        buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
 995        higher_buddy = higher_page + (buddy_pfn - combined_pfn);
 996
 997        return pfn_valid_within(buddy_pfn) &&
 998               page_is_buddy(higher_page, higher_buddy, order + 1);
 999}
1000
1001/*
1002 * Freeing function for a buddy system allocator.
1003 *
1004 * The concept of a buddy system is to maintain direct-mapped table
1005 * (containing bit values) for memory blocks of various "orders".
1006 * The bottom level table contains the map for the smallest allocatable
1007 * units of memory (here, pages), and each level above it describes
1008 * pairs of units from the levels below, hence, "buddies".
1009 * At a high level, all that happens here is marking the table entry
1010 * at the bottom level available, and propagating the changes upward
1011 * as necessary, plus some accounting needed to play nicely with other
1012 * parts of the VM system.
1013 * At each level, we keep a list of pages, which are heads of continuous
1014 * free pages of length of (1 << order) and marked with PageBuddy.
1015 * Page's order is recorded in page_private(page) field.
1016 * So when we are allocating or freeing one, we can derive the state of the
1017 * other.  That is, if we allocate a small block, and both were
1018 * free, the remainder of the region must be split into blocks.
1019 * If a block is freed, and its buddy is also free, then this
1020 * triggers coalescing into a block of larger size.
1021 *
1022 * -- nyc
1023 */
1024
1025static inline void __free_one_page(struct page *page,
1026                unsigned long pfn,
1027                struct zone *zone, unsigned int order,
1028                int migratetype, fpi_t fpi_flags)
1029{
1030        struct capture_control *capc = task_capc(zone);
1031        unsigned long buddy_pfn;
1032        unsigned long combined_pfn;
1033        unsigned int max_order;
1034        struct page *buddy;
1035        bool to_tail;
1036
1037        max_order = min_t(unsigned int, MAX_ORDER - 1, pageblock_order);
1038
1039        VM_BUG_ON(!zone_is_initialized(zone));
1040        VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
1041
1042        VM_BUG_ON(migratetype == -1);
1043        if (likely(!is_migrate_isolate(migratetype)))
1044                __mod_zone_freepage_state(zone, 1 << order, migratetype);
1045
1046        VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
1047        VM_BUG_ON_PAGE(bad_range(zone, page), page);
1048
1049continue_merging:
1050        while (order < max_order) {
1051                if (compaction_capture(capc, page, order, migratetype)) {
1052                        __mod_zone_freepage_state(zone, -(1 << order),
1053                                                                migratetype);
1054                        return;
1055                }
1056                buddy_pfn = __find_buddy_pfn(pfn, order);
1057                buddy = page + (buddy_pfn - pfn);
1058
1059                if (!pfn_valid_within(buddy_pfn))
1060                        goto done_merging;
1061                if (!page_is_buddy(page, buddy, order))
1062                        goto done_merging;
1063                /*
1064                 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
1065                 * merge with it and move up one order.
1066                 */
1067                if (page_is_guard(buddy))
1068                        clear_page_guard(zone, buddy, order, migratetype);
1069                else
1070                        del_page_from_free_list(buddy, zone, order);
1071                combined_pfn = buddy_pfn & pfn;
1072                page = page + (combined_pfn - pfn);
1073                pfn = combined_pfn;
1074                order++;
1075        }
1076        if (order < MAX_ORDER - 1) {
1077                /* If we are here, it means order is >= pageblock_order.
1078                 * We want to prevent merge between freepages on isolate
1079                 * pageblock and normal pageblock. Without this, pageblock
1080                 * isolation could cause incorrect freepage or CMA accounting.
1081                 *
1082                 * We don't want to hit this code for the more frequent
1083                 * low-order merging.
1084                 */
1085                if (unlikely(has_isolate_pageblock(zone))) {
1086                        int buddy_mt;
1087
1088                        buddy_pfn = __find_buddy_pfn(pfn, order);
1089                        buddy = page + (buddy_pfn - pfn);
1090                        buddy_mt = get_pageblock_migratetype(buddy);
1091
1092                        if (migratetype != buddy_mt
1093                                        && (is_migrate_isolate(migratetype) ||
1094                                                is_migrate_isolate(buddy_mt)))
1095                                goto done_merging;
1096                }
1097                max_order = order + 1;
1098                goto continue_merging;
1099        }
1100
1101done_merging:
1102        set_buddy_order(page, order);
1103
1104        if (fpi_flags & FPI_TO_TAIL)
1105                to_tail = true;
1106        else if (is_shuffle_order(order))
1107                to_tail = shuffle_pick_tail();
1108        else
1109                to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
1110
1111        if (to_tail)
1112                add_to_free_list_tail(page, zone, order, migratetype);
1113        else
1114                add_to_free_list(page, zone, order, migratetype);
1115
1116        /* Notify page reporting subsystem of freed page */
1117        if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
1118                page_reporting_notify_free(order);
1119}
1120
1121/*
1122 * A bad page could be due to a number of fields. Instead of multiple branches,
1123 * try and check multiple fields with one check. The caller must do a detailed
1124 * check if necessary.
1125 */
1126static inline bool page_expected_state(struct page *page,
1127                                        unsigned long check_flags)
1128{
1129        if (unlikely(atomic_read(&page->_mapcount) != -1))
1130                return false;
1131
1132        if (unlikely((unsigned long)page->mapping |
1133                        page_ref_count(page) |
1134#ifdef CONFIG_MEMCG
1135                        page->memcg_data |
1136#endif
1137                        (page->flags & check_flags)))
1138                return false;
1139
1140        return true;
1141}
1142
1143static const char *page_bad_reason(struct page *page, unsigned long flags)
1144{
1145        const char *bad_reason = NULL;
1146
1147        if (unlikely(atomic_read(&page->_mapcount) != -1))
1148                bad_reason = "nonzero mapcount";
1149        if (unlikely(page->mapping != NULL))
1150                bad_reason = "non-NULL mapping";
1151        if (unlikely(page_ref_count(page) != 0))
1152                bad_reason = "nonzero _refcount";
1153        if (unlikely(page->flags & flags)) {
1154                if (flags == PAGE_FLAGS_CHECK_AT_PREP)
1155                        bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
1156                else
1157                        bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
1158        }
1159#ifdef CONFIG_MEMCG
1160        if (unlikely(page->memcg_data))
1161                bad_reason = "page still charged to cgroup";
1162#endif
1163        return bad_reason;
1164}
1165
1166static void check_free_page_bad(struct page *page)
1167{
1168        bad_page(page,
1169                 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
1170}
1171
1172static inline int check_free_page(struct page *page)
1173{
1174        if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
1175                return 0;
1176
1177        /* Something has gone sideways, find it */
1178        check_free_page_bad(page);
1179        return 1;
1180}
1181
1182static int free_tail_pages_check(struct page *head_page, struct page *page)
1183{
1184        int ret = 1;
1185
1186        /*
1187         * We rely page->lru.next never has bit 0 set, unless the page
1188         * is PageTail(). Let's make sure that's true even for poisoned ->lru.
1189         */
1190        BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1191
1192        if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1193                ret = 0;
1194                goto out;
1195        }
1196        switch (page - head_page) {
1197        case 1:
1198                /* the first tail page: ->mapping may be compound_mapcount() */
1199                if (unlikely(compound_mapcount(page))) {
1200                        bad_page(page, "nonzero compound_mapcount");
1201                        goto out;
1202                }
1203                break;
1204        case 2:
1205                /*
1206                 * the second tail page: ->mapping is
1207                 * deferred_list.next -- ignore value.
1208                 */
1209                break;
1210        default:
1211                if (page->mapping != TAIL_MAPPING) {
1212                        bad_page(page, "corrupted mapping in tail page");
1213                        goto out;
1214                }
1215                break;
1216        }
1217        if (unlikely(!PageTail(page))) {
1218                bad_page(page, "PageTail not set");
1219                goto out;
1220        }
1221        if (unlikely(compound_head(page) != head_page)) {
1222                bad_page(page, "compound_head not consistent");
1223                goto out;
1224        }
1225        ret = 0;
1226out:
1227        page->mapping = NULL;
1228        clear_compound_head(page);
1229        return ret;
1230}
1231
1232static void kernel_init_free_pages(struct page *page, int numpages)
1233{
1234        int i;
1235
1236        /* s390's use of memset() could override KASAN redzones. */
1237        kasan_disable_current();
1238        for (i = 0; i < numpages; i++) {
1239                u8 tag = page_kasan_tag(page + i);
1240                page_kasan_tag_reset(page + i);
1241                clear_highpage(page + i);
1242                page_kasan_tag_set(page + i, tag);
1243        }
1244        kasan_enable_current();
1245}
1246
1247static __always_inline bool free_pages_prepare(struct page *page,
1248                        unsigned int order, bool check_free, fpi_t fpi_flags)
1249{
1250        int bad = 0;
1251        bool init;
1252
1253        VM_BUG_ON_PAGE(PageTail(page), page);
1254
1255        trace_mm_page_free(page, order);
1256
1257        if (unlikely(PageHWPoison(page)) && !order) {
1258                /*
1259                 * Do not let hwpoison pages hit pcplists/buddy
1260                 * Untie memcg state and reset page's owner
1261                 */
1262                if (memcg_kmem_enabled() && PageMemcgKmem(page))
1263                        __memcg_kmem_uncharge_page(page, order);
1264                reset_page_owner(page, order);
1265                return false;
1266        }
1267
1268        /*
1269         * Check tail pages before head page information is cleared to
1270         * avoid checking PageCompound for order-0 pages.
1271         */
1272        if (unlikely(order)) {
1273                bool compound = PageCompound(page);
1274                int i;
1275
1276                VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1277
1278                if (compound)
1279                        ClearPageDoubleMap(page);
1280                for (i = 1; i < (1 << order); i++) {
1281                        if (compound)
1282                                bad += free_tail_pages_check(page, page + i);
1283                        if (unlikely(check_free_page(page + i))) {
1284                                bad++;
1285                                continue;
1286                        }
1287                        (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1288                }
1289        }
1290        if (PageMappingFlags(page))
1291                page->mapping = NULL;
1292        if (memcg_kmem_enabled() && PageMemcgKmem(page))
1293                __memcg_kmem_uncharge_page(page, order);
1294        if (check_free)
1295                bad += check_free_page(page);
1296        if (bad)
1297                return false;
1298
1299        page_cpupid_reset_last(page);
1300        page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1301        reset_page_owner(page, order);
1302
1303        if (!PageHighMem(page)) {
1304                debug_check_no_locks_freed(page_address(page),
1305                                           PAGE_SIZE << order);
1306                debug_check_no_obj_freed(page_address(page),
1307                                           PAGE_SIZE << order);
1308        }
1309
1310        kernel_poison_pages(page, 1 << order);
1311
1312        /*
1313         * As memory initialization might be integrated into KASAN,
1314         * kasan_free_pages and kernel_init_free_pages must be
1315         * kept together to avoid discrepancies in behavior.
1316         *
1317         * With hardware tag-based KASAN, memory tags must be set before the
1318         * page becomes unavailable via debug_pagealloc or arch_free_page.
1319         */
1320        init = want_init_on_free();
1321        if (init && !kasan_has_integrated_init())
1322                kernel_init_free_pages(page, 1 << order);
1323        kasan_free_nondeferred_pages(page, order, init, fpi_flags);
1324
1325        /*
1326         * arch_free_page() can make the page's contents inaccessible.  s390
1327         * does this.  So nothing which can access the page's contents should
1328         * happen after this.
1329         */
1330        arch_free_page(page, order);
1331
1332        debug_pagealloc_unmap_pages(page, 1 << order);
1333
1334        return true;
1335}
1336
1337#ifdef CONFIG_DEBUG_VM
1338/*
1339 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed
1340 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when
1341 * moved from pcp lists to free lists.
1342 */
1343static bool free_pcp_prepare(struct page *page)
1344{
1345        return free_pages_prepare(page, 0, true, FPI_NONE);
1346}
1347
1348static bool bulkfree_pcp_prepare(struct page *page)
1349{
1350        if (debug_pagealloc_enabled_static())
1351                return check_free_page(page);
1352        else
1353                return false;
1354}
1355#else
1356/*
1357 * With DEBUG_VM disabled, order-0 pages being freed are checked only when
1358 * moving from pcp lists to free list in order to reduce overhead. With
1359 * debug_pagealloc enabled, they are checked also immediately when being freed
1360 * to the pcp lists.
1361 */
1362static bool free_pcp_prepare(struct page *page)
1363{
1364        if (debug_pagealloc_enabled_static())
1365                return free_pages_prepare(page, 0, true, FPI_NONE);
1366        else
1367                return free_pages_prepare(page, 0, false, FPI_NONE);
1368}
1369
1370static bool bulkfree_pcp_prepare(struct page *page)
1371{
1372        return check_free_page(page);
1373}
1374#endif /* CONFIG_DEBUG_VM */
1375
1376static inline void prefetch_buddy(struct page *page)
1377{
1378        unsigned long pfn = page_to_pfn(page);
1379        unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
1380        struct page *buddy = page + (buddy_pfn - pfn);
1381
1382        prefetch(buddy);
1383}
1384
1385/*
1386 * Frees a number of pages from the PCP lists
1387 * Assumes all pages on list are in same zone, and of same order.
1388 * count is the number of pages to free.
1389 *
1390 * If the zone was previously in an "all pages pinned" state then look to
1391 * see if this freeing clears that state.
1392 *
1393 * And clear the zone's pages_scanned counter, to hold off the "all pages are
1394 * pinned" detection logic.
1395 */
1396static void free_pcppages_bulk(struct zone *zone, int count,
1397                                        struct per_cpu_pages *pcp)
1398{
1399        int migratetype = 0;
1400        int batch_free = 0;
1401        int prefetch_nr = READ_ONCE(pcp->batch);
1402        bool isolated_pageblocks;
1403        struct page *page, *tmp;
1404        LIST_HEAD(head);
1405
1406        /*
1407         * Ensure proper count is passed which otherwise would stuck in the
1408         * below while (list_empty(list)) loop.
1409         */
1410        count = min(pcp->count, count);
1411        while (count) {
1412                struct list_head *list;
1413
1414                /*
1415                 * Remove pages from lists in a round-robin fashion. A
1416                 * batch_free count is maintained that is incremented when an
1417                 * empty list is encountered.  This is so more pages are freed
1418                 * off fuller lists instead of spinning excessively around empty
1419                 * lists
1420                 */
1421                do {
1422                        batch_free++;
1423                        if (++migratetype == MIGRATE_PCPTYPES)
1424                                migratetype = 0;
1425                        list = &pcp->lists[migratetype];
1426                } while (list_empty(list));
1427
1428                /* This is the only non-empty list. Free them all. */
1429                if (batch_free == MIGRATE_PCPTYPES)
1430                        batch_free = count;
1431
1432                do {
1433                        page = list_last_entry(list, struct page, lru);
1434                        /* must delete to avoid corrupting pcp list */
1435                        list_del(&page->lru);
1436                        pcp->count--;
1437
1438                        if (bulkfree_pcp_prepare(page))
1439                                continue;
1440
1441                        list_add_tail(&page->lru, &head);
1442
1443                        /*
1444                         * We are going to put the page back to the global
1445                         * pool, prefetch its buddy to speed up later access
1446                         * under zone->lock. It is believed the overhead of
1447                         * an additional test and calculating buddy_pfn here
1448                         * can be offset by reduced memory latency later. To
1449                         * avoid excessive prefetching due to large count, only
1450                         * prefetch buddy for the first pcp->batch nr of pages.
1451                         */
1452                        if (prefetch_nr) {
1453                                prefetch_buddy(page);
1454                                prefetch_nr--;
1455                        }
1456                } while (--count && --batch_free && !list_empty(list));
1457        }
1458
1459        spin_lock(&zone->lock);
1460        isolated_pageblocks = has_isolate_pageblock(zone);
1461
1462        /*
1463         * Use safe version since after __free_one_page(),
1464         * page->lru.next will not point to original list.
1465         */
1466        list_for_each_entry_safe(page, tmp, &head, lru) {
1467                int mt = get_pcppage_migratetype(page);
1468                /* MIGRATE_ISOLATE page should not go to pcplists */
1469                VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1470                /* Pageblock could have been isolated meanwhile */
1471                if (unlikely(isolated_pageblocks))
1472                        mt = get_pageblock_migratetype(page);
1473
1474                __free_one_page(page, page_to_pfn(page), zone, 0, mt, FPI_NONE);
1475                trace_mm_page_pcpu_drain(page, 0, mt);
1476        }
1477        spin_unlock(&zone->lock);
1478}
1479
1480static void free_one_page(struct zone *zone,
1481                                struct page *page, unsigned long pfn,
1482                                unsigned int order,
1483                                int migratetype, fpi_t fpi_flags)
1484{
1485        spin_lock(&zone->lock);
1486        if (unlikely(has_isolate_pageblock(zone) ||
1487                is_migrate_isolate(migratetype))) {
1488                migratetype = get_pfnblock_migratetype(page, pfn);
1489        }
1490        __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
1491        spin_unlock(&zone->lock);
1492}
1493
1494static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1495                                unsigned long zone, int nid)
1496{
1497        mm_zero_struct_page(page);
1498        set_page_links(page, zone, nid, pfn);
1499        init_page_count(page);
1500        page_mapcount_reset(page);
1501        page_cpupid_reset_last(page);
1502        page_kasan_tag_reset(page);
1503
1504        INIT_LIST_HEAD(&page->lru);
1505#ifdef WANT_PAGE_VIRTUAL
1506        /* The shift won't overflow because ZONE_NORMAL is below 4G. */
1507        if (!is_highmem_idx(zone))
1508                set_page_address(page, __va(pfn << PAGE_SHIFT));
1509#endif
1510}
1511
1512#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1513static void __meminit init_reserved_page(unsigned long pfn)
1514{
1515        pg_data_t *pgdat;
1516        int nid, zid;
1517
1518        if (!early_page_uninitialised(pfn))
1519                return;
1520
1521        nid = early_pfn_to_nid(pfn);
1522        pgdat = NODE_DATA(nid);
1523
1524        for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1525                struct zone *zone = &pgdat->node_zones[zid];
1526
1527                if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1528                        break;
1529        }
1530        __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
1531}
1532#else
1533static inline void init_reserved_page(unsigned long pfn)
1534{
1535}
1536#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1537
1538/*
1539 * Initialised pages do not have PageReserved set. This function is
1540 * called for each range allocated by the bootmem allocator and
1541 * marks the pages PageReserved. The remaining valid pages are later
1542 * sent to the buddy page allocator.
1543 */
1544void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1545{
1546        unsigned long start_pfn = PFN_DOWN(start);
1547        unsigned long end_pfn = PFN_UP(end);
1548
1549        for (; start_pfn < end_pfn; start_pfn++) {
1550                if (pfn_valid(start_pfn)) {
1551                        struct page *page = pfn_to_page(start_pfn);
1552
1553                        init_reserved_page(start_pfn);
1554
1555                        /* Avoid false-positive PageTail() */
1556                        INIT_LIST_HEAD(&page->lru);
1557
1558                        /*
1559                         * no need for atomic set_bit because the struct
1560                         * page is not visible yet so nobody should
1561                         * access it yet.
1562                         */
1563                        __SetPageReserved(page);
1564                }
1565        }
1566}
1567
1568static void __free_pages_ok(struct page *page, unsigned int order,
1569                            fpi_t fpi_flags)
1570{
1571        unsigned long flags;
1572        int migratetype;
1573        unsigned long pfn = page_to_pfn(page);
1574
1575        if (!free_pages_prepare(page, order, true, fpi_flags))
1576                return;
1577
1578        migratetype = get_pfnblock_migratetype(page, pfn);
1579        local_irq_save(flags);
1580        __count_vm_events(PGFREE, 1 << order);
1581        free_one_page(page_zone(page), page, pfn, order, migratetype,
1582                      fpi_flags);
1583        local_irq_restore(flags);
1584}
1585
1586void __free_pages_core(struct page *page, unsigned int order)
1587{
1588        unsigned int nr_pages = 1 << order;
1589        struct page *p = page;
1590        unsigned int loop;
1591
1592        /*
1593         * When initializing the memmap, __init_single_page() sets the refcount
1594         * of all pages to 1 ("allocated"/"not free"). We have to set the
1595         * refcount of all involved pages to 0.
1596         */
1597        prefetchw(p);
1598        for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1599                prefetchw(p + 1);
1600                __ClearPageReserved(p);
1601                set_page_count(p, 0);
1602        }
1603        __ClearPageReserved(p);
1604        set_page_count(p, 0);
1605
1606        atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
1607
1608        /*
1609         * Bypass PCP and place fresh pages right to the tail, primarily
1610         * relevant for memory onlining.
1611         */
1612        __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON);
1613}
1614
1615#ifdef CONFIG_NEED_MULTIPLE_NODES
1616
1617/*
1618 * During memory init memblocks map pfns to nids. The search is expensive and
1619 * this caches recent lookups. The implementation of __early_pfn_to_nid
1620 * treats start/end as pfns.
1621 */
1622struct mminit_pfnnid_cache {
1623        unsigned long last_start;
1624        unsigned long last_end;
1625        int last_nid;
1626};
1627
1628static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1629
1630/*
1631 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
1632 */
1633static int __meminit __early_pfn_to_nid(unsigned long pfn,
1634                                        struct mminit_pfnnid_cache *state)
1635{
1636        unsigned long start_pfn, end_pfn;
1637        int nid;
1638
1639        if (state->last_start <= pfn && pfn < state->last_end)
1640                return state->last_nid;
1641
1642        nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
1643        if (nid != NUMA_NO_NODE) {
1644                state->last_start = start_pfn;
1645                state->last_end = end_pfn;
1646                state->last_nid = nid;
1647        }
1648
1649        return nid;
1650}
1651
1652int __meminit early_pfn_to_nid(unsigned long pfn)
1653{
1654        static DEFINE_SPINLOCK(early_pfn_lock);
1655        int nid;
1656
1657        spin_lock(&early_pfn_lock);
1658        nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1659        if (nid < 0)
1660                nid = first_online_node;
1661        spin_unlock(&early_pfn_lock);
1662
1663        return nid;
1664}
1665#endif /* CONFIG_NEED_MULTIPLE_NODES */
1666
1667void __init memblock_free_pages(struct page *page, unsigned long pfn,
1668                                                        unsigned int order)
1669{
1670        if (early_page_uninitialised(pfn))
1671                return;
1672        __free_pages_core(page, order);
1673}
1674
1675/*
1676 * Check that the whole (or subset of) a pageblock given by the interval of
1677 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1678 * with the migration of free compaction scanner. The scanners then need to
1679 * use only pfn_valid_within() check for arches that allow holes within
1680 * pageblocks.
1681 *
1682 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1683 *
1684 * It's possible on some configurations to have a setup like node0 node1 node0
1685 * i.e. it's possible that all pages within a zones range of pages do not
1686 * belong to a single zone. We assume that a border between node0 and node1
1687 * can occur within a single pageblock, but not a node0 node1 node0
1688 * interleaving within a single pageblock. It is therefore sufficient to check
1689 * the first and last page of a pageblock and avoid checking each individual
1690 * page in a pageblock.
1691 */
1692struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1693                                     unsigned long end_pfn, struct zone *zone)
1694{
1695        struct page *start_page;
1696        struct page *end_page;
1697
1698        /* end_pfn is one past the range we are checking */
1699        end_pfn--;
1700
1701        if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1702                return NULL;
1703
1704        start_page = pfn_to_online_page(start_pfn);
1705        if (!start_page)
1706                return NULL;
1707
1708        if (page_zone(start_page) != zone)
1709                return NULL;
1710
1711        end_page = pfn_to_page(end_pfn);
1712
1713        /* This gives a shorter code than deriving page_zone(end_page) */
1714        if (page_zone_id(start_page) != page_zone_id(end_page))
1715                return NULL;
1716
1717        return start_page;
1718}
1719
1720void set_zone_contiguous(struct zone *zone)
1721{
1722        unsigned long block_start_pfn = zone->zone_start_pfn;
1723        unsigned long block_end_pfn;
1724
1725        block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1726        for (; block_start_pfn < zone_end_pfn(zone);
1727                        block_start_pfn = block_end_pfn,
1728                         block_end_pfn += pageblock_nr_pages) {
1729
1730                block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1731
1732                if (!__pageblock_pfn_to_page(block_start_pfn,
1733                                             block_end_pfn, zone))
1734                        return;
1735                cond_resched();
1736        }
1737
1738        /* We confirm that there is no hole */
1739        zone->contiguous = true;
1740}
1741
1742void clear_zone_contiguous(struct zone *zone)
1743{
1744        zone->contiguous = false;
1745}
1746
1747#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1748static void __init deferred_free_range(unsigned long pfn,
1749                                       unsigned long nr_pages)
1750{
1751        struct page *page;
1752        unsigned long i;
1753
1754        if (!nr_pages)
1755                return;
1756
1757        page = pfn_to_page(pfn);
1758
1759        /* Free a large naturally-aligned chunk if possible */
1760        if (nr_pages == pageblock_nr_pages &&
1761            (pfn & (pageblock_nr_pages - 1)) == 0) {
1762                set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1763                __free_pages_core(page, pageblock_order);
1764                return;
1765        }
1766
1767        for (i = 0; i < nr_pages; i++, page++, pfn++) {
1768                if ((pfn & (pageblock_nr_pages - 1)) == 0)
1769                        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1770                __free_pages_core(page, 0);
1771        }
1772}
1773
1774/* Completion tracking for deferred_init_memmap() threads */
1775static atomic_t pgdat_init_n_undone __initdata;
1776static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1777
1778static inline void __init pgdat_init_report_one_done(void)
1779{
1780        if (atomic_dec_and_test(&pgdat_init_n_undone))
1781                complete(&pgdat_init_all_done_comp);
1782}
1783
1784/*
1785 * Returns true if page needs to be initialized or freed to buddy allocator.
1786 *
1787 * First we check if pfn is valid on architectures where it is possible to have
1788 * holes within pageblock_nr_pages. On systems where it is not possible, this
1789 * function is optimized out.
1790 *
1791 * Then, we check if a current large page is valid by only checking the validity
1792 * of the head pfn.
1793 */
1794static inline bool __init deferred_pfn_valid(unsigned long pfn)
1795{
1796        if (!pfn_valid_within(pfn))
1797                return false;
1798        if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
1799                return false;
1800        return true;
1801}
1802
1803/*
1804 * Free pages to buddy allocator. Try to free aligned pages in
1805 * pageblock_nr_pages sizes.
1806 */
1807static void __init deferred_free_pages(unsigned long pfn,
1808                                       unsigned long end_pfn)
1809{
1810        unsigned long nr_pgmask = pageblock_nr_pages - 1;
1811        unsigned long nr_free = 0;
1812
1813        for (; pfn < end_pfn; pfn++) {
1814                if (!deferred_pfn_valid(pfn)) {
1815                        deferred_free_range(pfn - nr_free, nr_free);
1816                        nr_free = 0;
1817                } else if (!(pfn & nr_pgmask)) {
1818                        deferred_free_range(pfn - nr_free, nr_free);
1819                        nr_free = 1;
1820                } else {
1821                        nr_free++;
1822                }
1823        }
1824        /* Free the last block of pages to allocator */
1825        deferred_free_range(pfn - nr_free, nr_free);
1826}
1827
1828/*
1829 * Initialize struct pages.  We minimize pfn page lookups and scheduler checks
1830 * by performing it only once every pageblock_nr_pages.
1831 * Return number of pages initialized.
1832 */
1833static unsigned long  __init deferred_init_pages(struct zone *zone,
1834                                                 unsigned long pfn,
1835                                                 unsigned long end_pfn)
1836{
1837        unsigned long nr_pgmask = pageblock_nr_pages - 1;
1838        int nid = zone_to_nid(zone);
1839        unsigned long nr_pages = 0;
1840        int zid = zone_idx(zone);
1841        struct page *page = NULL;
1842
1843        for (; pfn < end_pfn; pfn++) {
1844                if (!deferred_pfn_valid(pfn)) {
1845                        page = NULL;
1846                        continue;
1847                } else if (!page || !(pfn & nr_pgmask)) {
1848                        page = pfn_to_page(pfn);
1849                } else {
1850                        page++;
1851                }
1852                __init_single_page(page, pfn, zid, nid);
1853                nr_pages++;
1854        }
1855        return (nr_pages);
1856}
1857
1858/*
1859 * This function is meant to pre-load the iterator for the zone init.
1860 * Specifically it walks through the ranges until we are caught up to the
1861 * first_init_pfn value and exits there. If we never encounter the value we
1862 * return false indicating there are no valid ranges left.
1863 */
1864static bool __init
1865deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
1866                                    unsigned long *spfn, unsigned long *epfn,
1867                                    unsigned long first_init_pfn)
1868{
1869        u64 j;
1870
1871        /*
1872         * Start out by walking through the ranges in this zone that have
1873         * already been initialized. We don't need to do anything with them
1874         * so we just need to flush them out of the system.
1875         */
1876        for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
1877                if (*epfn <= first_init_pfn)
1878                        continue;
1879                if (*spfn < first_init_pfn)
1880                        *spfn = first_init_pfn;
1881                *i = j;
1882                return true;
1883        }
1884
1885        return false;
1886}
1887
1888/*
1889 * Initialize and free pages. We do it in two loops: first we initialize
1890 * struct page, then free to buddy allocator, because while we are
1891 * freeing pages we can access pages that are ahead (computing buddy
1892 * page in __free_one_page()).
1893 *
1894 * In order to try and keep some memory in the cache we have the loop
1895 * broken along max page order boundaries. This way we will not cause
1896 * any issues with the buddy page computation.
1897 */
1898static unsigned long __init
1899deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
1900                       unsigned long *end_pfn)
1901{
1902        unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
1903        unsigned long spfn = *start_pfn, epfn = *end_pfn;
1904        unsigned long nr_pages = 0;
1905        u64 j = *i;
1906
1907        /* First we loop through and initialize the page values */
1908        for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
1909                unsigned long t;
1910
1911                if (mo_pfn <= *start_pfn)
1912                        break;
1913
1914                t = min(mo_pfn, *end_pfn);
1915                nr_pages += deferred_init_pages(zone, *start_pfn, t);
1916
1917                if (mo_pfn < *end_pfn) {
1918                        *start_pfn = mo_pfn;
1919                        break;
1920                }
1921        }
1922
1923        /* Reset values and now loop through freeing pages as needed */
1924        swap(j, *i);
1925
1926        for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
1927                unsigned long t;
1928
1929                if (mo_pfn <= spfn)
1930                        break;
1931
1932                t = min(mo_pfn, epfn);
1933                deferred_free_pages(spfn, t);
1934
1935                if (mo_pfn <= epfn)
1936                        break;
1937        }
1938
1939        return nr_pages;
1940}
1941
1942static void __init
1943deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
1944                           void *arg)
1945{
1946        unsigned long spfn, epfn;
1947        struct zone *zone = arg;
1948        u64 i;
1949
1950        deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
1951
1952        /*
1953         * Initialize and free pages in MAX_ORDER sized increments so that we
1954         * can avoid introducing any issues with the buddy allocator.
1955         */
1956        while (spfn < end_pfn) {
1957                deferred_init_maxorder(&i, zone, &spfn, &epfn);
1958                cond_resched();
1959        }
1960}
1961
1962/* An arch may override for more concurrency. */
1963__weak int __init
1964deferred_page_init_max_threads(const struct cpumask *node_cpumask)
1965{
1966        return 1;
1967}
1968
1969/* Initialise remaining memory on a node */
1970static int __init deferred_init_memmap(void *data)
1971{
1972        pg_data_t *pgdat = data;
1973        const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1974        unsigned long spfn = 0, epfn = 0;
1975        unsigned long first_init_pfn, flags;
1976        unsigned long start = jiffies;
1977        struct zone *zone;
1978        int zid, max_threads;
1979        u64 i;
1980
1981        /* Bind memory initialisation thread to a local node if possible */
1982        if (!cpumask_empty(cpumask))
1983                set_cpus_allowed_ptr(current, cpumask);
1984
1985        pgdat_resize_lock(pgdat, &flags);
1986        first_init_pfn = pgdat->first_deferred_pfn;
1987        if (first_init_pfn == ULONG_MAX) {
1988                pgdat_resize_unlock(pgdat, &flags);
1989                pgdat_init_report_one_done();
1990                return 0;
1991        }
1992
1993        /* Sanity check boundaries */
1994        BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1995        BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1996        pgdat->first_deferred_pfn = ULONG_MAX;
1997
1998        /*
1999         * Once we unlock here, the zone cannot be grown anymore, thus if an
2000         * interrupt thread must allocate this early in boot, zone must be
2001         * pre-grown prior to start of deferred page initialization.
2002         */
2003        pgdat_resize_unlock(pgdat, &flags);
2004
2005        /* Only the highest zone is deferred so find it */
2006        for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2007                zone = pgdat->node_zones + zid;
2008                if (first_init_pfn < zone_end_pfn(zone))
2009                        break;
2010        }
2011
2012        /* If the zone is empty somebody else may have cleared out the zone */
2013        if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2014                                                 first_init_pfn))
2015                goto zone_empty;
2016
2017        max_threads = deferred_page_init_max_threads(cpumask);
2018
2019        while (spfn < epfn) {
2020                unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
2021                struct padata_mt_job job = {
2022                        .thread_fn   = deferred_init_memmap_chunk,
2023                        .fn_arg      = zone,
2024                        .start       = spfn,
2025                        .size        = epfn_align - spfn,
2026                        .align       = PAGES_PER_SECTION,
2027                        .min_chunk   = PAGES_PER_SECTION,
2028                        .max_threads = max_threads,
2029                };
2030
2031                padata_do_multithreaded(&job);
2032                deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2033                                                    epfn_align);
2034        }
2035zone_empty:
2036        /* Sanity check that the next zone really is unpopulated */
2037        WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
2038
2039        pr_info("node %d deferred pages initialised in %ums\n",
2040                pgdat->node_id, jiffies_to_msecs(jiffies - start));
2041
2042        pgdat_init_report_one_done();
2043        return 0;
2044}
2045
2046/*
2047 * If this zone has deferred pages, try to grow it by initializing enough
2048 * deferred pages to satisfy the allocation specified by order, rounded up to
2049 * the nearest PAGES_PER_SECTION boundary.  So we're adding memory in increments
2050 * of SECTION_SIZE bytes by initializing struct pages in increments of
2051 * PAGES_PER_SECTION * sizeof(struct page) bytes.
2052 *
2053 * Return true when zone was grown, otherwise return false. We return true even
2054 * when we grow less than requested, to let the caller decide if there are
2055 * enough pages to satisfy the allocation.
2056 *
2057 * Note: We use noinline because this function is needed only during boot, and
2058 * it is called from a __ref function _deferred_grow_zone. This way we are
2059 * making sure that it is not inlined into permanent text section.
2060 */
2061static noinline bool __init
2062deferred_grow_zone(struct zone *zone, unsigned int order)
2063{
2064        unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
2065        pg_data_t *pgdat = zone->zone_pgdat;
2066        unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
2067        unsigned long spfn, epfn, flags;
2068        unsigned long nr_pages = 0;
2069        u64 i;
2070
2071        /* Only the last zone may have deferred pages */
2072        if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
2073                return false;
2074
2075        pgdat_resize_lock(pgdat, &flags);
2076
2077        /*
2078         * If someone grew this zone while we were waiting for spinlock, return
2079         * true, as there might be enough pages already.
2080         */
2081        if (first_deferred_pfn != pgdat->first_deferred_pfn) {
2082                pgdat_resize_unlock(pgdat, &flags);
2083                return true;
2084        }
2085
2086        /* If the zone is empty somebody else may have cleared out the zone */
2087        if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2088                                                 first_deferred_pfn)) {
2089                pgdat->first_deferred_pfn = ULONG_MAX;
2090                pgdat_resize_unlock(pgdat, &flags);
2091                /* Retry only once. */
2092                return first_deferred_pfn != ULONG_MAX;
2093        }
2094
2095        /*
2096         * Initialize and free pages in MAX_ORDER sized increments so
2097         * that we can avoid introducing any issues with the buddy
2098         * allocator.
2099         */
2100        while (spfn < epfn) {
2101                /* update our first deferred PFN for this section */
2102                first_deferred_pfn = spfn;
2103
2104                nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
2105                touch_nmi_watchdog();
2106
2107                /* We should only stop along section boundaries */
2108                if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2109                        continue;
2110
2111                /* If our quota has been met we can stop here */
2112                if (nr_pages >= nr_pages_needed)
2113                        break;
2114        }
2115
2116        pgdat->first_deferred_pfn = spfn;
2117        pgdat_resize_unlock(pgdat, &flags);
2118
2119        return nr_pages > 0;
2120}
2121
2122/*
2123 * deferred_grow_zone() is __init, but it is called from
2124 * get_page_from_freelist() during early boot until deferred_pages permanently
2125 * disables this call. This is why we have refdata wrapper to avoid warning,
2126 * and to ensure that the function body gets unloaded.
2127 */
2128static bool __ref
2129_deferred_grow_zone(struct zone *zone, unsigned int order)
2130{
2131        return deferred_grow_zone(zone, order);
2132}
2133
2134#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
2135
2136void __init page_alloc_init_late(void)
2137{
2138        struct zone *zone;
2139        int nid;
2140
2141#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
2142
2143        /* There will be num_node_state(N_MEMORY) threads */
2144        atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
2145        for_each_node_state(nid, N_MEMORY) {
2146                kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
2147        }
2148
2149        /* Block until all are initialised */
2150        wait_for_completion(&pgdat_init_all_done_comp);
2151
2152        /*
2153         * The number of managed pages has changed due to the initialisation
2154         * so the pcpu batch and high limits needs to be updated or the limits
2155         * will be artificially small.
2156         */
2157        for_each_populated_zone(zone)
2158                zone_pcp_update(zone);
2159
2160        /*
2161         * We initialized the rest of the deferred pages.  Permanently disable
2162         * on-demand struct page initialization.
2163         */
2164        static_branch_disable(&deferred_pages);
2165
2166        /* Reinit limits that are based on free pages after the kernel is up */
2167        files_maxfiles_init();
2168#endif
2169
2170        buffer_init();
2171
2172        /* Discard memblock private memory */
2173        memblock_discard();
2174
2175        for_each_node_state(nid, N_MEMORY)
2176                shuffle_free_memory(NODE_DATA(nid));
2177
2178        for_each_populated_zone(zone)
2179                set_zone_contiguous(zone);
2180}
2181
2182#ifdef CONFIG_CMA
2183/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
2184void __init init_cma_reserved_pageblock(struct page *page)
2185{
2186        unsigned i = pageblock_nr_pages;
2187        struct page *p = page;
2188
2189        do {
2190                __ClearPageReserved(p);
2191                set_page_count(p, 0);
2192        } while (++p, --i);
2193
2194        set_pageblock_migratetype(page, MIGRATE_CMA);
2195
2196        if (pageblock_order >= MAX_ORDER) {
2197                i = pageblock_nr_pages;
2198                p = page;
2199                do {
2200                        set_page_refcounted(p);
2201                        __free_pages(p, MAX_ORDER - 1);
2202                        p += MAX_ORDER_NR_PAGES;
2203                } while (i -= MAX_ORDER_NR_PAGES);
2204        } else {
2205                set_page_refcounted(page);
2206                __free_pages(page, pageblock_order);
2207        }
2208
2209        adjust_managed_page_count(page, pageblock_nr_pages);
2210        page_zone(page)->cma_pages += pageblock_nr_pages;
2211}
2212#endif
2213
2214/*
2215 * The order of subdivision here is critical for the IO subsystem.
2216 * Please do not alter this order without good reasons and regression
2217 * testing. Specifically, as large blocks of memory are subdivided,
2218 * the order in which smaller blocks are delivered depends on the order
2219 * they're subdivided in this function. This is the primary factor
2220 * influencing the order in which pages are delivered to the IO
2221 * subsystem according to empirical testing, and this is also justified
2222 * by considering the behavior of a buddy system containing a single
2223 * large block of memory acted on by a series of small allocations.
2224 * This behavior is a critical factor in sglist merging's success.
2225 *
2226 * -- nyc
2227 */
2228static inline void expand(struct zone *zone, struct page *page,
2229        int low, int high, int migratetype)
2230{
2231        unsigned long size = 1 << high;
2232
2233        while (high > low) {
2234                high--;
2235                size >>= 1;
2236                VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
2237
2238                /*
2239                 * Mark as guard pages (or page), that will allow to
2240                 * merge back to allocator when buddy will be freed.
2241                 * Corresponding page table entries will not be touched,
2242                 * pages will stay not present in virtual address space
2243                 */
2244                if (set_page_guard(zone, &page[size], high, migratetype))
2245                        continue;
2246
2247                add_to_free_list(&page[size], zone, high, migratetype);
2248                set_buddy_order(&page[size], high);
2249        }
2250}
2251
2252static void check_new_page_bad(struct page *page)
2253{
2254        if (unlikely(page->flags & __PG_HWPOISON)) {
2255                /* Don't complain about hwpoisoned pages */
2256                page_mapcount_reset(page); /* remove PageBuddy */
2257                return;
2258        }
2259
2260        bad_page(page,
2261                 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
2262}
2263
2264/*
2265 * This page is about to be returned from the page allocator
2266 */
2267static inline int check_new_page(struct page *page)
2268{
2269        if (likely(page_expected_state(page,
2270                                PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
2271                return 0;
2272
2273        check_new_page_bad(page);
2274        return 1;
2275}
2276
2277#ifdef CONFIG_DEBUG_VM
2278/*
2279 * With DEBUG_VM enabled, order-0 pages are checked for expected state when
2280 * being allocated from pcp lists. With debug_pagealloc also enabled, they are
2281 * also checked when pcp lists are refilled from the free lists.
2282 */
2283static inline bool check_pcp_refill(struct page *page)
2284{
2285        if (debug_pagealloc_enabled_static())
2286                return check_new_page(page);
2287        else
2288                return false;
2289}
2290
2291static inline bool check_new_pcp(struct page *page)
2292{
2293        return check_new_page(page);
2294}
2295#else
2296/*
2297 * With DEBUG_VM disabled, free order-0 pages are checked for expected state
2298 * when pcp lists are being refilled from the free lists. With debug_pagealloc
2299 * enabled, they are also checked when being allocated from the pcp lists.
2300 */
2301static inline bool check_pcp_refill(struct page *page)
2302{
2303        return check_new_page(page);
2304}
2305static inline bool check_new_pcp(struct page *page)
2306{
2307        if (debug_pagealloc_enabled_static())
2308                return check_new_page(page);
2309        else
2310                return false;
2311}
2312#endif /* CONFIG_DEBUG_VM */
2313
2314static bool check_new_pages(struct page *page, unsigned int order)
2315{
2316        int i;
2317        for (i = 0; i < (1 << order); i++) {
2318                struct page *p = page + i;
2319
2320                if (unlikely(check_new_page(p)))
2321                        return true;
2322        }
2323
2324        return false;
2325}
2326
2327inline void post_alloc_hook(struct page *page, unsigned int order,
2328                                gfp_t gfp_flags)
2329{
2330        bool init;
2331
2332        set_page_private(page, 0);
2333        set_page_refcounted(page);
2334
2335        arch_alloc_page(page, order);
2336        debug_pagealloc_map_pages(page, 1 << order);
2337
2338        /*
2339         * Page unpoisoning must happen before memory initialization.
2340         * Otherwise, the poison pattern will be overwritten for __GFP_ZERO
2341         * allocations and the page unpoisoning code will complain.
2342         */
2343        kernel_unpoison_pages(page, 1 << order);
2344
2345        /*
2346         * As memory initialization might be integrated into KASAN,
2347         * kasan_alloc_pages and kernel_init_free_pages must be
2348         * kept together to avoid discrepancies in behavior.
2349         */
2350        init = !want_init_on_free() && want_init_on_alloc(gfp_flags);
2351        kasan_alloc_pages(page, order, init);
2352        if (init && !kasan_has_integrated_init())
2353                kernel_init_free_pages(page, 1 << order);
2354
2355        set_page_owner(page, order, gfp_flags);
2356}
2357
2358static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
2359                                                        unsigned int alloc_flags)
2360{
2361        post_alloc_hook(page, order, gfp_flags);
2362
2363        if (order && (gfp_flags & __GFP_COMP))
2364                prep_compound_page(page, order);
2365
2366        /*
2367         * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
2368         * allocate the page. The expectation is that the caller is taking
2369         * steps that will free more memory. The caller should avoid the page
2370         * being used for !PFMEMALLOC purposes.
2371         */
2372        if (alloc_flags & ALLOC_NO_WATERMARKS)
2373                set_page_pfmemalloc(page);
2374        else
2375                clear_page_pfmemalloc(page);
2376}
2377
2378/*
2379 * Go through the free lists for the given migratetype and remove
2380 * the smallest available page from the freelists
2381 */
2382static __always_inline
2383struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
2384                                                int migratetype)
2385{
2386        unsigned int current_order;
2387        struct free_area *area;
2388        struct page *page;
2389
2390        /* Find a page of the appropriate size in the preferred list */
2391        for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2392                area = &(zone->free_area[current_order]);
2393                page = get_page_from_free_area(area, migratetype);
2394                if (!page)
2395                        continue;
2396                del_page_from_free_list(page, zone, current_order);
2397                expand(zone, page, order, current_order, migratetype);
2398                set_pcppage_migratetype(page, migratetype);
2399                return page;
2400        }
2401
2402        return NULL;
2403}
2404
2405
2406/*
2407 * This array describes the order lists are fallen back to when
2408 * the free lists for the desirable migrate type are depleted
2409 */
2410static int fallbacks[MIGRATE_TYPES][3] = {
2411        [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_TYPES },
2412        [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
2413        [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_TYPES },
2414#ifdef CONFIG_CMA
2415        [MIGRATE_CMA]         = { MIGRATE_TYPES }, /* Never used */
2416#endif
2417#ifdef CONFIG_MEMORY_ISOLATION
2418        [MIGRATE_ISOLATE]     = { MIGRATE_TYPES }, /* Never used */
2419#endif
2420};
2421
2422#ifdef CONFIG_CMA
2423static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2424                                        unsigned int order)
2425{
2426        return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2427}
2428#else
2429static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2430                                        unsigned int order) { return NULL; }
2431#endif
2432
2433/*
2434 * Move the free pages in a range to the freelist tail of the requested type.
2435 * Note that start_page and end_pages are not aligned on a pageblock
2436 * boundary. If alignment is required, use move_freepages_block()
2437 */
2438static int move_freepages(struct zone *zone,
2439                          unsigned long start_pfn, unsigned long end_pfn,
2440                          int migratetype, int *num_movable)
2441{
2442        struct page *page;
2443        unsigned long pfn;
2444        unsigned int order;
2445        int pages_moved = 0;
2446
2447        for (pfn = start_pfn; pfn <= end_pfn;) {
2448                if (!pfn_valid_within(pfn)) {
2449                        pfn++;
2450                        continue;
2451                }
2452
2453                page = pfn_to_page(pfn);
2454                if (!PageBuddy(page)) {
2455                        /*
2456                         * We assume that pages that could be isolated for
2457                         * migration are movable. But we don't actually try
2458                         * isolating, as that would be expensive.
2459                         */
2460                        if (num_movable &&
2461                                        (PageLRU(page) || __PageMovable(page)))
2462                                (*num_movable)++;
2463                        pfn++;
2464                        continue;
2465                }
2466
2467                /* Make sure we are not inadvertently changing nodes */
2468                VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2469                VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2470
2471                order = buddy_order(page);
2472                move_to_free_list(page, zone, order, migratetype);
2473                pfn += 1 << order;
2474                pages_moved += 1 << order;
2475        }
2476
2477        return pages_moved;
2478}
2479
2480int move_freepages_block(struct zone *zone, struct page *page,
2481                                int migratetype, int *num_movable)
2482{
2483        unsigned long start_pfn, end_pfn, pfn;
2484
2485        if (num_movable)
2486                *num_movable = 0;
2487
2488        pfn = page_to_pfn(page);
2489        start_pfn = pfn & ~(pageblock_nr_pages - 1);
2490        end_pfn = start_pfn + pageblock_nr_pages - 1;
2491
2492        /* Do not cross zone boundaries */
2493        if (!zone_spans_pfn(zone, start_pfn))
2494                start_pfn = pfn;
2495        if (!zone_spans_pfn(zone, end_pfn))
2496                return 0;
2497
2498        return move_freepages(zone, start_pfn, end_pfn, migratetype,
2499                                                                num_movable);
2500}
2501
2502static void change_pageblock_range(struct page *pageblock_page,
2503                                        int start_order, int migratetype)
2504{
2505        int nr_pageblocks = 1 << (start_order - pageblock_order);
2506
2507        while (nr_pageblocks--) {
2508                set_pageblock_migratetype(pageblock_page, migratetype);
2509                pageblock_page += pageblock_nr_pages;
2510        }
2511}
2512
2513/*
2514 * When we are falling back to another migratetype during allocation, try to
2515 * steal extra free pages from the same pageblocks to satisfy further
2516 * allocations, instead of polluting multiple pageblocks.
2517 *
2518 * If we are stealing a relatively large buddy page, it is likely there will
2519 * be more free pages in the pageblock, so try to steal them all. For
2520 * reclaimable and unmovable allocations, we steal regardless of page size,
2521 * as fragmentation caused by those allocations polluting movable pageblocks
2522 * is worse than movable allocations stealing from unmovable and reclaimable
2523 * pageblocks.
2524 */
2525static bool can_steal_fallback(unsigned int order, int start_mt)
2526{
2527        /*
2528         * Leaving this order check is intended, although there is
2529         * relaxed order check in next check. The reason is that
2530         * we can actually steal whole pageblock if this condition met,
2531         * but, below check doesn't guarantee it and that is just heuristic
2532         * so could be changed anytime.
2533         */
2534        if (order >= pageblock_order)
2535                return true;
2536
2537        if (order >= pageblock_order / 2 ||
2538                start_mt == MIGRATE_RECLAIMABLE ||
2539                start_mt == MIGRATE_UNMOVABLE ||
2540                page_group_by_mobility_disabled)
2541                return true;
2542
2543        return false;
2544}
2545
2546static inline bool boost_watermark(struct zone *zone)
2547{
2548        unsigned long max_boost;
2549
2550        if (!watermark_boost_factor)
2551                return false;
2552        /*
2553         * Don't bother in zones that are unlikely to produce results.
2554         * On small machines, including kdump capture kernels running
2555         * in a small area, boosting the watermark can cause an out of
2556         * memory situation immediately.
2557         */
2558        if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
2559                return false;
2560
2561        max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2562                        watermark_boost_factor, 10000);
2563
2564        /*
2565         * high watermark may be uninitialised if fragmentation occurs
2566         * very early in boot so do not boost. We do not fall
2567         * through and boost by pageblock_nr_pages as failing
2568         * allocations that early means that reclaim is not going
2569         * to help and it may even be impossible to reclaim the
2570         * boosted watermark resulting in a hang.
2571         */
2572        if (!max_boost)
2573                return false;
2574
2575        max_boost = max(pageblock_nr_pages, max_boost);
2576
2577        zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2578                max_boost);
2579
2580        return true;
2581}
2582
2583/*
2584 * This function implements actual steal behaviour. If order is large enough,
2585 * we can steal whole pageblock. If not, we first move freepages in this
2586 * pageblock to our migratetype and determine how many already-allocated pages
2587 * are there in the pageblock with a compatible migratetype. If at least half
2588 * of pages are free or compatible, we can change migratetype of the pageblock
2589 * itself, so pages freed in the future will be put on the correct free list.
2590 */
2591static void steal_suitable_fallback(struct zone *zone, struct page *page,
2592                unsigned int alloc_flags, int start_type, bool whole_block)
2593{
2594        unsigned int current_order = buddy_order(page);
2595        int free_pages, movable_pages, alike_pages;
2596        int old_block_type;
2597
2598        old_block_type = get_pageblock_migratetype(page);
2599
2600        /*
2601         * This can happen due to races and we want to prevent broken
2602         * highatomic accounting.
2603         */
2604        if (is_migrate_highatomic(old_block_type))
2605                goto single_page;
2606
2607        /* Take ownership for orders >= pageblock_order */
2608        if (current_order >= pageblock_order) {
2609                change_pageblock_range(page, current_order, start_type);
2610                goto single_page;
2611        }
2612
2613        /*
2614         * Boost watermarks to increase reclaim pressure to reduce the
2615         * likelihood of future fallbacks. Wake kswapd now as the node
2616         * may be balanced overall and kswapd will not wake naturally.
2617         */
2618        if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
2619                set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
2620
2621        /* We are not allowed to try stealing from the whole block */
2622        if (!whole_block)
2623                goto single_page;
2624
2625        free_pages = move_freepages_block(zone, page, start_type,
2626                                                &movable_pages);
2627        /*
2628         * Determine how many pages are compatible with our allocation.
2629         * For movable allocation, it's the number of movable pages which
2630         * we just obtained. For other types it's a bit more tricky.
2631         */
2632        if (start_type == MIGRATE_MOVABLE) {
2633                alike_pages = movable_pages;
2634        } else {
2635                /*
2636                 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2637                 * to MOVABLE pageblock, consider all non-movable pages as
2638                 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2639                 * vice versa, be conservative since we can't distinguish the
2640                 * exact migratetype of non-movable pages.
2641                 */
2642                if (old_block_type == MIGRATE_MOVABLE)
2643                        alike_pages = pageblock_nr_pages
2644                                                - (free_pages + movable_pages);
2645                else
2646                        alike_pages = 0;
2647        }
2648
2649        /* moving whole block can fail due to zone boundary conditions */
2650        if (!free_pages)
2651                goto single_page;
2652
2653        /*
2654         * If a sufficient number of pages in the block are either free or of
2655         * comparable migratability as our allocation, claim the whole block.
2656         */
2657        if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2658                        page_group_by_mobility_disabled)
2659                set_pageblock_migratetype(page, start_type);
2660
2661        return;
2662
2663single_page:
2664        move_to_free_list(page, zone, current_order, start_type);
2665}
2666
2667/*
2668 * Check whether there is a suitable fallback freepage with requested order.
2669 * If only_stealable is true, this function returns fallback_mt only if
2670 * we can steal other freepages all together. This would help to reduce
2671 * fragmentation due to mixed migratetype pages in one pageblock.
2672 */
2673int find_suitable_fallback(struct free_area *area, unsigned int order,
2674                        int migratetype, bool only_stealable, bool *can_steal)
2675{
2676        int i;
2677        int fallback_mt;
2678
2679        if (area->nr_free == 0)
2680                return -1;
2681
2682        *can_steal = false;
2683        for (i = 0;; i++) {
2684                fallback_mt = fallbacks[migratetype][i];
2685                if (fallback_mt == MIGRATE_TYPES)
2686                        break;
2687
2688                if (free_area_empty(area, fallback_mt))
2689                        continue;
2690
2691                if (can_steal_fallback(order, migratetype))
2692                        *can_steal = true;
2693
2694                if (!only_stealable)
2695                        return fallback_mt;
2696
2697                if (*can_steal)
2698                        return fallback_mt;
2699        }
2700
2701        return -1;
2702}
2703
2704/*
2705 * Reserve a pageblock for exclusive use of high-order atomic allocations if
2706 * there are no empty page blocks that contain a page with a suitable order
2707 */
2708static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2709                                unsigned int alloc_order)
2710{
2711        int mt;
2712        unsigned long max_managed, flags;
2713
2714        /*
2715         * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2716         * Check is race-prone but harmless.
2717         */
2718        max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
2719        if (zone->nr_reserved_highatomic >= max_managed)
2720                return;
2721
2722        spin_lock_irqsave(&zone->lock, flags);
2723
2724        /* Recheck the nr_reserved_highatomic limit under the lock */
2725        if (zone->nr_reserved_highatomic >= max_managed)
2726                goto out_unlock;
2727
2728        /* Yoink! */
2729        mt = get_pageblock_migratetype(page);
2730        if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2731            && !is_migrate_cma(mt)) {
2732                zone->nr_reserved_highatomic += pageblock_nr_pages;
2733                set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2734                move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2735        }
2736
2737out_unlock:
2738        spin_unlock_irqrestore(&zone->lock, flags);
2739}
2740
2741/*
2742 * Used when an allocation is about to fail under memory pressure. This
2743 * potentially hurts the reliability of high-order allocations when under
2744 * intense memory pressure but failed atomic allocations should be easier
2745 * to recover from than an OOM.
2746 *
2747 * If @force is true, try to unreserve a pageblock even though highatomic
2748 * pageblock is exhausted.
2749 */
2750static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2751                                                bool force)
2752{
2753        struct zonelist *zonelist = ac->zonelist;
2754        unsigned long flags;
2755        struct zoneref *z;
2756        struct zone *zone;
2757        struct page *page;
2758        int order;
2759        bool ret;
2760
2761        for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
2762                                                                ac->nodemask) {
2763                /*
2764                 * Preserve at least one pageblock unless memory pressure
2765                 * is really high.
2766                 */
2767                if (!force && zone->nr_reserved_highatomic <=
2768                                        pageblock_nr_pages)
2769                        continue;
2770
2771                spin_lock_irqsave(&zone->lock, flags);
2772                for (order = 0; order < MAX_ORDER; order++) {
2773                        struct free_area *area = &(zone->free_area[order]);
2774
2775                        page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
2776                        if (!page)
2777                                continue;
2778
2779                        /*
2780                         * In page freeing path, migratetype change is racy so
2781                         * we can counter several free pages in a pageblock
2782                         * in this loop although we changed the pageblock type
2783                         * from highatomic to ac->migratetype. So we should
2784                         * adjust the count once.
2785                         */
2786                        if (is_migrate_highatomic_page(page)) {
2787                                /*
2788                                 * It should never happen but changes to
2789                                 * locking could inadvertently allow a per-cpu
2790                                 * drain to add pages to MIGRATE_HIGHATOMIC
2791                                 * while unreserving so be safe and watch for
2792                                 * underflows.
2793                                 */
2794                                zone->nr_reserved_highatomic -= min(
2795                                                pageblock_nr_pages,
2796                                                zone->nr_reserved_highatomic);
2797                        }
2798
2799                        /*
2800                         * Convert to ac->migratetype and avoid the normal
2801                         * pageblock stealing heuristics. Minimally, the caller
2802                         * is doing the work and needs the pages. More
2803                         * importantly, if the block was always converted to
2804                         * MIGRATE_UNMOVABLE or another type then the number
2805                         * of pageblocks that cannot be completely freed
2806                         * may increase.
2807                         */
2808                        set_pageblock_migratetype(page, ac->migratetype);
2809                        ret = move_freepages_block(zone, page, ac->migratetype,
2810                                                                        NULL);
2811                        if (ret) {
2812                                spin_unlock_irqrestore(&zone->lock, flags);
2813                                return ret;
2814                        }
2815                }
2816                spin_unlock_irqrestore(&zone->lock, flags);
2817        }
2818
2819        return false;
2820}
2821
2822/*
2823 * Try finding a free buddy page on the fallback list and put it on the free
2824 * list of requested migratetype, possibly along with other pages from the same
2825 * block, depending on fragmentation avoidance heuristics. Returns true if
2826 * fallback was found so that __rmqueue_smallest() can grab it.
2827 *
2828 * The use of signed ints for order and current_order is a deliberate
2829 * deviation from the rest of this file, to make the for loop
2830 * condition simpler.
2831 */
2832static __always_inline bool
2833__rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
2834                                                unsigned int alloc_flags)
2835{
2836        struct free_area *area;
2837        int current_order;
2838        int min_order = order;
2839        struct page *page;
2840        int fallback_mt;
2841        bool can_steal;
2842
2843        /*
2844         * Do not steal pages from freelists belonging to other pageblocks
2845         * i.e. orders < pageblock_order. If there are no local zones free,
2846         * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
2847         */
2848        if (alloc_flags & ALLOC_NOFRAGMENT)
2849                min_order = pageblock_order;
2850
2851        /*
2852         * Find the largest available free page in the other list. This roughly
2853         * approximates finding the pageblock with the most free pages, which
2854         * would be too costly to do exactly.
2855         */
2856        for (current_order = MAX_ORDER - 1; current_order >= min_order;
2857                                --current_order) {
2858                area = &(zone->free_area[current_order]);
2859                fallback_mt = find_suitable_fallback(area, current_order,
2860                                start_migratetype, false, &can_steal);
2861                if (fallback_mt == -1)
2862                        continue;
2863
2864                /*
2865                 * We cannot steal all free pages from the pageblock and the
2866                 * requested migratetype is movable. In that case it's better to
2867                 * steal and split the smallest available page instead of the
2868                 * largest available page, because even if the next movable
2869                 * allocation falls back into a different pageblock than this
2870                 * one, it won't cause permanent fragmentation.
2871                 */
2872                if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2873                                        && current_order > order)
2874                        goto find_smallest;
2875
2876                goto do_steal;
2877        }
2878
2879        return false;
2880
2881find_smallest:
2882        for (current_order = order; current_order < MAX_ORDER;
2883                                                        current_order++) {
2884                area = &(zone->free_area[current_order]);
2885                fallback_mt = find_suitable_fallback(area, current_order,
2886                                start_migratetype, false, &can_steal);
2887                if (fallback_mt != -1)
2888                        break;
2889        }
2890
2891        /*
2892         * This should not happen - we already found a suitable fallback
2893         * when looking for the largest page.
2894         */
2895        VM_BUG_ON(current_order == MAX_ORDER);
2896
2897do_steal:
2898        page = get_page_from_free_area(area, fallback_mt);
2899
2900        steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2901                                                                can_steal);
2902
2903        trace_mm_page_alloc_extfrag(page, order, current_order,
2904                start_migratetype, fallback_mt);
2905
2906        return true;
2907
2908}
2909
2910/*
2911 * Do the hard work of removing an element from the buddy allocator.
2912 * Call me with the zone->lock already held.
2913 */
2914static __always_inline struct page *
2915__rmqueue(struct zone *zone, unsigned int order, int migratetype,
2916                                                unsigned int alloc_flags)
2917{
2918        struct page *page;
2919
2920        if (IS_ENABLED(CONFIG_CMA)) {
2921                /*
2922                 * Balance movable allocations between regular and CMA areas by
2923                 * allocating from CMA when over half of the zone's free memory
2924                 * is in the CMA area.
2925                 */
2926                if (alloc_flags & ALLOC_CMA &&
2927                    zone_page_state(zone, NR_FREE_CMA_PAGES) >
2928                    zone_page_state(zone, NR_FREE_PAGES) / 2) {
2929                        page = __rmqueue_cma_fallback(zone, order);
2930                        if (page)
2931                                goto out;
2932                }
2933        }
2934retry:
2935        page = __rmqueue_smallest(zone, order, migratetype);
2936        if (unlikely(!page)) {
2937                if (alloc_flags & ALLOC_CMA)
2938                        page = __rmqueue_cma_fallback(zone, order);
2939
2940                if (!page && __rmqueue_fallback(zone, order, migratetype,
2941                                                                alloc_flags))
2942                        goto retry;
2943        }
2944out:
2945        if (page)
2946                trace_mm_page_alloc_zone_locked(page, order, migratetype);
2947        return page;
2948}
2949
2950/*
2951 * Obtain a specified number of elements from the buddy allocator, all under
2952 * a single hold of the lock, for efficiency.  Add them to the supplied list.
2953 * Returns the number of new pages which were placed at *list.
2954 */
2955static int rmqueue_bulk(struct zone *zone, unsigned int order,
2956                        unsigned long count, struct list_head *list,
2957                        int migratetype, unsigned int alloc_flags)
2958{
2959        int i, allocated = 0;
2960
2961        spin_lock(&zone->lock);
2962        for (i = 0; i < count; ++i) {
2963                struct page *page = __rmqueue(zone, order, migratetype,
2964                                                                alloc_flags);
2965                if (unlikely(page == NULL))
2966                        break;
2967
2968                if (unlikely(check_pcp_refill(page)))
2969                        continue;
2970
2971                /*
2972                 * Split buddy pages returned by expand() are received here in
2973                 * physical page order. The page is added to the tail of
2974                 * caller's list. From the callers perspective, the linked list
2975                 * is ordered by page number under some conditions. This is
2976                 * useful for IO devices that can forward direction from the
2977                 * head, thus also in the physical page order. This is useful
2978                 * for IO devices that can merge IO requests if the physical
2979                 * pages are ordered properly.
2980                 */
2981                list_add_tail(&page->lru, list);
2982                allocated++;
2983                if (is_migrate_cma(get_pcppage_migratetype(page)))
2984                        __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2985                                              -(1 << order));
2986        }
2987
2988        /*
2989         * i pages were removed from the buddy list even if some leak due
2990         * to check_pcp_refill failing so adjust NR_FREE_PAGES based
2991         * on i. Do not confuse with 'allocated' which is the number of
2992         * pages added to the pcp list.
2993         */
2994        __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2995        spin_unlock(&zone->lock);
2996        return allocated;
2997}
2998
2999#ifdef CONFIG_NUMA
3000/*
3001 * Called from the vmstat counter updater to drain pagesets of this
3002 * currently executing processor on remote nodes after they have
3003 * expired.
3004 *
3005 * Note that this function must be called with the thread pinned to
3006 * a single processor.
3007 */
3008void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
3009{
3010        unsigned long flags;
3011        int to_drain, batch;
3012
3013        local_irq_save(flags);
3014        batch = READ_ONCE(pcp->batch);
3015        to_drain = min(pcp->count, batch);
3016        if (to_drain > 0)
3017                free_pcppages_bulk(zone, to_drain, pcp);
3018        local_irq_restore(flags);
3019}
3020#endif
3021
3022/*
3023 * Drain pcplists of the indicated processor and zone.
3024 *
3025 * The processor must either be the current processor and the
3026 * thread pinned to the current processor or a processor that
3027 * is not online.
3028 */
3029static void drain_pages_zone(unsigned int cpu, struct zone *zone)
3030{
3031        unsigned long flags;
3032        struct per_cpu_pageset *pset;
3033        struct per_cpu_pages *pcp;
3034
3035        local_irq_save(flags);
3036        pset = per_cpu_ptr(zone->pageset, cpu);
3037
3038        pcp = &pset->pcp;
3039        if (pcp->count)
3040                free_pcppages_bulk(zone, pcp->count, pcp);
3041        local_irq_restore(flags);
3042}
3043
3044/*
3045 * Drain pcplists of all zones on the indicated processor.
3046 *
3047 * The processor must either be the current processor and the
3048 * thread pinned to the current processor or a processor that
3049 * is not online.
3050 */
3051static void drain_pages(unsigned int cpu)
3052{
3053        struct zone *zone;
3054
3055        for_each_populated_zone(zone) {
3056                drain_pages_zone(cpu, zone);
3057        }
3058}
3059
3060/*
3061 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
3062 *
3063 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
3064 * the single zone's pages.
3065 */
3066void drain_local_pages(struct zone *zone)
3067{
3068        int cpu = smp_processor_id();
3069
3070        if (zone)
3071                drain_pages_zone(cpu, zone);
3072        else
3073                drain_pages(cpu);
3074}
3075
3076static void drain_local_pages_wq(struct work_struct *work)
3077{
3078        struct pcpu_drain *drain;
3079
3080        drain = container_of(work, struct pcpu_drain, work);
3081
3082        /*
3083         * drain_all_pages doesn't use proper cpu hotplug protection so
3084         * we can race with cpu offline when the WQ can move this from
3085         * a cpu pinned worker to an unbound one. We can operate on a different
3086         * cpu which is alright but we also have to make sure to not move to
3087         * a different one.
3088         */
3089        preempt_disable();
3090        drain_local_pages(drain->zone);
3091        preempt_enable();
3092}
3093
3094/*
3095 * The implementation of drain_all_pages(), exposing an extra parameter to
3096 * drain on all cpus.
3097 *
3098 * drain_all_pages() is optimized to only execute on cpus where pcplists are
3099 * not empty. The check for non-emptiness can however race with a free to
3100 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers
3101 * that need the guarantee that every CPU has drained can disable the
3102 * optimizing racy check.
3103 */
3104static void __drain_all_pages(struct zone *zone, bool force_all_cpus)
3105{
3106        int cpu;
3107
3108        /*
3109         * Allocate in the BSS so we wont require allocation in
3110         * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
3111         */
3112        static cpumask_t cpus_with_pcps;
3113
3114        /*
3115         * Make sure nobody triggers this path before mm_percpu_wq is fully
3116         * initialized.
3117         */
3118        if (WARN_ON_ONCE(!mm_percpu_wq))
3119                return;
3120
3121        /*
3122         * Do not drain if one is already in progress unless it's specific to
3123         * a zone. Such callers are primarily CMA and memory hotplug and need
3124         * the drain to be complete when the call returns.
3125         */
3126        if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
3127                if (!zone)
3128                        return;
3129                mutex_lock(&pcpu_drain_mutex);
3130        }
3131
3132        /*
3133         * We don't care about racing with CPU hotplug event
3134         * as offline notification will cause the notified
3135         * cpu to drain that CPU pcps and on_each_cpu_mask
3136         * disables preemption as part of its processing
3137         */
3138        for_each_online_cpu(cpu) {
3139                struct per_cpu_pageset *pcp;
3140                struct zone *z;
3141                bool has_pcps = false;
3142
3143                if (force_all_cpus) {
3144                        /*
3145                         * The pcp.count check is racy, some callers need a
3146                         * guarantee that no cpu is missed.
3147                         */
3148                        has_pcps = true;
3149                } else if (zone) {
3150                        pcp = per_cpu_ptr(zone->pageset, cpu);
3151                        if (pcp->pcp.count)
3152                                has_pcps = true;
3153                } else {
3154                        for_each_populated_zone(z) {
3155                                pcp = per_cpu_ptr(z->pageset, cpu);
3156                                if (pcp->pcp.count) {
3157                                        has_pcps = true;
3158                                        break;
3159                                }
3160                        }
3161                }
3162
3163                if (has_pcps)
3164                        cpumask_set_cpu(cpu, &cpus_with_pcps);
3165                else
3166                        cpumask_clear_cpu(cpu, &cpus_with_pcps);
3167        }
3168
3169        for_each_cpu(cpu, &cpus_with_pcps) {
3170                struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
3171
3172                drain->zone = zone;
3173                INIT_WORK(&drain->work, drain_local_pages_wq);
3174                queue_work_on(cpu, mm_percpu_wq, &drain->work);
3175        }
3176        for_each_cpu(cpu, &cpus_with_pcps)
3177                flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
3178
3179        mutex_unlock(&pcpu_drain_mutex);
3180}
3181
3182/*
3183 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
3184 *
3185 * When zone parameter is non-NULL, spill just the single zone's pages.
3186 *
3187 * Note that this can be extremely slow as the draining happens in a workqueue.
3188 */
3189void drain_all_pages(struct zone *zone)
3190{
3191        __drain_all_pages(zone, false);
3192}
3193
3194#ifdef CONFIG_HIBERNATION
3195
3196/*
3197 * Touch the watchdog for every WD_PAGE_COUNT pages.
3198 */
3199#define WD_PAGE_COUNT   (128*1024)
3200
3201void mark_free_pages(struct zone *zone)
3202{
3203        unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
3204        unsigned long flags;
3205        unsigned int order, t;
3206        struct page *page;
3207
3208        if (zone_is_empty(zone))
3209                return;
3210
3211        spin_lock_irqsave(&zone->lock, flags);
3212
3213        max_zone_pfn = zone_end_pfn(zone);
3214        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
3215                if (pfn_valid(pfn)) {
3216                        page = pfn_to_page(pfn);
3217
3218                        if (!--page_count) {
3219                                touch_nmi_watchdog();
3220                                page_count = WD_PAGE_COUNT;
3221                        }
3222
3223                        if (page_zone(page) != zone)
3224                                continue;
3225
3226                        if (!swsusp_page_is_forbidden(page))
3227                                swsusp_unset_page_free(page);
3228                }
3229
3230        for_each_migratetype_order(order, t) {
3231                list_for_each_entry(page,
3232                                &zone->free_area[order].free_list[t], lru) {
3233                        unsigned long i;
3234
3235                        pfn = page_to_pfn(page);
3236                        for (i = 0; i < (1UL << order); i++) {
3237                                if (!--page_count) {
3238                                        touch_nmi_watchdog();
3239                                        page_count = WD_PAGE_COUNT;
3240                                }
3241                                swsusp_set_page_free(pfn_to_page(pfn + i));
3242                        }
3243                }
3244        }
3245        spin_unlock_irqrestore(&zone->lock, flags);
3246}
3247#endif /* CONFIG_PM */
3248
3249static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
3250{
3251        int migratetype;
3252
3253        if (!free_pcp_prepare(page))
3254                return false;
3255
3256        migratetype = get_pfnblock_migratetype(page, pfn);
3257        set_pcppage_migratetype(page, migratetype);
3258        return true;
3259}
3260
3261static void free_unref_page_commit(struct page *page, unsigned long pfn)
3262{
3263        struct zone *zone = page_zone(page);
3264        struct per_cpu_pages *pcp;
3265        int migratetype;
3266
3267        migratetype = get_pcppage_migratetype(page);
3268        __count_vm_event(PGFREE);
3269
3270        /*
3271         * We only track unmovable, reclaimable and movable on pcp lists.
3272         * Free ISOLATE pages back to the allocator because they are being
3273         * offlined but treat HIGHATOMIC as movable pages so we can get those
3274         * areas back if necessary. Otherwise, we may have to free
3275         * excessively into the page allocator
3276         */
3277        if (migratetype >= MIGRATE_PCPTYPES) {
3278                if (unlikely(is_migrate_isolate(migratetype))) {
3279                        free_one_page(zone, page, pfn, 0, migratetype,
3280                                      FPI_NONE);
3281                        return;
3282                }
3283                migratetype = MIGRATE_MOVABLE;
3284        }
3285
3286        pcp = &this_cpu_ptr(zone->pageset)->pcp;
3287        list_add(&page->lru, &pcp->lists[migratetype]);
3288        pcp->count++;
3289        if (pcp->count >= READ_ONCE(pcp->high))
3290                free_pcppages_bulk(zone, READ_ONCE(pcp->batch), pcp);
3291}
3292
3293/*
3294 * Free a 0-order page
3295 */
3296void free_unref_page(struct page *page)
3297{
3298        unsigned long flags;
3299        unsigned long pfn = page_to_pfn(page);
3300
3301        if (!free_unref_page_prepare(page, pfn))
3302                return;
3303
3304        local_irq_save(flags);
3305        free_unref_page_commit(page, pfn);
3306        local_irq_restore(flags);
3307}
3308
3309/*
3310 * Free a list of 0-order pages
3311 */
3312void free_unref_page_list(struct list_head *list)
3313{
3314        struct page *page, *next;
3315        unsigned long flags, pfn;
3316        int batch_count = 0;
3317
3318        /* Prepare pages for freeing */
3319        list_for_each_entry_safe(page, next, list, lru) {
3320                pfn = page_to_pfn(page);
3321                if (!free_unref_page_prepare(page, pfn))
3322                        list_del(&page->lru);
3323                set_page_private(page, pfn);
3324        }
3325
3326        local_irq_save(flags);
3327        list_for_each_entry_safe(page, next, list, lru) {
3328                unsigned long pfn = page_private(page);
3329
3330                set_page_private(page, 0);
3331                trace_mm_page_free_batched(page);
3332                free_unref_page_commit(page, pfn);
3333
3334                /*
3335                 * Guard against excessive IRQ disabled times when we get
3336                 * a large list of pages to free.
3337                 */
3338                if (++batch_count == SWAP_CLUSTER_MAX) {
3339                        local_irq_restore(flags);
3340                        batch_count = 0;
3341                        local_irq_save(flags);
3342                }
3343        }
3344        local_irq_restore(flags);
3345}
3346
3347/*
3348 * split_page takes a non-compound higher-order page, and splits it into
3349 * n (1<<order) sub-pages: page[0..n]
3350 * Each sub-page must be freed individually.
3351 *
3352 * Note: this is probably too low level an operation for use in drivers.
3353 * Please consult with lkml before using this in your driver.
3354 */
3355void split_page(struct page *page, unsigned int order)
3356{
3357        int i;
3358
3359        VM_BUG_ON_PAGE(PageCompound(page), page);
3360        VM_BUG_ON_PAGE(!page_count(page), page);
3361
3362        for (i = 1; i < (1 << order); i++)
3363                set_page_refcounted(page + i);
3364        split_page_owner(page, 1 << order);
3365        split_page_memcg(page, 1 << order);
3366}
3367EXPORT_SYMBOL_GPL(split_page);
3368
3369int __isolate_free_page(struct page *page, unsigned int order)
3370{
3371        unsigned long watermark;
3372        struct zone *zone;
3373        int mt;
3374
3375        BUG_ON(!PageBuddy(page));
3376
3377        zone = page_zone(page);
3378        mt = get_pageblock_migratetype(page);
3379
3380        if (!is_migrate_isolate(mt)) {
3381                /*
3382                 * Obey watermarks as if the page was being allocated. We can
3383                 * emulate a high-order watermark check with a raised order-0
3384                 * watermark, because we already know our high-order page
3385                 * exists.
3386                 */
3387                watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
3388                if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
3389                        return 0;
3390
3391                __mod_zone_freepage_state(zone, -(1UL << order), mt);
3392        }
3393
3394        /* Remove page from free list */
3395
3396        del_page_from_free_list(page, zone, order);
3397
3398        /*
3399         * Set the pageblock if the isolated page is at least half of a
3400         * pageblock
3401         */
3402        if (order >= pageblock_order - 1) {
3403                struct page *endpage = page + (1 << order) - 1;
3404                for (; page < endpage; page += pageblock_nr_pages) {
3405                        int mt = get_pageblock_migratetype(page);
3406                        if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
3407                            && !is_migrate_highatomic(mt))
3408                                set_pageblock_migratetype(page,
3409                                                          MIGRATE_MOVABLE);
3410                }
3411        }
3412
3413
3414        return 1UL << order;
3415}
3416
3417/**
3418 * __putback_isolated_page - Return a now-isolated page back where we got it
3419 * @page: Page that was isolated
3420 * @order: Order of the isolated page
3421 * @mt: The page's pageblock's migratetype
3422 *
3423 * This function is meant to return a page pulled from the free lists via
3424 * __isolate_free_page back to the free lists they were pulled from.
3425 */
3426void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3427{
3428        struct zone *zone = page_zone(page);
3429
3430        /* zone lock should be held when this function is called */
3431        lockdep_assert_held(&zone->lock);
3432
3433        /* Return isolated page to tail of freelist. */
3434        __free_one_page(page, page_to_pfn(page), zone, order, mt,
3435                        FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
3436}
3437
3438/*
3439 * Update NUMA hit/miss statistics
3440 *
3441 * Must be called with interrupts disabled.
3442 */
3443static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
3444{
3445#ifdef CONFIG_NUMA
3446        enum numa_stat_item local_stat = NUMA_LOCAL;
3447
3448        /* skip numa counters update if numa stats is disabled */
3449        if (!static_branch_likely(&vm_numa_stat_key))
3450                return;
3451
3452        if (zone_to_nid(z) != numa_node_id())
3453                local_stat = NUMA_OTHER;
3454
3455        if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3456                __inc_numa_state(z, NUMA_HIT);
3457        else {
3458                __inc_numa_state(z, NUMA_MISS);
3459                __inc_numa_state(preferred_zone, NUMA_FOREIGN);
3460        }
3461        __inc_numa_state(z, local_stat);
3462#endif
3463}
3464
3465/* Remove page from the per-cpu list, caller must protect the list */
3466static inline
3467struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
3468                        unsigned int alloc_flags,
3469                        struct per_cpu_pages *pcp,
3470                        struct list_head *list)
3471{
3472        struct page *page;
3473
3474        do {
3475                if (list_empty(list)) {
3476                        pcp->count += rmqueue_bulk(zone, 0,
3477                                        READ_ONCE(pcp->batch), list,
3478                                        migratetype, alloc_flags);
3479                        if (unlikely(list_empty(list)))
3480                                return NULL;
3481                }
3482
3483                page = list_first_entry(list, struct page, lru);
3484                list_del(&page->lru);
3485                pcp->count--;
3486        } while (check_new_pcp(page));
3487
3488        return page;
3489}
3490
3491/* Lock and remove page from the per-cpu list */
3492static struct page *rmqueue_pcplist(struct zone *preferred_zone,
3493                        struct zone *zone, gfp_t gfp_flags,
3494                        int migratetype, unsigned int alloc_flags)
3495{
3496        struct per_cpu_pages *pcp;
3497        struct list_head *list;
3498        struct page *page;
3499        unsigned long flags;
3500
3501        local_irq_save(flags);
3502        pcp = &this_cpu_ptr(zone->pageset)->pcp;
3503        list = &pcp->lists[migratetype];
3504        page = __rmqueue_pcplist(zone,  migratetype, alloc_flags, pcp, list);
3505        if (page) {
3506                __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
3507                zone_statistics(preferred_zone, zone);
3508        }
3509        local_irq_restore(flags);
3510        return page;
3511}
3512
3513/*
3514 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
3515 */
3516static inline
3517struct page *rmqueue(struct zone *preferred_zone,
3518                        struct zone *zone, unsigned int order,
3519                        gfp_t gfp_flags, unsigned int alloc_flags,
3520                        int migratetype)
3521{
3522        unsigned long flags;
3523        struct page *page;
3524
3525        if (likely(order == 0)) {
3526                /*
3527                 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and
3528                 * we need to skip it when CMA area isn't allowed.
3529                 */
3530                if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA ||
3531                                migratetype != MIGRATE_MOVABLE) {
3532                        page = rmqueue_pcplist(preferred_zone, zone, gfp_flags,
3533                                        migratetype, alloc_flags);
3534                        goto out;
3535                }
3536        }
3537
3538        /*
3539         * We most definitely don't want callers attempting to
3540         * allocate greater than order-1 page units with __GFP_NOFAIL.
3541         */
3542        WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3543        spin_lock_irqsave(&zone->lock, flags);
3544
3545        do {
3546                page = NULL;
3547                /*
3548                 * order-0 request can reach here when the pcplist is skipped
3549                 * due to non-CMA allocation context. HIGHATOMIC area is
3550                 * reserved for high-order atomic allocation, so order-0
3551                 * request should skip it.
3552                 */
3553                if (order > 0 && alloc_flags & ALLOC_HARDER) {
3554                        page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3555                        if (page)
3556                                trace_mm_page_alloc_zone_locked(page, order, migratetype);
3557                }
3558                if (!page)
3559                        page = __rmqueue(zone, order, migratetype, alloc_flags);
3560        } while (page && check_new_pages(page, order));
3561        spin_unlock(&zone->lock);
3562        if (!page)
3563                goto failed;
3564        __mod_zone_freepage_state(zone, -(1 << order),
3565                                  get_pcppage_migratetype(page));
3566
3567        __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3568        zone_statistics(preferred_zone, zone);
3569        local_irq_restore(flags);
3570
3571out:
3572        /* Separate test+clear to avoid unnecessary atomics */
3573        if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3574                clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3575                wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3576        }
3577
3578        VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
3579        return page;
3580
3581failed:
3582        local_irq_restore(flags);
3583        return NULL;
3584}
3585
3586#ifdef CONFIG_FAIL_PAGE_ALLOC
3587
3588static struct {
3589        struct fault_attr attr;
3590
3591        bool ignore_gfp_highmem;
3592        bool ignore_gfp_reclaim;
3593        u32 min_order;
3594} fail_page_alloc = {
3595        .attr = FAULT_ATTR_INITIALIZER,
3596        .ignore_gfp_reclaim = true,
3597        .ignore_gfp_highmem = true,
3598        .min_order = 1,
3599};
3600
3601static int __init setup_fail_page_alloc(char *str)
3602{
3603        return setup_fault_attr(&fail_page_alloc.attr, str);
3604}
3605__setup("fail_page_alloc=", setup_fail_page_alloc);
3606
3607static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3608{
3609        if (order < fail_page_alloc.min_order)
3610                return false;
3611        if (gfp_mask & __GFP_NOFAIL)
3612                return false;
3613        if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
3614                return false;
3615        if (fail_page_alloc.ignore_gfp_reclaim &&
3616                        (gfp_mask & __GFP_DIRECT_RECLAIM))
3617                return false;
3618
3619        return should_fail(&fail_page_alloc.attr, 1 << order);
3620}
3621
3622#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3623
3624static int __init fail_page_alloc_debugfs(void)
3625{
3626        umode_t mode = S_IFREG | 0600;
3627        struct dentry *dir;
3628
3629        dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3630                                        &fail_page_alloc.attr);
3631
3632        debugfs_create_bool("ignore-gfp-wait", mode, dir,
3633                            &fail_page_alloc.ignore_gfp_reclaim);
3634        debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3635                            &fail_page_alloc.ignore_gfp_highmem);
3636        debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
3637
3638        return 0;
3639}
3640
3641late_initcall(fail_page_alloc_debugfs);
3642
3643#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
3644
3645#else /* CONFIG_FAIL_PAGE_ALLOC */
3646
3647static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3648{
3649        return false;
3650}
3651
3652#endif /* CONFIG_FAIL_PAGE_ALLOC */
3653
3654noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3655{
3656        return __should_fail_alloc_page(gfp_mask, order);
3657}
3658ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3659
3660static inline long __zone_watermark_unusable_free(struct zone *z,
3661                                unsigned int order, unsigned int alloc_flags)
3662{
3663        const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3664        long unusable_free = (1 << order) - 1;
3665
3666        /*
3667         * If the caller does not have rights to ALLOC_HARDER then subtract
3668         * the high-atomic reserves. This will over-estimate the size of the
3669         * atomic reserve but it avoids a search.
3670         */
3671        if (likely(!alloc_harder))
3672                unusable_free += z->nr_reserved_highatomic;
3673
3674#ifdef CONFIG_CMA
3675        /* If allocation can't use CMA areas don't use free CMA pages */
3676        if (!(alloc_flags & ALLOC_CMA))
3677                unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
3678#endif
3679
3680        return unusable_free;
3681}
3682
3683/*
3684 * Return true if free base pages are above 'mark'. For high-order checks it
3685 * will return true of the order-0 watermark is reached and there is at least
3686 * one free page of a suitable size. Checking now avoids taking the zone lock
3687 * to check in the allocation paths if no pages are free.
3688 */
3689bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3690                         int highest_zoneidx, unsigned int alloc_flags,
3691                         long free_pages)
3692{
3693        long min = mark;
3694        int o;
3695        const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3696
3697        /* free_pages may go negative - that's OK */
3698        free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
3699
3700        if (alloc_flags & ALLOC_HIGH)
3701                min -= min / 2;
3702
3703        if (unlikely(alloc_harder)) {
3704                /*
3705                 * OOM victims can try even harder than normal ALLOC_HARDER
3706                 * users on the grounds that it's definitely going to be in
3707                 * the exit path shortly and free memory. Any allocation it
3708                 * makes during the free path will be small and short-lived.
3709                 */
3710                if (alloc_flags & ALLOC_OOM)
3711                        min -= min / 2;
3712                else
3713                        min -= min / 4;
3714        }
3715
3716        /*
3717         * Check watermarks for an order-0 allocation request. If these
3718         * are not met, then a high-order request also cannot go ahead
3719         * even if a suitable page happened to be free.
3720         */
3721        if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
3722                return false;
3723
3724        /* If this is an order-0 request then the watermark is fine */
3725        if (!order)
3726                return true;
3727
3728        /* For a high-order request, check at least one suitable page is free */
3729        for (o = order; o < MAX_ORDER; o++) {
3730                struct free_area *area = &z->free_area[o];
3731                int mt;
3732
3733                if (!area->nr_free)
3734                        continue;
3735
3736                for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3737                        if (!free_area_empty(area, mt))
3738                                return true;
3739                }
3740
3741#ifdef CONFIG_CMA
3742                if ((alloc_flags & ALLOC_CMA) &&
3743                    !free_area_empty(area, MIGRATE_CMA)) {
3744                        return true;
3745                }
3746#endif
3747                if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC))
3748                        return true;
3749        }
3750        return false;
3751}
3752
3753bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3754                      int highest_zoneidx, unsigned int alloc_flags)
3755{
3756        return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3757                                        zone_page_state(z, NR_FREE_PAGES));
3758}
3759
3760static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
3761                                unsigned long mark, int highest_zoneidx,
3762                                unsigned int alloc_flags, gfp_t gfp_mask)
3763{
3764        long free_pages;
3765
3766        free_pages = zone_page_state(z, NR_FREE_PAGES);
3767
3768        /*
3769         * Fast check for order-0 only. If this fails then the reserves
3770         * need to be calculated.
3771         */
3772        if (!order) {
3773                long fast_free;
3774
3775                fast_free = free_pages;
3776                fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags);
3777                if (fast_free > mark + z->lowmem_reserve[highest_zoneidx])
3778                        return true;
3779        }
3780
3781        if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3782                                        free_pages))
3783                return true;
3784        /*
3785         * Ignore watermark boosting for GFP_ATOMIC order-0 allocations
3786         * when checking the min watermark. The min watermark is the
3787         * point where boosting is ignored so that kswapd is woken up
3788         * when below the low watermark.
3789         */
3790        if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost
3791                && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
3792                mark = z->_watermark[WMARK_MIN];
3793                return __zone_watermark_ok(z, order, mark, highest_zoneidx,
3794                                        alloc_flags, free_pages);
3795        }
3796
3797        return false;
3798}
3799
3800bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
3801                        unsigned long mark, int highest_zoneidx)
3802{
3803        long free_pages = zone_page_state(z, NR_FREE_PAGES);
3804
3805        if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3806                free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3807
3808        return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0,
3809                                                                free_pages);
3810}
3811
3812#ifdef CONFIG_NUMA
3813static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3814{
3815        return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3816                                node_reclaim_distance;
3817}
3818#else   /* CONFIG_NUMA */
3819static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3820{
3821        return true;
3822}
3823#endif  /* CONFIG_NUMA */
3824
3825/*
3826 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3827 * fragmentation is subtle. If the preferred zone was HIGHMEM then
3828 * premature use of a lower zone may cause lowmem pressure problems that
3829 * are worse than fragmentation. If the next zone is ZONE_DMA then it is
3830 * probably too small. It only makes sense to spread allocations to avoid
3831 * fragmentation between the Normal and DMA32 zones.
3832 */
3833static inline unsigned int
3834alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
3835{
3836        unsigned int alloc_flags;
3837
3838        /*
3839         * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
3840         * to save a branch.
3841         */
3842        alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
3843
3844#ifdef CONFIG_ZONE_DMA32
3845        if (!zone)
3846                return alloc_flags;
3847
3848        if (zone_idx(zone) != ZONE_NORMAL)
3849                return alloc_flags;
3850
3851        /*
3852         * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
3853         * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
3854         * on UMA that if Normal is populated then so is DMA32.
3855         */
3856        BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
3857        if (nr_online_nodes > 1 && !populated_zone(--zone))
3858                return alloc_flags;
3859
3860        alloc_flags |= ALLOC_NOFRAGMENT;
3861#endif /* CONFIG_ZONE_DMA32 */
3862        return alloc_flags;
3863}
3864
3865/* Must be called after current_gfp_context() which can change gfp_mask */
3866static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask,
3867                                                  unsigned int alloc_flags)
3868{
3869#ifdef CONFIG_CMA
3870        if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
3871                alloc_flags |= ALLOC_CMA;
3872#endif
3873        return alloc_flags;
3874}
3875
3876/*
3877 * get_page_from_freelist goes through the zonelist trying to allocate
3878 * a page.
3879 */
3880static struct page *
3881get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3882                                                const struct alloc_context *ac)
3883{
3884        struct zoneref *z;
3885        struct zone *zone;
3886        struct pglist_data *last_pgdat_dirty_limit = NULL;
3887        bool no_fallback;
3888
3889retry:
3890        /*
3891         * Scan zonelist, looking for a zone with enough free.
3892         * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
3893         */
3894        no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
3895        z = ac->preferred_zoneref;
3896        for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
3897                                        ac->nodemask) {
3898                struct page *page;
3899                unsigned long mark;
3900
3901                if (cpusets_enabled() &&
3902                        (alloc_flags & ALLOC_CPUSET) &&
3903                        !__cpuset_zone_allowed(zone, gfp_mask))
3904                                continue;
3905                /*
3906                 * When allocating a page cache page for writing, we
3907                 * want to get it from a node that is within its dirty
3908                 * limit, such that no single node holds more than its
3909                 * proportional share of globally allowed dirty pages.
3910                 * The dirty limits take into account the node's
3911                 * lowmem reserves and high watermark so that kswapd
3912                 * should be able to balance it without having to
3913                 * write pages from its LRU list.
3914                 *
3915                 * XXX: For now, allow allocations to potentially
3916                 * exceed the per-node dirty limit in the slowpath
3917                 * (spread_dirty_pages unset) before going into reclaim,
3918                 * which is important when on a NUMA setup the allowed
3919                 * nodes are together not big enough to reach the
3920                 * global limit.  The proper fix for these situations
3921                 * will require awareness of nodes in the
3922                 * dirty-throttling and the flusher threads.
3923                 */
3924                if (ac->spread_dirty_pages) {
3925                        if (last_pgdat_dirty_limit == zone->zone_pgdat)
3926                                continue;
3927
3928                        if (!node_dirty_ok(zone->zone_pgdat)) {
3929                                last_pgdat_dirty_limit = zone->zone_pgdat;
3930                                continue;
3931                        }
3932                }
3933
3934                if (no_fallback && nr_online_nodes > 1 &&
3935                    zone != ac->preferred_zoneref->zone) {
3936                        int local_nid;
3937
3938                        /*
3939                         * If moving to a remote node, retry but allow
3940                         * fragmenting fallbacks. Locality is more important
3941                         * than fragmentation avoidance.
3942                         */
3943                        local_nid = zone_to_nid(ac->preferred_zoneref->zone);
3944                        if (zone_to_nid(zone) != local_nid) {
3945                                alloc_flags &= ~ALLOC_NOFRAGMENT;
3946                                goto retry;
3947                        }
3948                }
3949
3950                mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
3951                if (!zone_watermark_fast(zone, order, mark,
3952                                       ac->highest_zoneidx, alloc_flags,
3953                                       gfp_mask)) {
3954                        int ret;
3955
3956#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
3957                        /*
3958                         * Watermark failed for this zone, but see if we can
3959                         * grow this zone if it contains deferred pages.
3960                         */
3961                        if (static_branch_unlikely(&deferred_pages)) {
3962                                if (_deferred_grow_zone(zone, order))
3963                                        goto try_this_zone;
3964                        }
3965#endif
3966                        /* Checked here to keep the fast path fast */
3967                        BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3968                        if (alloc_flags & ALLOC_NO_WATERMARKS)
3969                                goto try_this_zone;
3970
3971                        if (!node_reclaim_enabled() ||
3972                            !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
3973                                continue;
3974
3975                        ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3976                        switch (ret) {
3977                        case NODE_RECLAIM_NOSCAN:
3978                                /* did not scan */
3979                                continue;
3980                        case NODE_RECLAIM_FULL:
3981                                /* scanned but unreclaimable */
3982                                continue;
3983                        default:
3984                                /* did we reclaim enough */
3985                                if (zone_watermark_ok(zone, order, mark,
3986                                        ac->highest_zoneidx, alloc_flags))
3987                                        goto try_this_zone;
3988
3989                                continue;
3990                        }
3991                }
3992
3993try_this_zone:
3994                page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3995                                gfp_mask, alloc_flags, ac->migratetype);
3996                if (page) {
3997                        prep_new_page(page, order, gfp_mask, alloc_flags);
3998
3999                        /*
4000                         * If this is a high-order atomic allocation then check
4001                         * if the pageblock should be reserved for the future
4002                         */
4003                        if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
4004                                reserve_highatomic_pageblock(page, zone, order);
4005
4006                        return page;
4007                } else {
4008#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
4009                        /* Try again if zone has deferred pages */
4010                        if (static_branch_unlikely(&deferred_pages)) {
4011                                if (_deferred_grow_zone(zone, order))
4012                                        goto try_this_zone;
4013                        }
4014#endif
4015                }
4016        }
4017
4018        /*
4019         * It's possible on a UMA machine to get through all zones that are
4020         * fragmented. If avoiding fragmentation, reset and try again.
4021         */
4022        if (no_fallback) {
4023                alloc_flags &= ~ALLOC_NOFRAGMENT;
4024                goto retry;
4025        }
4026
4027        return NULL;
4028}
4029
4030static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
4031{
4032        unsigned int filter = SHOW_MEM_FILTER_NODES;
4033
4034        /*
4035         * This documents exceptions given to allocations in certain
4036         * contexts that are allowed to allocate outside current's set
4037         * of allowed nodes.
4038         */
4039        if (!(gfp_mask & __GFP_NOMEMALLOC))
4040                if (tsk_is_oom_victim(current) ||
4041                    (current->flags & (PF_MEMALLOC | PF_EXITING)))
4042                        filter &= ~SHOW_MEM_FILTER_NODES;
4043        if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
4044                filter &= ~SHOW_MEM_FILTER_NODES;
4045
4046        show_mem(filter, nodemask);
4047}
4048
4049void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
4050{
4051        struct va_format vaf;
4052        va_list args;
4053        static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
4054
4055        if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
4056                return;
4057
4058        va_start(args, fmt);
4059        vaf.fmt = fmt;
4060        vaf.va = &args;
4061        pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
4062                        current->comm, &vaf, gfp_mask, &gfp_mask,
4063                        nodemask_pr_args(nodemask));
4064        va_end(args);
4065
4066        cpuset_print_current_mems_allowed();
4067        pr_cont("\n");
4068        dump_stack();
4069        warn_alloc_show_mem(gfp_mask, nodemask);
4070}
4071
4072static inline struct page *
4073__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
4074                              unsigned int alloc_flags,
4075                              const struct alloc_context *ac)
4076{
4077        struct page *page;
4078
4079        page = get_page_from_freelist(gfp_mask, order,
4080                        alloc_flags|ALLOC_CPUSET, ac);
4081        /*
4082         * fallback to ignore cpuset restriction if our nodes
4083         * are depleted
4084         */
4085        if (!page)
4086                page = get_page_from_freelist(gfp_mask, order,
4087                                alloc_flags, ac);
4088
4089        return page;
4090}
4091
4092static inline struct page *
4093__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
4094        const struct alloc_context *ac, unsigned long *did_some_progress)
4095{
4096        struct oom_control oc = {
4097                .zonelist = ac->zonelist,
4098                .nodemask = ac->nodemask,
4099                .memcg = NULL,
4100                .gfp_mask = gfp_mask,
4101                .order = order,
4102        };
4103        struct page *page;
4104
4105        *did_some_progress = 0;
4106
4107        /*
4108         * Acquire the oom lock.  If that fails, somebody else is
4109         * making progress for us.
4110         */
4111        if (!mutex_trylock(&oom_lock)) {
4112                *did_some_progress = 1;
4113                schedule_timeout_uninterruptible(1);
4114                return NULL;
4115        }
4116
4117        /*
4118         * Go through the zonelist yet one more time, keep very high watermark
4119         * here, this is only to catch a parallel oom killing, we must fail if
4120         * we're still under heavy pressure. But make sure that this reclaim
4121         * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
4122         * allocation which will never fail due to oom_lock already held.
4123         */
4124        page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
4125                                      ~__GFP_DIRECT_RECLAIM, order,
4126                                      ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
4127        if (page)
4128                goto out;
4129
4130        /* Coredumps can quickly deplete all memory reserves */
4131        if (current->flags & PF_DUMPCORE)
4132                goto out;
4133        /* The OOM killer will not help higher order allocs */
4134        if (order > PAGE_ALLOC_COSTLY_ORDER)
4135                goto out;
4136        /*
4137         * We have already exhausted all our reclaim opportunities without any
4138         * success so it is time to admit defeat. We will skip the OOM killer
4139         * because it is very likely that the caller has a more reasonable
4140         * fallback than shooting a random task.
4141         *
4142         * The OOM killer may not free memory on a specific node.
4143         */
4144        if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))
4145                goto out;
4146        /* The OOM killer does not needlessly kill tasks for lowmem */
4147        if (ac->highest_zoneidx < ZONE_NORMAL)
4148                goto out;
4149        if (pm_suspended_storage())
4150                goto out;
4151        /*
4152         * XXX: GFP_NOFS allocations should rather fail than rely on
4153         * other request to make a forward progress.
4154         * We are in an unfortunate situation where out_of_memory cannot
4155         * do much for this context but let's try it to at least get
4156         * access to memory reserved if the current task is killed (see
4157         * out_of_memory). Once filesystems are ready to handle allocation
4158         * failures more gracefully we should just bail out here.
4159         */
4160
4161        /* Exhausted what can be done so it's blame time */
4162        if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
4163                *did_some_progress = 1;
4164
4165                /*
4166                 * Help non-failing allocations by giving them access to memory
4167                 * reserves
4168                 */
4169                if (gfp_mask & __GFP_NOFAIL)
4170                        page = __alloc_pages_cpuset_fallback(gfp_mask, order,
4171                                        ALLOC_NO_WATERMARKS, ac);
4172        }
4173out:
4174        mutex_unlock(&oom_lock);
4175        return page;
4176}
4177
4178/*
4179 * Maximum number of compaction retries with a progress before OOM
4180 * killer is consider as the only way to move forward.
4181 */
4182#define MAX_COMPACT_RETRIES 16
4183
4184#ifdef CONFIG_COMPACTION
4185/* Try memory compaction for high-order allocations before reclaim */
4186static struct page *
4187__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
4188                unsigned int alloc_flags, const struct alloc_context *ac,
4189                enum compact_priority prio, enum compact_result *compact_result)
4190{
4191        struct page *page = NULL;
4192        unsigned long pflags;
4193        unsigned int noreclaim_flag;
4194
4195        if (!order)
4196                return NULL;
4197
4198        psi_memstall_enter(&pflags);
4199        noreclaim_flag = memalloc_noreclaim_save();
4200
4201        *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
4202                                                                prio, &page);
4203
4204        memalloc_noreclaim_restore(noreclaim_flag);
4205        psi_memstall_leave(&pflags);
4206
4207        if (*compact_result == COMPACT_SKIPPED)
4208                return NULL;
4209        /*
4210         * At least in one zone compaction wasn't deferred or skipped, so let's
4211         * count a compaction stall
4212         */
4213        count_vm_event(COMPACTSTALL);
4214
4215        /* Prep a captured page if available */
4216        if (page)
4217                prep_new_page(page, order, gfp_mask, alloc_flags);
4218
4219        /* Try get a page from the freelist if available */
4220        if (!page)
4221                page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4222
4223        if (page) {
4224                struct zone *zone = page_zone(page);
4225
4226                zone->compact_blockskip_flush = false;
4227                compaction_defer_reset(zone, order, true);
4228                count_vm_event(COMPACTSUCCESS);
4229                return page;
4230        }
4231
4232        /*
4233         * It's bad if compaction run occurs and fails. The most likely reason
4234         * is that pages exist, but not enough to satisfy watermarks.
4235         */
4236        count_vm_event(COMPACTFAIL);
4237
4238        cond_resched();
4239
4240        return NULL;
4241}
4242
4243static inline bool
4244should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
4245                     enum compact_result compact_result,
4246                     enum compact_priority *compact_priority,
4247                     int *compaction_retries)
4248{
4249        int max_retries = MAX_COMPACT_RETRIES;
4250        int min_priority;
4251        bool ret = false;
4252        int retries = *compaction_retries;
4253        enum compact_priority priority = *compact_priority;
4254
4255        if (!order)
4256                return false;
4257
4258        if (compaction_made_progress(compact_result))
4259                (*compaction_retries)++;
4260
4261        /*
4262         * compaction considers all the zone as desperately out of memory
4263         * so it doesn't really make much sense to retry except when the
4264         * failure could be caused by insufficient priority
4265         */
4266        if (compaction_failed(compact_result))
4267                goto check_priority;
4268
4269        /*
4270         * compaction was skipped because there are not enough order-0 pages
4271         * to work with, so we retry only if it looks like reclaim can help.
4272         */
4273        if (compaction_needs_reclaim(compact_result)) {
4274                ret = compaction_zonelist_suitable(ac, order, alloc_flags);
4275                goto out;
4276        }
4277
4278        /*
4279         * make sure the compaction wasn't deferred or didn't bail out early
4280         * due to locks contention before we declare that we should give up.
4281         * But the next retry should use a higher priority if allowed, so
4282         * we don't just keep bailing out endlessly.
4283         */
4284        if (compaction_withdrawn(compact_result)) {
4285                goto check_priority;
4286        }
4287
4288        /*
4289         * !costly requests are much more important than __GFP_RETRY_MAYFAIL
4290         * costly ones because they are de facto nofail and invoke OOM
4291         * killer to move on while costly can fail and users are ready
4292         * to cope with that. 1/4 retries is rather arbitrary but we
4293         * would need much more detailed feedback from compaction to
4294         * make a better decision.
4295         */
4296        if (order > PAGE_ALLOC_COSTLY_ORDER)
4297                max_retries /= 4;
4298        if (*compaction_retries <= max_retries) {
4299                ret = true;
4300                goto out;
4301        }
4302
4303        /*
4304         * Make sure there are attempts at the highest priority if we exhausted
4305         * all retries or failed at the lower priorities.
4306         */
4307check_priority:
4308        min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
4309                        MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
4310
4311        if (*compact_priority > min_priority) {
4312                (*compact_priority)--;
4313                *compaction_retries = 0;
4314                ret = true;
4315        }
4316out:
4317        trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
4318        return ret;
4319}
4320#else
4321static inline struct page *
4322__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
4323                unsigned int alloc_flags, const struct alloc_context *ac,
4324                enum compact_priority prio, enum compact_result *compact_result)
4325{
4326        *compact_result = COMPACT_SKIPPED;
4327        return NULL;
4328}
4329
4330static inline bool
4331should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
4332                     enum compact_result compact_result,
4333                     enum compact_priority *compact_priority,
4334                     int *compaction_retries)
4335{
4336        struct zone *zone;
4337        struct zoneref *z;
4338
4339        if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
4340                return false;
4341
4342        /*
4343         * There are setups with compaction disabled which would prefer to loop
4344         * inside the allocator rather than hit the oom killer prematurely.
4345         * Let's give them a good hope and keep retrying while the order-0
4346         * watermarks are OK.
4347         */
4348        for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4349                                ac->highest_zoneidx, ac->nodemask) {
4350                if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
4351                                        ac->highest_zoneidx, alloc_flags))
4352                        return true;
4353        }
4354        return false;
4355}
4356#endif /* CONFIG_COMPACTION */
4357
4358#ifdef CONFIG_LOCKDEP
4359static struct lockdep_map __fs_reclaim_map =
4360        STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
4361
4362static bool __need_reclaim(gfp_t gfp_mask)
4363{
4364        /* no reclaim without waiting on it */
4365        if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
4366                return false;
4367
4368        /* this guy won't enter reclaim */
4369        if (current->flags & PF_MEMALLOC)
4370                return false;
4371
4372        if (gfp_mask & __GFP_NOLOCKDEP)
4373                return false;
4374
4375        return true;
4376}
4377
4378void __fs_reclaim_acquire(void)
4379{
4380        lock_map_acquire(&__fs_reclaim_map);
4381}
4382
4383void __fs_reclaim_release(void)
4384{
4385        lock_map_release(&__fs_reclaim_map);
4386}
4387
4388void fs_reclaim_acquire(gfp_t gfp_mask)
4389{
4390        gfp_mask = current_gfp_context(gfp_mask);
4391
4392        if (__need_reclaim(gfp_mask)) {
4393                if (gfp_mask & __GFP_FS)
4394                        __fs_reclaim_acquire();
4395
4396#ifdef CONFIG_MMU_NOTIFIER
4397                lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
4398                lock_map_release(&__mmu_notifier_invalidate_range_start_map);
4399#endif
4400
4401        }
4402}
4403EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
4404
4405void fs_reclaim_release(gfp_t gfp_mask)
4406{
4407        gfp_mask = current_gfp_context(gfp_mask);
4408
4409        if (__need_reclaim(gfp_mask)) {
4410                if (gfp_mask & __GFP_FS)
4411                        __fs_reclaim_release();
4412        }
4413}
4414EXPORT_SYMBOL_GPL(fs_reclaim_release);
4415#endif
4416
4417/* Perform direct synchronous page reclaim */
4418static unsigned long
4419__perform_reclaim(gfp_t gfp_mask, unsigned int order,
4420                                        const struct alloc_context *ac)
4421{
4422        unsigned int noreclaim_flag;
4423        unsigned long pflags, progress;
4424
4425        cond_resched();
4426
4427        /* We now go into synchronous reclaim */
4428        cpuset_memory_pressure_bump();
4429        psi_memstall_enter(&pflags);
4430        fs_reclaim_acquire(gfp_mask);
4431        noreclaim_flag = memalloc_noreclaim_save();
4432
4433        progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
4434                                                                ac->nodemask);
4435
4436        memalloc_noreclaim_restore(noreclaim_flag);
4437        fs_reclaim_release(gfp_mask);
4438        psi_memstall_leave(&pflags);
4439
4440        cond_resched();
4441
4442        return progress;
4443}
4444
4445/* The really slow allocator path where we enter direct reclaim */
4446static inline struct page *
4447__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
4448                unsigned int alloc_flags, const struct alloc_context *ac,
4449                unsigned long *did_some_progress)
4450{
4451        struct page *page = NULL;
4452        bool drained = false;
4453
4454        *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
4455        if (unlikely(!(*did_some_progress)))
4456                return NULL;
4457
4458retry:
4459        page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4460
4461        /*
4462         * If an allocation failed after direct reclaim, it could be because
4463         * pages are pinned on the per-cpu lists or in high alloc reserves.
4464         * Shrink them and try again
4465         */
4466        if (!page && !drained) {
4467                unreserve_highatomic_pageblock(ac, false);
4468                drain_all_pages(NULL);
4469                drained = true;
4470                goto retry;
4471        }
4472
4473        return page;
4474}
4475
4476static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4477                             const struct alloc_context *ac)
4478{
4479        struct zoneref *z;
4480        struct zone *zone;
4481        pg_data_t *last_pgdat = NULL;
4482        enum zone_type highest_zoneidx = ac->highest_zoneidx;
4483
4484        for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,
4485                                        ac->nodemask) {
4486                if (last_pgdat != zone->zone_pgdat)
4487                        wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);
4488                last_pgdat = zone->zone_pgdat;
4489        }
4490}
4491
4492static inline unsigned int
4493gfp_to_alloc_flags(gfp_t gfp_mask)
4494{
4495        unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
4496
4497        /*
4498         * __GFP_HIGH is assumed to be the same as ALLOC_HIGH
4499         * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
4500         * to save two branches.
4501         */
4502        BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
4503        BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
4504
4505        /*
4506         * The caller may dip into page reserves a bit more if the caller
4507         * cannot run direct reclaim, or if the caller has realtime scheduling
4508         * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
4509         * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
4510         */
4511        alloc_flags |= (__force int)
4512                (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
4513
4514        if (gfp_mask & __GFP_ATOMIC) {
4515                /*
4516                 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
4517                 * if it can't schedule.
4518                 */
4519                if (!(gfp_mask & __GFP_NOMEMALLOC))
4520                        alloc_flags |= ALLOC_HARDER;
4521                /*
4522                 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
4523                 * comment for __cpuset_node_allowed().
4524                 */
4525                alloc_flags &= ~ALLOC_CPUSET;
4526        } else if (unlikely(rt_task(current)) && !in_interrupt())
4527                alloc_flags |= ALLOC_HARDER;
4528
4529        alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);
4530
4531        return alloc_flags;
4532}
4533
4534static bool oom_reserves_allowed(struct task_struct *tsk)
4535{
4536        if (!tsk_is_oom_victim(tsk))
4537                return false;
4538
4539        /*
4540         * !MMU doesn't have oom reaper so give access to memory reserves
4541         * only to the thread with TIF_MEMDIE set
4542         */
4543        if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4544                return false;
4545
4546        return true;
4547}
4548
4549/*
4550 * Distinguish requests which really need access to full memory
4551 * reserves from oom victims which can live with a portion of it
4552 */
4553static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4554{
4555        if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4556                return 0;
4557        if (gfp_mask & __GFP_MEMALLOC)
4558                return ALLOC_NO_WATERMARKS;
4559        if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4560                return ALLOC_NO_WATERMARKS;
4561        if (!in_interrupt()) {
4562                if (current->flags & PF_MEMALLOC)
4563                        return ALLOC_NO_WATERMARKS;
4564                else if (oom_reserves_allowed(current))
4565                        return ALLOC_OOM;
4566        }
4567
4568        return 0;
4569}
4570
4571bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4572{
4573        return !!__gfp_pfmemalloc_flags(gfp_mask);
4574}
4575
4576/*
4577 * Checks whether it makes sense to retry the reclaim to make a forward progress
4578 * for the given allocation request.
4579 *
4580 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
4581 * without success, or when we couldn't even meet the watermark if we
4582 * reclaimed all remaining pages on the LRU lists.
4583 *
4584 * Returns true if a retry is viable or false to enter the oom path.
4585 */
4586static inline bool
4587should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4588                     struct alloc_context *ac, int alloc_flags,
4589                     bool did_some_progress, int *no_progress_loops)
4590{
4591        struct zone *zone;
4592        struct zoneref *z;
4593        bool ret = false;
4594
4595        /*
4596         * Costly allocations might have made a progress but this doesn't mean
4597         * their order will become available due to high fragmentation so
4598         * always increment the no progress counter for them
4599         */
4600        if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4601                *no_progress_loops = 0;
4602        else
4603                (*no_progress_loops)++;
4604
4605        /*
4606         * Make sure we converge to OOM if we cannot make any progress
4607         * several times in the row.
4608         */
4609        if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
4610                /* Before OOM, exhaust highatomic_reserve */
4611                return unreserve_highatomic_pageblock(ac, true);
4612        }
4613
4614        /*
4615         * Keep reclaiming pages while there is a chance this will lead
4616         * somewhere.  If none of the target zones can satisfy our allocation
4617         * request even if all reclaimable pages are considered then we are
4618         * screwed and have to go OOM.
4619         */
4620        for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4621                                ac->highest_zoneidx, ac->nodemask) {
4622                unsigned long available;
4623                unsigned long reclaimable;
4624                unsigned long min_wmark = min_wmark_pages(zone);
4625                bool wmark;
4626
4627                available = reclaimable = zone_reclaimable_pages(zone);
4628                available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
4629
4630                /*
4631                 * Would the allocation succeed if we reclaimed all
4632                 * reclaimable pages?
4633                 */
4634                wmark = __zone_watermark_ok(zone, order, min_wmark,
4635                                ac->highest_zoneidx, alloc_flags, available);
4636                trace_reclaim_retry_zone(z, order, reclaimable,
4637                                available, min_wmark, *no_progress_loops, wmark);
4638                if (wmark) {
4639                        /*
4640                         * If we didn't make any progress and have a lot of
4641                         * dirty + writeback pages then we should wait for
4642                         * an IO to complete to slow down the reclaim and
4643                         * prevent from pre mature OOM
4644                         */
4645                        if (!did_some_progress) {
4646                                unsigned long write_pending;
4647
4648                                write_pending = zone_page_state_snapshot(zone,
4649                                                        NR_ZONE_WRITE_PENDING);
4650
4651                                if (2 * write_pending > reclaimable) {
4652                                        congestion_wait(BLK_RW_ASYNC, HZ/10);
4653                                        return true;
4654                                }
4655                        }
4656
4657                        ret = true;
4658                        goto out;
4659                }
4660        }
4661
4662out:
4663        /*
4664         * Memory allocation/reclaim might be called from a WQ context and the
4665         * current implementation of the WQ concurrency control doesn't
4666         * recognize that a particular WQ is congested if the worker thread is
4667         * looping without ever sleeping. Therefore we have to do a short sleep
4668         * here rather than calling cond_resched().
4669         */
4670        if (current->flags & PF_WQ_WORKER)
4671                schedule_timeout_uninterruptible(1);
4672        else
4673                cond_resched();
4674        return ret;
4675}
4676
4677static inline bool
4678check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4679{
4680        /*
4681         * It's possible that cpuset's mems_allowed and the nodemask from
4682         * mempolicy don't intersect. This should be normally dealt with by
4683         * policy_nodemask(), but it's possible to race with cpuset update in
4684         * such a way the check therein was true, and then it became false
4685         * before we got our cpuset_mems_cookie here.
4686         * This assumes that for all allocations, ac->nodemask can come only
4687         * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
4688         * when it does not intersect with the cpuset restrictions) or the
4689         * caller can deal with a violated nodemask.
4690         */
4691        if (cpusets_enabled() && ac->nodemask &&
4692                        !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4693                ac->nodemask = NULL;
4694                return true;
4695        }
4696
4697        /*
4698         * When updating a task's mems_allowed or mempolicy nodemask, it is
4699         * possible to race with parallel threads in such a way that our
4700         * allocation can fail while the mask is being updated. If we are about
4701         * to fail, check if the cpuset changed during allocation and if so,
4702         * retry.
4703         */
4704        if (read_mems_allowed_retry(cpuset_mems_cookie))
4705                return true;
4706
4707        return false;
4708}
4709
4710static inline struct page *
4711__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
4712                                                struct alloc_context *ac)
4713{
4714        bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
4715        const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
4716        struct page *page = NULL;
4717        unsigned int alloc_flags;
4718        unsigned long did_some_progress;
4719        enum compact_priority compact_priority;
4720        enum compact_result compact_result;
4721        int compaction_retries;
4722        int no_progress_loops;
4723        unsigned int cpuset_mems_cookie;
4724        int reserve_flags;
4725
4726        /*
4727         * We also sanity check to catch abuse of atomic reserves being used by
4728         * callers that are not in atomic context.
4729         */
4730        if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
4731                                (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
4732                gfp_mask &= ~__GFP_ATOMIC;
4733
4734retry_cpuset:
4735        compaction_retries = 0;
4736        no_progress_loops = 0;
4737        compact_priority = DEF_COMPACT_PRIORITY;
4738        cpuset_mems_cookie = read_mems_allowed_begin();
4739
4740        /*
4741         * The fast path uses conservative alloc_flags to succeed only until
4742         * kswapd needs to be woken up, and to avoid the cost of setting up
4743         * alloc_flags precisely. So we do that now.
4744         */
4745        alloc_flags = gfp_to_alloc_flags(gfp_mask);
4746
4747        /*
4748         * We need to recalculate the starting point for the zonelist iterator
4749         * because we might have used different nodemask in the fast path, or
4750         * there was a cpuset modification and we are retrying - otherwise we
4751         * could end up iterating over non-eligible zones endlessly.
4752         */
4753        ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4754                                        ac->highest_zoneidx, ac->nodemask);
4755        if (!ac->preferred_zoneref->zone)
4756                goto nopage;
4757
4758        if (alloc_flags & ALLOC_KSWAPD)
4759                wake_all_kswapds(order, gfp_mask, ac);
4760
4761        /*
4762         * The adjusted alloc_flags might result in immediate success, so try
4763         * that first
4764         */
4765        page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4766        if (page)
4767                goto got_pg;
4768
4769        /*
4770         * For costly allocations, try direct compaction first, as it's likely
4771         * that we have enough base pages and don't need to reclaim. For non-
4772         * movable high-order allocations, do that as well, as compaction will
4773         * try prevent permanent fragmentation by migrating from blocks of the
4774         * same migratetype.
4775         * Don't try this for allocations that are allowed to ignore
4776         * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
4777         */
4778        if (can_direct_reclaim &&
4779                        (costly_order ||
4780                           (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4781                        && !gfp_pfmemalloc_allowed(gfp_mask)) {
4782                page = __alloc_pages_direct_compact(