linux/mm/page_alloc.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/mm/page_alloc.c
   4 *
   5 *  Manages the free list, the system allocates free pages here.
   6 *  Note that kmalloc() lives in slab.c
   7 *
   8 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   9 *  Swap reorganised 29.12.95, Stephen Tweedie
  10 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  11 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  12 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  13 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  14 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  15 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  16 */
  17
  18#include <linux/stddef.h>
  19#include <linux/mm.h>
  20#include <linux/highmem.h>
  21#include <linux/swap.h>
  22#include <linux/interrupt.h>
  23#include <linux/pagemap.h>
  24#include <linux/jiffies.h>
  25#include <linux/memblock.h>
  26#include <linux/compiler.h>
  27#include <linux/kernel.h>
  28#include <linux/kasan.h>
  29#include <linux/module.h>
  30#include <linux/suspend.h>
  31#include <linux/pagevec.h>
  32#include <linux/blkdev.h>
  33#include <linux/slab.h>
  34#include <linux/ratelimit.h>
  35#include <linux/oom.h>
  36#include <linux/topology.h>
  37#include <linux/sysctl.h>
  38#include <linux/cpu.h>
  39#include <linux/cpuset.h>
  40#include <linux/memory_hotplug.h>
  41#include <linux/nodemask.h>
  42#include <linux/vmalloc.h>
  43#include <linux/vmstat.h>
  44#include <linux/mempolicy.h>
  45#include <linux/memremap.h>
  46#include <linux/stop_machine.h>
  47#include <linux/random.h>
  48#include <linux/sort.h>
  49#include <linux/pfn.h>
  50#include <linux/backing-dev.h>
  51#include <linux/fault-inject.h>
  52#include <linux/page-isolation.h>
  53#include <linux/debugobjects.h>
  54#include <linux/kmemleak.h>
  55#include <linux/compaction.h>
  56#include <trace/events/kmem.h>
  57#include <trace/events/oom.h>
  58#include <linux/prefetch.h>
  59#include <linux/mm_inline.h>
  60#include <linux/mmu_notifier.h>
  61#include <linux/migrate.h>
  62#include <linux/hugetlb.h>
  63#include <linux/sched/rt.h>
  64#include <linux/sched/mm.h>
  65#include <linux/page_owner.h>
  66#include <linux/kthread.h>
  67#include <linux/memcontrol.h>
  68#include <linux/ftrace.h>
  69#include <linux/lockdep.h>
  70#include <linux/nmi.h>
  71#include <linux/psi.h>
  72#include <linux/padata.h>
  73#include <linux/khugepaged.h>
  74#include <linux/buffer_head.h>
  75#include <asm/sections.h>
  76#include <asm/tlbflush.h>
  77#include <asm/div64.h>
  78#include "internal.h"
  79#include "shuffle.h"
  80#include "page_reporting.h"
  81
  82/* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
  83typedef int __bitwise fpi_t;
  84
  85/* No special request */
  86#define FPI_NONE                ((__force fpi_t)0)
  87
  88/*
  89 * Skip free page reporting notification for the (possibly merged) page.
  90 * This does not hinder free page reporting from grabbing the page,
  91 * reporting it and marking it "reported" -  it only skips notifying
  92 * the free page reporting infrastructure about a newly freed page. For
  93 * example, used when temporarily pulling a page from a freelist and
  94 * putting it back unmodified.
  95 */
  96#define FPI_SKIP_REPORT_NOTIFY  ((__force fpi_t)BIT(0))
  97
  98/*
  99 * Place the (possibly merged) page to the tail of the freelist. Will ignore
 100 * page shuffling (relevant code - e.g., memory onlining - is expected to
 101 * shuffle the whole zone).
 102 *
 103 * Note: No code should rely on this flag for correctness - it's purely
 104 *       to allow for optimizations when handing back either fresh pages
 105 *       (memory onlining) or untouched pages (page isolation, free page
 106 *       reporting).
 107 */
 108#define FPI_TO_TAIL             ((__force fpi_t)BIT(1))
 109
 110/*
 111 * Don't poison memory with KASAN (only for the tag-based modes).
 112 * During boot, all non-reserved memblock memory is exposed to page_alloc.
 113 * Poisoning all that memory lengthens boot time, especially on systems with
 114 * large amount of RAM. This flag is used to skip that poisoning.
 115 * This is only done for the tag-based KASAN modes, as those are able to
 116 * detect memory corruptions with the memory tags assigned by default.
 117 * All memory allocated normally after boot gets poisoned as usual.
 118 */
 119#define FPI_SKIP_KASAN_POISON   ((__force fpi_t)BIT(2))
 120
 121/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
 122static DEFINE_MUTEX(pcp_batch_high_lock);
 123#define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)
 124
 125struct pagesets {
 126        local_lock_t lock;
 127};
 128static DEFINE_PER_CPU(struct pagesets, pagesets) = {
 129        .lock = INIT_LOCAL_LOCK(lock),
 130};
 131
 132#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
 133DEFINE_PER_CPU(int, numa_node);
 134EXPORT_PER_CPU_SYMBOL(numa_node);
 135#endif
 136
 137DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
 138
 139#ifdef CONFIG_HAVE_MEMORYLESS_NODES
 140/*
 141 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 142 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 143 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 144 * defined in <linux/topology.h>.
 145 */
 146DEFINE_PER_CPU(int, _numa_mem_);                /* Kernel "local memory" node */
 147EXPORT_PER_CPU_SYMBOL(_numa_mem_);
 148#endif
 149
 150/* work_structs for global per-cpu drains */
 151struct pcpu_drain {
 152        struct zone *zone;
 153        struct work_struct work;
 154};
 155static DEFINE_MUTEX(pcpu_drain_mutex);
 156static DEFINE_PER_CPU(struct pcpu_drain, pcpu_drain);
 157
 158#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
 159volatile unsigned long latent_entropy __latent_entropy;
 160EXPORT_SYMBOL(latent_entropy);
 161#endif
 162
 163/*
 164 * Array of node states.
 165 */
 166nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
 167        [N_POSSIBLE] = NODE_MASK_ALL,
 168        [N_ONLINE] = { { [0] = 1UL } },
 169#ifndef CONFIG_NUMA
 170        [N_NORMAL_MEMORY] = { { [0] = 1UL } },
 171#ifdef CONFIG_HIGHMEM
 172        [N_HIGH_MEMORY] = { { [0] = 1UL } },
 173#endif
 174        [N_MEMORY] = { { [0] = 1UL } },
 175        [N_CPU] = { { [0] = 1UL } },
 176#endif  /* NUMA */
 177};
 178EXPORT_SYMBOL(node_states);
 179
 180atomic_long_t _totalram_pages __read_mostly;
 181EXPORT_SYMBOL(_totalram_pages);
 182unsigned long totalreserve_pages __read_mostly;
 183unsigned long totalcma_pages __read_mostly;
 184
 185int percpu_pagelist_high_fraction;
 186gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
 187DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
 188EXPORT_SYMBOL(init_on_alloc);
 189
 190DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
 191EXPORT_SYMBOL(init_on_free);
 192
 193static bool _init_on_alloc_enabled_early __read_mostly
 194                                = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
 195static int __init early_init_on_alloc(char *buf)
 196{
 197
 198        return kstrtobool(buf, &_init_on_alloc_enabled_early);
 199}
 200early_param("init_on_alloc", early_init_on_alloc);
 201
 202static bool _init_on_free_enabled_early __read_mostly
 203                                = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);
 204static int __init early_init_on_free(char *buf)
 205{
 206        return kstrtobool(buf, &_init_on_free_enabled_early);
 207}
 208early_param("init_on_free", early_init_on_free);
 209
 210/*
 211 * A cached value of the page's pageblock's migratetype, used when the page is
 212 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
 213 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
 214 * Also the migratetype set in the page does not necessarily match the pcplist
 215 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
 216 * other index - this ensures that it will be put on the correct CMA freelist.
 217 */
 218static inline int get_pcppage_migratetype(struct page *page)
 219{
 220        return page->index;
 221}
 222
 223static inline void set_pcppage_migratetype(struct page *page, int migratetype)
 224{
 225        page->index = migratetype;
 226}
 227
 228#ifdef CONFIG_PM_SLEEP
 229/*
 230 * The following functions are used by the suspend/hibernate code to temporarily
 231 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
 232 * while devices are suspended.  To avoid races with the suspend/hibernate code,
 233 * they should always be called with system_transition_mutex held
 234 * (gfp_allowed_mask also should only be modified with system_transition_mutex
 235 * held, unless the suspend/hibernate code is guaranteed not to run in parallel
 236 * with that modification).
 237 */
 238
 239static gfp_t saved_gfp_mask;
 240
 241void pm_restore_gfp_mask(void)
 242{
 243        WARN_ON(!mutex_is_locked(&system_transition_mutex));
 244        if (saved_gfp_mask) {
 245                gfp_allowed_mask = saved_gfp_mask;
 246                saved_gfp_mask = 0;
 247        }
 248}
 249
 250void pm_restrict_gfp_mask(void)
 251{
 252        WARN_ON(!mutex_is_locked(&system_transition_mutex));
 253        WARN_ON(saved_gfp_mask);
 254        saved_gfp_mask = gfp_allowed_mask;
 255        gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
 256}
 257
 258bool pm_suspended_storage(void)
 259{
 260        if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
 261                return false;
 262        return true;
 263}
 264#endif /* CONFIG_PM_SLEEP */
 265
 266#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 267unsigned int pageblock_order __read_mostly;
 268#endif
 269
 270static void __free_pages_ok(struct page *page, unsigned int order,
 271                            fpi_t fpi_flags);
 272
 273/*
 274 * results with 256, 32 in the lowmem_reserve sysctl:
 275 *      1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 276 *      1G machine -> (16M dma, 784M normal, 224M high)
 277 *      NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 278 *      HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
 279 *      HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
 280 *
 281 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 282 * don't need any ZONE_NORMAL reservation
 283 */
 284int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
 285#ifdef CONFIG_ZONE_DMA
 286        [ZONE_DMA] = 256,
 287#endif
 288#ifdef CONFIG_ZONE_DMA32
 289        [ZONE_DMA32] = 256,
 290#endif
 291        [ZONE_NORMAL] = 32,
 292#ifdef CONFIG_HIGHMEM
 293        [ZONE_HIGHMEM] = 0,
 294#endif
 295        [ZONE_MOVABLE] = 0,
 296};
 297
 298static char * const zone_names[MAX_NR_ZONES] = {
 299#ifdef CONFIG_ZONE_DMA
 300         "DMA",
 301#endif
 302#ifdef CONFIG_ZONE_DMA32
 303         "DMA32",
 304#endif
 305         "Normal",
 306#ifdef CONFIG_HIGHMEM
 307         "HighMem",
 308#endif
 309         "Movable",
 310#ifdef CONFIG_ZONE_DEVICE
 311         "Device",
 312#endif
 313};
 314
 315const char * const migratetype_names[MIGRATE_TYPES] = {
 316        "Unmovable",
 317        "Movable",
 318        "Reclaimable",
 319        "HighAtomic",
 320#ifdef CONFIG_CMA
 321        "CMA",
 322#endif
 323#ifdef CONFIG_MEMORY_ISOLATION
 324        "Isolate",
 325#endif
 326};
 327
 328compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {
 329        [NULL_COMPOUND_DTOR] = NULL,
 330        [COMPOUND_PAGE_DTOR] = free_compound_page,
 331#ifdef CONFIG_HUGETLB_PAGE
 332        [HUGETLB_PAGE_DTOR] = free_huge_page,
 333#endif
 334#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 335        [TRANSHUGE_PAGE_DTOR] = free_transhuge_page,
 336#endif
 337};
 338
 339int min_free_kbytes = 1024;
 340int user_min_free_kbytes = -1;
 341int watermark_boost_factor __read_mostly = 15000;
 342int watermark_scale_factor = 10;
 343
 344static unsigned long nr_kernel_pages __initdata;
 345static unsigned long nr_all_pages __initdata;
 346static unsigned long dma_reserve __initdata;
 347
 348static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
 349static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
 350static unsigned long required_kernelcore __initdata;
 351static unsigned long required_kernelcore_percent __initdata;
 352static unsigned long required_movablecore __initdata;
 353static unsigned long required_movablecore_percent __initdata;
 354static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
 355static bool mirrored_kernelcore __meminitdata;
 356
 357/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
 358int movable_zone;
 359EXPORT_SYMBOL(movable_zone);
 360
 361#if MAX_NUMNODES > 1
 362unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
 363unsigned int nr_online_nodes __read_mostly = 1;
 364EXPORT_SYMBOL(nr_node_ids);
 365EXPORT_SYMBOL(nr_online_nodes);
 366#endif
 367
 368int page_group_by_mobility_disabled __read_mostly;
 369
 370#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
 371/*
 372 * During boot we initialize deferred pages on-demand, as needed, but once
 373 * page_alloc_init_late() has finished, the deferred pages are all initialized,
 374 * and we can permanently disable that path.
 375 */
 376static DEFINE_STATIC_KEY_TRUE(deferred_pages);
 377
 378/*
 379 * Calling kasan_poison_pages() only after deferred memory initialization
 380 * has completed. Poisoning pages during deferred memory init will greatly
 381 * lengthen the process and cause problem in large memory systems as the
 382 * deferred pages initialization is done with interrupt disabled.
 383 *
 384 * Assuming that there will be no reference to those newly initialized
 385 * pages before they are ever allocated, this should have no effect on
 386 * KASAN memory tracking as the poison will be properly inserted at page
 387 * allocation time. The only corner case is when pages are allocated by
 388 * on-demand allocation and then freed again before the deferred pages
 389 * initialization is done, but this is not likely to happen.
 390 */
 391static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
 392{
 393        return static_branch_unlikely(&deferred_pages) ||
 394               (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
 395                (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
 396               PageSkipKASanPoison(page);
 397}
 398
 399/* Returns true if the struct page for the pfn is uninitialised */
 400static inline bool __meminit early_page_uninitialised(unsigned long pfn)
 401{
 402        int nid = early_pfn_to_nid(pfn);
 403
 404        if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
 405                return true;
 406
 407        return false;
 408}
 409
 410/*
 411 * Returns true when the remaining initialisation should be deferred until
 412 * later in the boot cycle when it can be parallelised.
 413 */
 414static bool __meminit
 415defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
 416{
 417        static unsigned long prev_end_pfn, nr_initialised;
 418
 419        /*
 420         * prev_end_pfn static that contains the end of previous zone
 421         * No need to protect because called very early in boot before smp_init.
 422         */
 423        if (prev_end_pfn != end_pfn) {
 424                prev_end_pfn = end_pfn;
 425                nr_initialised = 0;
 426        }
 427
 428        /* Always populate low zones for address-constrained allocations */
 429        if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
 430                return false;
 431
 432        if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
 433                return true;
 434        /*
 435         * We start only with one section of pages, more pages are added as
 436         * needed until the rest of deferred pages are initialized.
 437         */
 438        nr_initialised++;
 439        if ((nr_initialised > PAGES_PER_SECTION) &&
 440            (pfn & (PAGES_PER_SECTION - 1)) == 0) {
 441                NODE_DATA(nid)->first_deferred_pfn = pfn;
 442                return true;
 443        }
 444        return false;
 445}
 446#else
 447static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
 448{
 449        return (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
 450                (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
 451               PageSkipKASanPoison(page);
 452}
 453
 454static inline bool early_page_uninitialised(unsigned long pfn)
 455{
 456        return false;
 457}
 458
 459static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
 460{
 461        return false;
 462}
 463#endif
 464
 465/* Return a pointer to the bitmap storing bits affecting a block of pages */
 466static inline unsigned long *get_pageblock_bitmap(const struct page *page,
 467                                                        unsigned long pfn)
 468{
 469#ifdef CONFIG_SPARSEMEM
 470        return section_to_usemap(__pfn_to_section(pfn));
 471#else
 472        return page_zone(page)->pageblock_flags;
 473#endif /* CONFIG_SPARSEMEM */
 474}
 475
 476static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn)
 477{
 478#ifdef CONFIG_SPARSEMEM
 479        pfn &= (PAGES_PER_SECTION-1);
 480#else
 481        pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
 482#endif /* CONFIG_SPARSEMEM */
 483        return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
 484}
 485
 486static __always_inline
 487unsigned long __get_pfnblock_flags_mask(const struct page *page,
 488                                        unsigned long pfn,
 489                                        unsigned long mask)
 490{
 491        unsigned long *bitmap;
 492        unsigned long bitidx, word_bitidx;
 493        unsigned long word;
 494
 495        bitmap = get_pageblock_bitmap(page, pfn);
 496        bitidx = pfn_to_bitidx(page, pfn);
 497        word_bitidx = bitidx / BITS_PER_LONG;
 498        bitidx &= (BITS_PER_LONG-1);
 499
 500        word = bitmap[word_bitidx];
 501        return (word >> bitidx) & mask;
 502}
 503
 504/**
 505 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
 506 * @page: The page within the block of interest
 507 * @pfn: The target page frame number
 508 * @mask: mask of bits that the caller is interested in
 509 *
 510 * Return: pageblock_bits flags
 511 */
 512unsigned long get_pfnblock_flags_mask(const struct page *page,
 513                                        unsigned long pfn, unsigned long mask)
 514{
 515        return __get_pfnblock_flags_mask(page, pfn, mask);
 516}
 517
 518static __always_inline int get_pfnblock_migratetype(const struct page *page,
 519                                        unsigned long pfn)
 520{
 521        return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
 522}
 523
 524/**
 525 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
 526 * @page: The page within the block of interest
 527 * @flags: The flags to set
 528 * @pfn: The target page frame number
 529 * @mask: mask of bits that the caller is interested in
 530 */
 531void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
 532                                        unsigned long pfn,
 533                                        unsigned long mask)
 534{
 535        unsigned long *bitmap;
 536        unsigned long bitidx, word_bitidx;
 537        unsigned long old_word, word;
 538
 539        BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
 540        BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
 541
 542        bitmap = get_pageblock_bitmap(page, pfn);
 543        bitidx = pfn_to_bitidx(page, pfn);
 544        word_bitidx = bitidx / BITS_PER_LONG;
 545        bitidx &= (BITS_PER_LONG-1);
 546
 547        VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
 548
 549        mask <<= bitidx;
 550        flags <<= bitidx;
 551
 552        word = READ_ONCE(bitmap[word_bitidx]);
 553        for (;;) {
 554                old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
 555                if (word == old_word)
 556                        break;
 557                word = old_word;
 558        }
 559}
 560
 561void set_pageblock_migratetype(struct page *page, int migratetype)
 562{
 563        if (unlikely(page_group_by_mobility_disabled &&
 564                     migratetype < MIGRATE_PCPTYPES))
 565                migratetype = MIGRATE_UNMOVABLE;
 566
 567        set_pfnblock_flags_mask(page, (unsigned long)migratetype,
 568                                page_to_pfn(page), MIGRATETYPE_MASK);
 569}
 570
 571#ifdef CONFIG_DEBUG_VM
 572static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 573{
 574        int ret = 0;
 575        unsigned seq;
 576        unsigned long pfn = page_to_pfn(page);
 577        unsigned long sp, start_pfn;
 578
 579        do {
 580                seq = zone_span_seqbegin(zone);
 581                start_pfn = zone->zone_start_pfn;
 582                sp = zone->spanned_pages;
 583                if (!zone_spans_pfn(zone, pfn))
 584                        ret = 1;
 585        } while (zone_span_seqretry(zone, seq));
 586
 587        if (ret)
 588                pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
 589                        pfn, zone_to_nid(zone), zone->name,
 590                        start_pfn, start_pfn + sp);
 591
 592        return ret;
 593}
 594
 595static int page_is_consistent(struct zone *zone, struct page *page)
 596{
 597        if (!pfn_valid_within(page_to_pfn(page)))
 598                return 0;
 599        if (zone != page_zone(page))
 600                return 0;
 601
 602        return 1;
 603}
 604/*
 605 * Temporary debugging check for pages not lying within a given zone.
 606 */
 607static int __maybe_unused bad_range(struct zone *zone, struct page *page)
 608{
 609        if (page_outside_zone_boundaries(zone, page))
 610                return 1;
 611        if (!page_is_consistent(zone, page))
 612                return 1;
 613
 614        return 0;
 615}
 616#else
 617static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
 618{
 619        return 0;
 620}
 621#endif
 622
 623static void bad_page(struct page *page, const char *reason)
 624{
 625        static unsigned long resume;
 626        static unsigned long nr_shown;
 627        static unsigned long nr_unshown;
 628
 629        /*
 630         * Allow a burst of 60 reports, then keep quiet for that minute;
 631         * or allow a steady drip of one report per second.
 632         */
 633        if (nr_shown == 60) {
 634                if (time_before(jiffies, resume)) {
 635                        nr_unshown++;
 636                        goto out;
 637                }
 638                if (nr_unshown) {
 639                        pr_alert(
 640                              "BUG: Bad page state: %lu messages suppressed\n",
 641                                nr_unshown);
 642                        nr_unshown = 0;
 643                }
 644                nr_shown = 0;
 645        }
 646        if (nr_shown++ == 0)
 647                resume = jiffies + 60 * HZ;
 648
 649        pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
 650                current->comm, page_to_pfn(page));
 651        dump_page(page, reason);
 652
 653        print_modules();
 654        dump_stack();
 655out:
 656        /* Leave bad fields for debug, except PageBuddy could make trouble */
 657        page_mapcount_reset(page); /* remove PageBuddy */
 658        add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 659}
 660
 661static inline unsigned int order_to_pindex(int migratetype, int order)
 662{
 663        int base = order;
 664
 665#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 666        if (order > PAGE_ALLOC_COSTLY_ORDER) {
 667                VM_BUG_ON(order != pageblock_order);
 668                base = PAGE_ALLOC_COSTLY_ORDER + 1;
 669        }
 670#else
 671        VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
 672#endif
 673
 674        return (MIGRATE_PCPTYPES * base) + migratetype;
 675}
 676
 677static inline int pindex_to_order(unsigned int pindex)
 678{
 679        int order = pindex / MIGRATE_PCPTYPES;
 680
 681#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 682        if (order > PAGE_ALLOC_COSTLY_ORDER) {
 683                order = pageblock_order;
 684                VM_BUG_ON(order != pageblock_order);
 685        }
 686#else
 687        VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
 688#endif
 689
 690        return order;
 691}
 692
 693static inline bool pcp_allowed_order(unsigned int order)
 694{
 695        if (order <= PAGE_ALLOC_COSTLY_ORDER)
 696                return true;
 697#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 698        if (order == pageblock_order)
 699                return true;
 700#endif
 701        return false;
 702}
 703
 704static inline void free_the_page(struct page *page, unsigned int order)
 705{
 706        if (pcp_allowed_order(order))           /* Via pcp? */
 707                free_unref_page(page, order);
 708        else
 709                __free_pages_ok(page, order, FPI_NONE);
 710}
 711
 712/*
 713 * Higher-order pages are called "compound pages".  They are structured thusly:
 714 *
 715 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
 716 *
 717 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
 718 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
 719 *
 720 * The first tail page's ->compound_dtor holds the offset in array of compound
 721 * page destructors. See compound_page_dtors.
 722 *
 723 * The first tail page's ->compound_order holds the order of allocation.
 724 * This usage means that zero-order pages may not be compound.
 725 */
 726
 727void free_compound_page(struct page *page)
 728{
 729        mem_cgroup_uncharge(page);
 730        free_the_page(page, compound_order(page));
 731}
 732
 733void prep_compound_page(struct page *page, unsigned int order)
 734{
 735        int i;
 736        int nr_pages = 1 << order;
 737
 738        __SetPageHead(page);
 739        for (i = 1; i < nr_pages; i++) {
 740                struct page *p = page + i;
 741                p->mapping = TAIL_MAPPING;
 742                set_compound_head(p, page);
 743        }
 744
 745        set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
 746        set_compound_order(page, order);
 747        atomic_set(compound_mapcount_ptr(page), -1);
 748        if (hpage_pincount_available(page))
 749                atomic_set(compound_pincount_ptr(page), 0);
 750}
 751
 752#ifdef CONFIG_DEBUG_PAGEALLOC
 753unsigned int _debug_guardpage_minorder;
 754
 755bool _debug_pagealloc_enabled_early __read_mostly
 756                        = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
 757EXPORT_SYMBOL(_debug_pagealloc_enabled_early);
 758DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
 759EXPORT_SYMBOL(_debug_pagealloc_enabled);
 760
 761DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
 762
 763static int __init early_debug_pagealloc(char *buf)
 764{
 765        return kstrtobool(buf, &_debug_pagealloc_enabled_early);
 766}
 767early_param("debug_pagealloc", early_debug_pagealloc);
 768
 769static int __init debug_guardpage_minorder_setup(char *buf)
 770{
 771        unsigned long res;
 772
 773        if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
 774                pr_err("Bad debug_guardpage_minorder value\n");
 775                return 0;
 776        }
 777        _debug_guardpage_minorder = res;
 778        pr_info("Setting debug_guardpage_minorder to %lu\n", res);
 779        return 0;
 780}
 781early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
 782
 783static inline bool set_page_guard(struct zone *zone, struct page *page,
 784                                unsigned int order, int migratetype)
 785{
 786        if (!debug_guardpage_enabled())
 787                return false;
 788
 789        if (order >= debug_guardpage_minorder())
 790                return false;
 791
 792        __SetPageGuard(page);
 793        INIT_LIST_HEAD(&page->lru);
 794        set_page_private(page, order);
 795        /* Guard pages are not available for any usage */
 796        __mod_zone_freepage_state(zone, -(1 << order), migratetype);
 797
 798        return true;
 799}
 800
 801static inline void clear_page_guard(struct zone *zone, struct page *page,
 802                                unsigned int order, int migratetype)
 803{
 804        if (!debug_guardpage_enabled())
 805                return;
 806
 807        __ClearPageGuard(page);
 808
 809        set_page_private(page, 0);
 810        if (!is_migrate_isolate(migratetype))
 811                __mod_zone_freepage_state(zone, (1 << order), migratetype);
 812}
 813#else
 814static inline bool set_page_guard(struct zone *zone, struct page *page,
 815                        unsigned int order, int migratetype) { return false; }
 816static inline void clear_page_guard(struct zone *zone, struct page *page,
 817                                unsigned int order, int migratetype) {}
 818#endif
 819
 820/*
 821 * Enable static keys related to various memory debugging and hardening options.
 822 * Some override others, and depend on early params that are evaluated in the
 823 * order of appearance. So we need to first gather the full picture of what was
 824 * enabled, and then make decisions.
 825 */
 826void init_mem_debugging_and_hardening(void)
 827{
 828        bool page_poisoning_requested = false;
 829
 830#ifdef CONFIG_PAGE_POISONING
 831        /*
 832         * Page poisoning is debug page alloc for some arches. If
 833         * either of those options are enabled, enable poisoning.
 834         */
 835        if (page_poisoning_enabled() ||
 836             (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
 837              debug_pagealloc_enabled())) {
 838                static_branch_enable(&_page_poisoning_enabled);
 839                page_poisoning_requested = true;
 840        }
 841#endif
 842
 843        if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
 844            page_poisoning_requested) {
 845                pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
 846                        "will take precedence over init_on_alloc and init_on_free\n");
 847                _init_on_alloc_enabled_early = false;
 848                _init_on_free_enabled_early = false;
 849        }
 850
 851        if (_init_on_alloc_enabled_early)
 852                static_branch_enable(&init_on_alloc);
 853        else
 854                static_branch_disable(&init_on_alloc);
 855
 856        if (_init_on_free_enabled_early)
 857                static_branch_enable(&init_on_free);
 858        else
 859                static_branch_disable(&init_on_free);
 860
 861#ifdef CONFIG_DEBUG_PAGEALLOC
 862        if (!debug_pagealloc_enabled())
 863                return;
 864
 865        static_branch_enable(&_debug_pagealloc_enabled);
 866
 867        if (!debug_guardpage_minorder())
 868                return;
 869
 870        static_branch_enable(&_debug_guardpage_enabled);
 871#endif
 872}
 873
 874static inline void set_buddy_order(struct page *page, unsigned int order)
 875{
 876        set_page_private(page, order);
 877        __SetPageBuddy(page);
 878}
 879
 880/*
 881 * This function checks whether a page is free && is the buddy
 882 * we can coalesce a page and its buddy if
 883 * (a) the buddy is not in a hole (check before calling!) &&
 884 * (b) the buddy is in the buddy system &&
 885 * (c) a page and its buddy have the same order &&
 886 * (d) a page and its buddy are in the same zone.
 887 *
 888 * For recording whether a page is in the buddy system, we set PageBuddy.
 889 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
 890 *
 891 * For recording page's order, we use page_private(page).
 892 */
 893static inline bool page_is_buddy(struct page *page, struct page *buddy,
 894                                                        unsigned int order)
 895{
 896        if (!page_is_guard(buddy) && !PageBuddy(buddy))
 897                return false;
 898
 899        if (buddy_order(buddy) != order)
 900                return false;
 901
 902        /*
 903         * zone check is done late to avoid uselessly calculating
 904         * zone/node ids for pages that could never merge.
 905         */
 906        if (page_zone_id(page) != page_zone_id(buddy))
 907                return false;
 908
 909        VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
 910
 911        return true;
 912}
 913
 914#ifdef CONFIG_COMPACTION
 915static inline struct capture_control *task_capc(struct zone *zone)
 916{
 917        struct capture_control *capc = current->capture_control;
 918
 919        return unlikely(capc) &&
 920                !(current->flags & PF_KTHREAD) &&
 921                !capc->page &&
 922                capc->cc->zone == zone ? capc : NULL;
 923}
 924
 925static inline bool
 926compaction_capture(struct capture_control *capc, struct page *page,
 927                   int order, int migratetype)
 928{
 929        if (!capc || order != capc->cc->order)
 930                return false;
 931
 932        /* Do not accidentally pollute CMA or isolated regions*/
 933        if (is_migrate_cma(migratetype) ||
 934            is_migrate_isolate(migratetype))
 935                return false;
 936
 937        /*
 938         * Do not let lower order allocations pollute a movable pageblock.
 939         * This might let an unmovable request use a reclaimable pageblock
 940         * and vice-versa but no more than normal fallback logic which can
 941         * have trouble finding a high-order free page.
 942         */
 943        if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
 944                return false;
 945
 946        capc->page = page;
 947        return true;
 948}
 949
 950#else
 951static inline struct capture_control *task_capc(struct zone *zone)
 952{
 953        return NULL;
 954}
 955
 956static inline bool
 957compaction_capture(struct capture_control *capc, struct page *page,
 958                   int order, int migratetype)
 959{
 960        return false;
 961}
 962#endif /* CONFIG_COMPACTION */
 963
 964/* Used for pages not on another list */
 965static inline void add_to_free_list(struct page *page, struct zone *zone,
 966                                    unsigned int order, int migratetype)
 967{
 968        struct free_area *area = &zone->free_area[order];
 969
 970        list_add(&page->lru, &area->free_list[migratetype]);
 971        area->nr_free++;
 972}
 973
 974/* Used for pages not on another list */
 975static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
 976                                         unsigned int order, int migratetype)
 977{
 978        struct free_area *area = &zone->free_area[order];
 979
 980        list_add_tail(&page->lru, &area->free_list[migratetype]);
 981        area->nr_free++;
 982}
 983
 984/*
 985 * Used for pages which are on another list. Move the pages to the tail
 986 * of the list - so the moved pages won't immediately be considered for
 987 * allocation again (e.g., optimization for memory onlining).
 988 */
 989static inline void move_to_free_list(struct page *page, struct zone *zone,
 990                                     unsigned int order, int migratetype)
 991{
 992        struct free_area *area = &zone->free_area[order];
 993
 994        list_move_tail(&page->lru, &area->free_list[migratetype]);
 995}
 996
 997static inline void del_page_from_free_list(struct page *page, struct zone *zone,
 998                                           unsigned int order)
 999{
1000        /* clear reported state and update reported page count */
1001        if (page_reported(page))
1002                __ClearPageReported(page);
1003
1004        list_del(&page->lru);
1005        __ClearPageBuddy(page);
1006        set_page_private(page, 0);
1007        zone->free_area[order].nr_free--;
1008}
1009
1010/*
1011 * If this is not the largest possible page, check if the buddy
1012 * of the next-highest order is free. If it is, it's possible
1013 * that pages are being freed that will coalesce soon. In case,
1014 * that is happening, add the free page to the tail of the list
1015 * so it's less likely to be used soon and more likely to be merged
1016 * as a higher order page
1017 */
1018static inline bool
1019buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
1020                   struct page *page, unsigned int order)
1021{
1022        struct page *higher_page, *higher_buddy;
1023        unsigned long combined_pfn;
1024
1025        if (order >= MAX_ORDER - 2)
1026                return false;
1027
1028        if (!pfn_valid_within(buddy_pfn))
1029                return false;
1030
1031        combined_pfn = buddy_pfn & pfn;
1032        higher_page = page + (combined_pfn - pfn);
1033        buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
1034        higher_buddy = higher_page + (buddy_pfn - combined_pfn);
1035
1036        return pfn_valid_within(buddy_pfn) &&
1037               page_is_buddy(higher_page, higher_buddy, order + 1);
1038}
1039
1040/*
1041 * Freeing function for a buddy system allocator.
1042 *
1043 * The concept of a buddy system is to maintain direct-mapped table
1044 * (containing bit values) for memory blocks of various "orders".
1045 * The bottom level table contains the map for the smallest allocatable
1046 * units of memory (here, pages), and each level above it describes
1047 * pairs of units from the levels below, hence, "buddies".
1048 * At a high level, all that happens here is marking the table entry
1049 * at the bottom level available, and propagating the changes upward
1050 * as necessary, plus some accounting needed to play nicely with other
1051 * parts of the VM system.
1052 * At each level, we keep a list of pages, which are heads of continuous
1053 * free pages of length of (1 << order) and marked with PageBuddy.
1054 * Page's order is recorded in page_private(page) field.
1055 * So when we are allocating or freeing one, we can derive the state of the
1056 * other.  That is, if we allocate a small block, and both were
1057 * free, the remainder of the region must be split into blocks.
1058 * If a block is freed, and its buddy is also free, then this
1059 * triggers coalescing into a block of larger size.
1060 *
1061 * -- nyc
1062 */
1063
1064static inline void __free_one_page(struct page *page,
1065                unsigned long pfn,
1066                struct zone *zone, unsigned int order,
1067                int migratetype, fpi_t fpi_flags)
1068{
1069        struct capture_control *capc = task_capc(zone);
1070        unsigned long buddy_pfn;
1071        unsigned long combined_pfn;
1072        unsigned int max_order;
1073        struct page *buddy;
1074        bool to_tail;
1075
1076        max_order = min_t(unsigned int, MAX_ORDER - 1, pageblock_order);
1077
1078        VM_BUG_ON(!zone_is_initialized(zone));
1079        VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
1080
1081        VM_BUG_ON(migratetype == -1);
1082        if (likely(!is_migrate_isolate(migratetype)))
1083                __mod_zone_freepage_state(zone, 1 << order, migratetype);
1084
1085        VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
1086        VM_BUG_ON_PAGE(bad_range(zone, page), page);
1087
1088continue_merging:
1089        while (order < max_order) {
1090                if (compaction_capture(capc, page, order, migratetype)) {
1091                        __mod_zone_freepage_state(zone, -(1 << order),
1092                                                                migratetype);
1093                        return;
1094                }
1095                buddy_pfn = __find_buddy_pfn(pfn, order);
1096                buddy = page + (buddy_pfn - pfn);
1097
1098                if (!pfn_valid_within(buddy_pfn))
1099                        goto done_merging;
1100                if (!page_is_buddy(page, buddy, order))
1101                        goto done_merging;
1102                /*
1103                 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
1104                 * merge with it and move up one order.
1105                 */
1106                if (page_is_guard(buddy))
1107                        clear_page_guard(zone, buddy, order, migratetype);
1108                else
1109                        del_page_from_free_list(buddy, zone, order);
1110                combined_pfn = buddy_pfn & pfn;
1111                page = page + (combined_pfn - pfn);
1112                pfn = combined_pfn;
1113                order++;
1114        }
1115        if (order < MAX_ORDER - 1) {
1116                /* If we are here, it means order is >= pageblock_order.
1117                 * We want to prevent merge between freepages on isolate
1118                 * pageblock and normal pageblock. Without this, pageblock
1119                 * isolation could cause incorrect freepage or CMA accounting.
1120                 *
1121                 * We don't want to hit this code for the more frequent
1122                 * low-order merging.
1123                 */
1124                if (unlikely(has_isolate_pageblock(zone))) {
1125                        int buddy_mt;
1126
1127                        buddy_pfn = __find_buddy_pfn(pfn, order);
1128                        buddy = page + (buddy_pfn - pfn);
1129                        buddy_mt = get_pageblock_migratetype(buddy);
1130
1131                        if (migratetype != buddy_mt
1132                                        && (is_migrate_isolate(migratetype) ||
1133                                                is_migrate_isolate(buddy_mt)))
1134                                goto done_merging;
1135                }
1136                max_order = order + 1;
1137                goto continue_merging;
1138        }
1139
1140done_merging:
1141        set_buddy_order(page, order);
1142
1143        if (fpi_flags & FPI_TO_TAIL)
1144                to_tail = true;
1145        else if (is_shuffle_order(order))
1146                to_tail = shuffle_pick_tail();
1147        else
1148                to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
1149
1150        if (to_tail)
1151                add_to_free_list_tail(page, zone, order, migratetype);
1152        else
1153                add_to_free_list(page, zone, order, migratetype);
1154
1155        /* Notify page reporting subsystem of freed page */
1156        if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
1157                page_reporting_notify_free(order);
1158}
1159
1160/*
1161 * A bad page could be due to a number of fields. Instead of multiple branches,
1162 * try and check multiple fields with one check. The caller must do a detailed
1163 * check if necessary.
1164 */
1165static inline bool page_expected_state(struct page *page,
1166                                        unsigned long check_flags)
1167{
1168        if (unlikely(atomic_read(&page->_mapcount) != -1))
1169                return false;
1170
1171        if (unlikely((unsigned long)page->mapping |
1172                        page_ref_count(page) |
1173#ifdef CONFIG_MEMCG
1174                        page->memcg_data |
1175#endif
1176                        (page->flags & check_flags)))
1177                return false;
1178
1179        return true;
1180}
1181
1182static const char *page_bad_reason(struct page *page, unsigned long flags)
1183{
1184        const char *bad_reason = NULL;
1185
1186        if (unlikely(atomic_read(&page->_mapcount) != -1))
1187                bad_reason = "nonzero mapcount";
1188        if (unlikely(page->mapping != NULL))
1189                bad_reason = "non-NULL mapping";
1190        if (unlikely(page_ref_count(page) != 0))
1191                bad_reason = "nonzero _refcount";
1192        if (unlikely(page->flags & flags)) {
1193                if (flags == PAGE_FLAGS_CHECK_AT_PREP)
1194                        bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
1195                else
1196                        bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
1197        }
1198#ifdef CONFIG_MEMCG
1199        if (unlikely(page->memcg_data))
1200                bad_reason = "page still charged to cgroup";
1201#endif
1202        return bad_reason;
1203}
1204
1205static void check_free_page_bad(struct page *page)
1206{
1207        bad_page(page,
1208                 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
1209}
1210
1211static inline int check_free_page(struct page *page)
1212{
1213        if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
1214                return 0;
1215
1216        /* Something has gone sideways, find it */
1217        check_free_page_bad(page);
1218        return 1;
1219}
1220
1221static int free_tail_pages_check(struct page *head_page, struct page *page)
1222{
1223        int ret = 1;
1224
1225        /*
1226         * We rely page->lru.next never has bit 0 set, unless the page
1227         * is PageTail(). Let's make sure that's true even for poisoned ->lru.
1228         */
1229        BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1230
1231        if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
1232                ret = 0;
1233                goto out;
1234        }
1235        switch (page - head_page) {
1236        case 1:
1237                /* the first tail page: ->mapping may be compound_mapcount() */
1238                if (unlikely(compound_mapcount(page))) {
1239                        bad_page(page, "nonzero compound_mapcount");
1240                        goto out;
1241                }
1242                break;
1243        case 2:
1244                /*
1245                 * the second tail page: ->mapping is
1246                 * deferred_list.next -- ignore value.
1247                 */
1248                break;
1249        default:
1250                if (page->mapping != TAIL_MAPPING) {
1251                        bad_page(page, "corrupted mapping in tail page");
1252                        goto out;
1253                }
1254                break;
1255        }
1256        if (unlikely(!PageTail(page))) {
1257                bad_page(page, "PageTail not set");
1258                goto out;
1259        }
1260        if (unlikely(compound_head(page) != head_page)) {
1261                bad_page(page, "compound_head not consistent");
1262                goto out;
1263        }
1264        ret = 0;
1265out:
1266        page->mapping = NULL;
1267        clear_compound_head(page);
1268        return ret;
1269}
1270
1271static void kernel_init_free_pages(struct page *page, int numpages, bool zero_tags)
1272{
1273        int i;
1274
1275        if (zero_tags) {
1276                for (i = 0; i < numpages; i++)
1277                        tag_clear_highpage(page + i);
1278                return;
1279        }
1280
1281        /* s390's use of memset() could override KASAN redzones. */
1282        kasan_disable_current();
1283        for (i = 0; i < numpages; i++) {
1284                u8 tag = page_kasan_tag(page + i);
1285                page_kasan_tag_reset(page + i);
1286                clear_highpage(page + i);
1287                page_kasan_tag_set(page + i, tag);
1288        }
1289        kasan_enable_current();
1290}
1291
1292static __always_inline bool free_pages_prepare(struct page *page,
1293                        unsigned int order, bool check_free, fpi_t fpi_flags)
1294{
1295        int bad = 0;
1296        bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags);
1297
1298        VM_BUG_ON_PAGE(PageTail(page), page);
1299
1300        trace_mm_page_free(page, order);
1301
1302        if (unlikely(PageHWPoison(page)) && !order) {
1303                /*
1304                 * Do not let hwpoison pages hit pcplists/buddy
1305                 * Untie memcg state and reset page's owner
1306                 */
1307                if (memcg_kmem_enabled() && PageMemcgKmem(page))
1308                        __memcg_kmem_uncharge_page(page, order);
1309                reset_page_owner(page, order);
1310                return false;
1311        }
1312
1313        /*
1314         * Check tail pages before head page information is cleared to
1315         * avoid checking PageCompound for order-0 pages.
1316         */
1317        if (unlikely(order)) {
1318                bool compound = PageCompound(page);
1319                int i;
1320
1321                VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1322
1323                if (compound)
1324                        ClearPageDoubleMap(page);
1325                for (i = 1; i < (1 << order); i++) {
1326                        if (compound)
1327                                bad += free_tail_pages_check(page, page + i);
1328                        if (unlikely(check_free_page(page + i))) {
1329                                bad++;
1330                                continue;
1331                        }
1332                        (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1333                }
1334        }
1335        if (PageMappingFlags(page))
1336                page->mapping = NULL;
1337        if (memcg_kmem_enabled() && PageMemcgKmem(page))
1338                __memcg_kmem_uncharge_page(page, order);
1339        if (check_free)
1340                bad += check_free_page(page);
1341        if (bad)
1342                return false;
1343
1344        page_cpupid_reset_last(page);
1345        page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1346        reset_page_owner(page, order);
1347
1348        if (!PageHighMem(page)) {
1349                debug_check_no_locks_freed(page_address(page),
1350                                           PAGE_SIZE << order);
1351                debug_check_no_obj_freed(page_address(page),
1352                                           PAGE_SIZE << order);
1353        }
1354
1355        kernel_poison_pages(page, 1 << order);
1356
1357        /*
1358         * As memory initialization might be integrated into KASAN,
1359         * kasan_free_pages and kernel_init_free_pages must be
1360         * kept together to avoid discrepancies in behavior.
1361         *
1362         * With hardware tag-based KASAN, memory tags must be set before the
1363         * page becomes unavailable via debug_pagealloc or arch_free_page.
1364         */
1365        if (kasan_has_integrated_init()) {
1366                if (!skip_kasan_poison)
1367                        kasan_free_pages(page, order);
1368        } else {
1369                bool init = want_init_on_free();
1370
1371                if (init)
1372                        kernel_init_free_pages(page, 1 << order, false);
1373                if (!skip_kasan_poison)
1374                        kasan_poison_pages(page, order, init);
1375        }
1376
1377        /*
1378         * arch_free_page() can make the page's contents inaccessible.  s390
1379         * does this.  So nothing which can access the page's contents should
1380         * happen after this.
1381         */
1382        arch_free_page(page, order);
1383
1384        debug_pagealloc_unmap_pages(page, 1 << order);
1385
1386        return true;
1387}
1388
1389#ifdef CONFIG_DEBUG_VM
1390/*
1391 * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed
1392 * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when
1393 * moved from pcp lists to free lists.
1394 */
1395static bool free_pcp_prepare(struct page *page, unsigned int order)
1396{
1397        return free_pages_prepare(page, order, true, FPI_NONE);
1398}
1399
1400static bool bulkfree_pcp_prepare(struct page *page)
1401{
1402        if (debug_pagealloc_enabled_static())
1403                return check_free_page(page);
1404        else
1405                return false;
1406}
1407#else
1408/*
1409 * With DEBUG_VM disabled, order-0 pages being freed are checked only when
1410 * moving from pcp lists to free list in order to reduce overhead. With
1411 * debug_pagealloc enabled, they are checked also immediately when being freed
1412 * to the pcp lists.
1413 */
1414static bool free_pcp_prepare(struct page *page, unsigned int order)
1415{
1416        if (debug_pagealloc_enabled_static())
1417                return free_pages_prepare(page, order, true, FPI_NONE);
1418        else
1419                return free_pages_prepare(page, order, false, FPI_NONE);
1420}
1421
1422static bool bulkfree_pcp_prepare(struct page *page)
1423{
1424        return check_free_page(page);
1425}
1426#endif /* CONFIG_DEBUG_VM */
1427
1428static inline void prefetch_buddy(struct page *page)
1429{
1430        unsigned long pfn = page_to_pfn(page);
1431        unsigned long buddy_pfn = __find_buddy_pfn(pfn, 0);
1432        struct page *buddy = page + (buddy_pfn - pfn);
1433
1434        prefetch(buddy);
1435}
1436
1437/*
1438 * Frees a number of pages from the PCP lists
1439 * Assumes all pages on list are in same zone, and of same order.
1440 * count is the number of pages to free.
1441 *
1442 * If the zone was previously in an "all pages pinned" state then look to
1443 * see if this freeing clears that state.
1444 *
1445 * And clear the zone's pages_scanned counter, to hold off the "all pages are
1446 * pinned" detection logic.
1447 */
1448static void free_pcppages_bulk(struct zone *zone, int count,
1449                                        struct per_cpu_pages *pcp)
1450{
1451        int pindex = 0;
1452        int batch_free = 0;
1453        int nr_freed = 0;
1454        unsigned int order;
1455        int prefetch_nr = READ_ONCE(pcp->batch);
1456        bool isolated_pageblocks;
1457        struct page *page, *tmp;
1458        LIST_HEAD(head);
1459
1460        /*
1461         * Ensure proper count is passed which otherwise would stuck in the
1462         * below while (list_empty(list)) loop.
1463         */
1464        count = min(pcp->count, count);
1465        while (count > 0) {
1466                struct list_head *list;
1467
1468                /*
1469                 * Remove pages from lists in a round-robin fashion. A
1470                 * batch_free count is maintained that is incremented when an
1471                 * empty list is encountered.  This is so more pages are freed
1472                 * off fuller lists instead of spinning excessively around empty
1473                 * lists
1474                 */
1475                do {
1476                        batch_free++;
1477                        if (++pindex == NR_PCP_LISTS)
1478                                pindex = 0;
1479                        list = &pcp->lists[pindex];
1480                } while (list_empty(list));
1481
1482                /* This is the only non-empty list. Free them all. */
1483                if (batch_free == NR_PCP_LISTS)
1484                        batch_free = count;
1485
1486                order = pindex_to_order(pindex);
1487                BUILD_BUG_ON(MAX_ORDER >= (1<<NR_PCP_ORDER_WIDTH));
1488                do {
1489                        page = list_last_entry(list, struct page, lru);
1490                        /* must delete to avoid corrupting pcp list */
1491                        list_del(&page->lru);
1492                        nr_freed += 1 << order;
1493                        count -= 1 << order;
1494
1495                        if (bulkfree_pcp_prepare(page))
1496                                continue;
1497
1498                        /* Encode order with the migratetype */
1499                        page->index <<= NR_PCP_ORDER_WIDTH;
1500                        page->index |= order;
1501
1502                        list_add_tail(&page->lru, &head);
1503
1504                        /*
1505                         * We are going to put the page back to the global
1506                         * pool, prefetch its buddy to speed up later access
1507                         * under zone->lock. It is believed the overhead of
1508                         * an additional test and calculating buddy_pfn here
1509                         * can be offset by reduced memory latency later. To
1510                         * avoid excessive prefetching due to large count, only
1511                         * prefetch buddy for the first pcp->batch nr of pages.
1512                         */
1513                        if (prefetch_nr) {
1514                                prefetch_buddy(page);
1515                                prefetch_nr--;
1516                        }
1517                } while (count > 0 && --batch_free && !list_empty(list));
1518        }
1519        pcp->count -= nr_freed;
1520
1521        /*
1522         * local_lock_irq held so equivalent to spin_lock_irqsave for
1523         * both PREEMPT_RT and non-PREEMPT_RT configurations.
1524         */
1525        spin_lock(&zone->lock);
1526        isolated_pageblocks = has_isolate_pageblock(zone);
1527
1528        /*
1529         * Use safe version since after __free_one_page(),
1530         * page->lru.next will not point to original list.
1531         */
1532        list_for_each_entry_safe(page, tmp, &head, lru) {
1533                int mt = get_pcppage_migratetype(page);
1534
1535                /* mt has been encoded with the order (see above) */
1536                order = mt & NR_PCP_ORDER_MASK;
1537                mt >>= NR_PCP_ORDER_WIDTH;
1538
1539                /* MIGRATE_ISOLATE page should not go to pcplists */
1540                VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1541                /* Pageblock could have been isolated meanwhile */
1542                if (unlikely(isolated_pageblocks))
1543                        mt = get_pageblock_migratetype(page);
1544
1545                __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE);
1546                trace_mm_page_pcpu_drain(page, order, mt);
1547        }
1548        spin_unlock(&zone->lock);
1549}
1550
1551static void free_one_page(struct zone *zone,
1552                                struct page *page, unsigned long pfn,
1553                                unsigned int order,
1554                                int migratetype, fpi_t fpi_flags)
1555{
1556        unsigned long flags;
1557
1558        spin_lock_irqsave(&zone->lock, flags);
1559        if (unlikely(has_isolate_pageblock(zone) ||
1560                is_migrate_isolate(migratetype))) {
1561                migratetype = get_pfnblock_migratetype(page, pfn);
1562        }
1563        __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
1564        spin_unlock_irqrestore(&zone->lock, flags);
1565}
1566
1567static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1568                                unsigned long zone, int nid)
1569{
1570        mm_zero_struct_page(page);
1571        set_page_links(page, zone, nid, pfn);
1572        init_page_count(page);
1573        page_mapcount_reset(page);
1574        page_cpupid_reset_last(page);
1575        page_kasan_tag_reset(page);
1576
1577        INIT_LIST_HEAD(&page->lru);
1578#ifdef WANT_PAGE_VIRTUAL
1579        /* The shift won't overflow because ZONE_NORMAL is below 4G. */
1580        if (!is_highmem_idx(zone))
1581                set_page_address(page, __va(pfn << PAGE_SHIFT));
1582#endif
1583}
1584
1585#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1586static void __meminit init_reserved_page(unsigned long pfn)
1587{
1588        pg_data_t *pgdat;
1589        int nid, zid;
1590
1591        if (!early_page_uninitialised(pfn))
1592                return;
1593
1594        nid = early_pfn_to_nid(pfn);
1595        pgdat = NODE_DATA(nid);
1596
1597        for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1598                struct zone *zone = &pgdat->node_zones[zid];
1599
1600                if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1601                        break;
1602        }
1603        __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
1604}
1605#else
1606static inline void init_reserved_page(unsigned long pfn)
1607{
1608}
1609#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1610
1611/*
1612 * Initialised pages do not have PageReserved set. This function is
1613 * called for each range allocated by the bootmem allocator and
1614 * marks the pages PageReserved. The remaining valid pages are later
1615 * sent to the buddy page allocator.
1616 */
1617void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1618{
1619        unsigned long start_pfn = PFN_DOWN(start);
1620        unsigned long end_pfn = PFN_UP(end);
1621
1622        for (; start_pfn < end_pfn; start_pfn++) {
1623                if (pfn_valid(start_pfn)) {
1624                        struct page *page = pfn_to_page(start_pfn);
1625
1626                        init_reserved_page(start_pfn);
1627
1628                        /* Avoid false-positive PageTail() */
1629                        INIT_LIST_HEAD(&page->lru);
1630
1631                        /*
1632                         * no need for atomic set_bit because the struct
1633                         * page is not visible yet so nobody should
1634                         * access it yet.
1635                         */
1636                        __SetPageReserved(page);
1637                }
1638        }
1639}
1640
1641static void __free_pages_ok(struct page *page, unsigned int order,
1642                            fpi_t fpi_flags)
1643{
1644        unsigned long flags;
1645        int migratetype;
1646        unsigned long pfn = page_to_pfn(page);
1647        struct zone *zone = page_zone(page);
1648
1649        if (!free_pages_prepare(page, order, true, fpi_flags))
1650                return;
1651
1652        migratetype = get_pfnblock_migratetype(page, pfn);
1653
1654        spin_lock_irqsave(&zone->lock, flags);
1655        if (unlikely(has_isolate_pageblock(zone) ||
1656                is_migrate_isolate(migratetype))) {
1657                migratetype = get_pfnblock_migratetype(page, pfn);
1658        }
1659        __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
1660        spin_unlock_irqrestore(&zone->lock, flags);
1661
1662        __count_vm_events(PGFREE, 1 << order);
1663}
1664
1665void __free_pages_core(struct page *page, unsigned int order)
1666{
1667        unsigned int nr_pages = 1 << order;
1668        struct page *p = page;
1669        unsigned int loop;
1670
1671        /*
1672         * When initializing the memmap, __init_single_page() sets the refcount
1673         * of all pages to 1 ("allocated"/"not free"). We have to set the
1674         * refcount of all involved pages to 0.
1675         */
1676        prefetchw(p);
1677        for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1678                prefetchw(p + 1);
1679                __ClearPageReserved(p);
1680                set_page_count(p, 0);
1681        }
1682        __ClearPageReserved(p);
1683        set_page_count(p, 0);
1684
1685        atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
1686
1687        /*
1688         * Bypass PCP and place fresh pages right to the tail, primarily
1689         * relevant for memory onlining.
1690         */
1691        __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON);
1692}
1693
1694#ifdef CONFIG_NUMA
1695
1696/*
1697 * During memory init memblocks map pfns to nids. The search is expensive and
1698 * this caches recent lookups. The implementation of __early_pfn_to_nid
1699 * treats start/end as pfns.
1700 */
1701struct mminit_pfnnid_cache {
1702        unsigned long last_start;
1703        unsigned long last_end;
1704        int last_nid;
1705};
1706
1707static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1708
1709/*
1710 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
1711 */
1712static int __meminit __early_pfn_to_nid(unsigned long pfn,
1713                                        struct mminit_pfnnid_cache *state)
1714{
1715        unsigned long start_pfn, end_pfn;
1716        int nid;
1717
1718        if (state->last_start <= pfn && pfn < state->last_end)
1719                return state->last_nid;
1720
1721        nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
1722        if (nid != NUMA_NO_NODE) {
1723                state->last_start = start_pfn;
1724                state->last_end = end_pfn;
1725                state->last_nid = nid;
1726        }
1727
1728        return nid;
1729}
1730
1731int __meminit early_pfn_to_nid(unsigned long pfn)
1732{
1733        static DEFINE_SPINLOCK(early_pfn_lock);
1734        int nid;
1735
1736        spin_lock(&early_pfn_lock);
1737        nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1738        if (nid < 0)
1739                nid = first_online_node;
1740        spin_unlock(&early_pfn_lock);
1741
1742        return nid;
1743}
1744#endif /* CONFIG_NUMA */
1745
1746void __init memblock_free_pages(struct page *page, unsigned long pfn,
1747                                                        unsigned int order)
1748{
1749        if (early_page_uninitialised(pfn))
1750                return;
1751        __free_pages_core(page, order);
1752}
1753
1754/*
1755 * Check that the whole (or subset of) a pageblock given by the interval of
1756 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1757 * with the migration of free compaction scanner. The scanners then need to
1758 * use only pfn_valid_within() check for arches that allow holes within
1759 * pageblocks.
1760 *
1761 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1762 *
1763 * It's possible on some configurations to have a setup like node0 node1 node0
1764 * i.e. it's possible that all pages within a zones range of pages do not
1765 * belong to a single zone. We assume that a border between node0 and node1
1766 * can occur within a single pageblock, but not a node0 node1 node0
1767 * interleaving within a single pageblock. It is therefore sufficient to check
1768 * the first and last page of a pageblock and avoid checking each individual
1769 * page in a pageblock.
1770 */
1771struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1772                                     unsigned long end_pfn, struct zone *zone)
1773{
1774        struct page *start_page;
1775        struct page *end_page;
1776
1777        /* end_pfn is one past the range we are checking */
1778        end_pfn--;
1779
1780        if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1781                return NULL;
1782
1783        start_page = pfn_to_online_page(start_pfn);
1784        if (!start_page)
1785                return NULL;
1786
1787        if (page_zone(start_page) != zone)
1788                return NULL;
1789
1790        end_page = pfn_to_page(end_pfn);
1791
1792        /* This gives a shorter code than deriving page_zone(end_page) */
1793        if (page_zone_id(start_page) != page_zone_id(end_page))
1794                return NULL;
1795
1796        return start_page;
1797}
1798
1799void set_zone_contiguous(struct zone *zone)
1800{
1801        unsigned long block_start_pfn = zone->zone_start_pfn;
1802        unsigned long block_end_pfn;
1803
1804        block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1805        for (; block_start_pfn < zone_end_pfn(zone);
1806                        block_start_pfn = block_end_pfn,
1807                         block_end_pfn += pageblock_nr_pages) {
1808
1809                block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1810
1811                if (!__pageblock_pfn_to_page(block_start_pfn,
1812                                             block_end_pfn, zone))
1813                        return;
1814                cond_resched();
1815        }
1816
1817        /* We confirm that there is no hole */
1818        zone->contiguous = true;
1819}
1820
1821void clear_zone_contiguous(struct zone *zone)
1822{
1823        zone->contiguous = false;
1824}
1825
1826#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1827static void __init deferred_free_range(unsigned long pfn,
1828                                       unsigned long nr_pages)
1829{
1830        struct page *page;
1831        unsigned long i;
1832
1833        if (!nr_pages)
1834                return;
1835
1836        page = pfn_to_page(pfn);
1837
1838        /* Free a large naturally-aligned chunk if possible */
1839        if (nr_pages == pageblock_nr_pages &&
1840            (pfn & (pageblock_nr_pages - 1)) == 0) {
1841                set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1842                __free_pages_core(page, pageblock_order);
1843                return;
1844        }
1845
1846        for (i = 0; i < nr_pages; i++, page++, pfn++) {
1847                if ((pfn & (pageblock_nr_pages - 1)) == 0)
1848                        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1849                __free_pages_core(page, 0);
1850        }
1851}
1852
1853/* Completion tracking for deferred_init_memmap() threads */
1854static atomic_t pgdat_init_n_undone __initdata;
1855static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1856
1857static inline void __init pgdat_init_report_one_done(void)
1858{
1859        if (atomic_dec_and_test(&pgdat_init_n_undone))
1860                complete(&pgdat_init_all_done_comp);
1861}
1862
1863/*
1864 * Returns true if page needs to be initialized or freed to buddy allocator.
1865 *
1866 * First we check if pfn is valid on architectures where it is possible to have
1867 * holes within pageblock_nr_pages. On systems where it is not possible, this
1868 * function is optimized out.
1869 *
1870 * Then, we check if a current large page is valid by only checking the validity
1871 * of the head pfn.
1872 */
1873static inline bool __init deferred_pfn_valid(unsigned long pfn)
1874{
1875        if (!pfn_valid_within(pfn))
1876                return false;
1877        if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn))
1878                return false;
1879        return true;
1880}
1881
1882/*
1883 * Free pages to buddy allocator. Try to free aligned pages in
1884 * pageblock_nr_pages sizes.
1885 */
1886static void __init deferred_free_pages(unsigned long pfn,
1887                                       unsigned long end_pfn)
1888{
1889        unsigned long nr_pgmask = pageblock_nr_pages - 1;
1890        unsigned long nr_free = 0;
1891
1892        for (; pfn < end_pfn; pfn++) {
1893                if (!deferred_pfn_valid(pfn)) {
1894                        deferred_free_range(pfn - nr_free, nr_free);
1895                        nr_free = 0;
1896                } else if (!(pfn & nr_pgmask)) {
1897                        deferred_free_range(pfn - nr_free, nr_free);
1898                        nr_free = 1;
1899                } else {
1900                        nr_free++;
1901                }
1902        }
1903        /* Free the last block of pages to allocator */
1904        deferred_free_range(pfn - nr_free, nr_free);
1905}
1906
1907/*
1908 * Initialize struct pages.  We minimize pfn page lookups and scheduler checks
1909 * by performing it only once every pageblock_nr_pages.
1910 * Return number of pages initialized.
1911 */
1912static unsigned long  __init deferred_init_pages(struct zone *zone,
1913                                                 unsigned long pfn,
1914                                                 unsigned long end_pfn)
1915{
1916        unsigned long nr_pgmask = pageblock_nr_pages - 1;
1917        int nid = zone_to_nid(zone);
1918        unsigned long nr_pages = 0;
1919        int zid = zone_idx(zone);
1920        struct page *page = NULL;
1921
1922        for (; pfn < end_pfn; pfn++) {
1923                if (!deferred_pfn_valid(pfn)) {
1924                        page = NULL;
1925                        continue;
1926                } else if (!page || !(pfn & nr_pgmask)) {
1927                        page = pfn_to_page(pfn);
1928                } else {
1929                        page++;
1930                }
1931                __init_single_page(page, pfn, zid, nid);
1932                nr_pages++;
1933        }
1934        return (nr_pages);
1935}
1936
1937/*
1938 * This function is meant to pre-load the iterator for the zone init.
1939 * Specifically it walks through the ranges until we are caught up to the
1940 * first_init_pfn value and exits there. If we never encounter the value we
1941 * return false indicating there are no valid ranges left.
1942 */
1943static bool __init
1944deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
1945                                    unsigned long *spfn, unsigned long *epfn,
1946                                    unsigned long first_init_pfn)
1947{
1948        u64 j;
1949
1950        /*
1951         * Start out by walking through the ranges in this zone that have
1952         * already been initialized. We don't need to do anything with them
1953         * so we just need to flush them out of the system.
1954         */
1955        for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
1956                if (*epfn <= first_init_pfn)
1957                        continue;
1958                if (*spfn < first_init_pfn)
1959                        *spfn = first_init_pfn;
1960                *i = j;
1961                return true;
1962        }
1963
1964        return false;
1965}
1966
1967/*
1968 * Initialize and free pages. We do it in two loops: first we initialize
1969 * struct page, then free to buddy allocator, because while we are
1970 * freeing pages we can access pages that are ahead (computing buddy
1971 * page in __free_one_page()).
1972 *
1973 * In order to try and keep some memory in the cache we have the loop
1974 * broken along max page order boundaries. This way we will not cause
1975 * any issues with the buddy page computation.
1976 */
1977static unsigned long __init
1978deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
1979                       unsigned long *end_pfn)
1980{
1981        unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
1982        unsigned long spfn = *start_pfn, epfn = *end_pfn;
1983        unsigned long nr_pages = 0;
1984        u64 j = *i;
1985
1986        /* First we loop through and initialize the page values */
1987        for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
1988                unsigned long t;
1989
1990                if (mo_pfn <= *start_pfn)
1991                        break;
1992
1993                t = min(mo_pfn, *end_pfn);
1994                nr_pages += deferred_init_pages(zone, *start_pfn, t);
1995
1996                if (mo_pfn < *end_pfn) {
1997                        *start_pfn = mo_pfn;
1998                        break;
1999                }
2000        }
2001
2002        /* Reset values and now loop through freeing pages as needed */
2003        swap(j, *i);
2004
2005        for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
2006                unsigned long t;
2007
2008                if (mo_pfn <= spfn)
2009                        break;
2010
2011                t = min(mo_pfn, epfn);
2012                deferred_free_pages(spfn, t);
2013
2014                if (mo_pfn <= epfn)
2015                        break;
2016        }
2017
2018        return nr_pages;
2019}
2020
2021static void __init
2022deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
2023                           void *arg)
2024{
2025        unsigned long spfn, epfn;
2026        struct zone *zone = arg;
2027        u64 i;
2028
2029        deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
2030
2031        /*
2032         * Initialize and free pages in MAX_ORDER sized increments so that we
2033         * can avoid introducing any issues with the buddy allocator.
2034         */
2035        while (spfn < end_pfn) {
2036                deferred_init_maxorder(&i, zone, &spfn, &epfn);
2037                cond_resched();
2038        }
2039}
2040
2041/* An arch may override for more concurrency. */
2042__weak int __init
2043deferred_page_init_max_threads(const struct cpumask *node_cpumask)
2044{
2045        return 1;
2046}
2047
2048/* Initialise remaining memory on a node */
2049static int __init deferred_init_memmap(void *data)
2050{
2051        pg_data_t *pgdat = data;
2052        const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2053        unsigned long spfn = 0, epfn = 0;
2054        unsigned long first_init_pfn, flags;
2055        unsigned long start = jiffies;
2056        struct zone *zone;
2057        int zid, max_threads;
2058        u64 i;
2059
2060        /* Bind memory initialisation thread to a local node if possible */
2061        if (!cpumask_empty(cpumask))
2062                set_cpus_allowed_ptr(current, cpumask);
2063
2064        pgdat_resize_lock(pgdat, &flags);
2065        first_init_pfn = pgdat->first_deferred_pfn;
2066        if (first_init_pfn == ULONG_MAX) {
2067                pgdat_resize_unlock(pgdat, &flags);
2068                pgdat_init_report_one_done();
2069                return 0;
2070        }
2071
2072        /* Sanity check boundaries */
2073        BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
2074        BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
2075        pgdat->first_deferred_pfn = ULONG_MAX;
2076
2077        /*
2078         * Once we unlock here, the zone cannot be grown anymore, thus if an
2079         * interrupt thread must allocate this early in boot, zone must be
2080         * pre-grown prior to start of deferred page initialization.
2081         */
2082        pgdat_resize_unlock(pgdat, &flags);
2083
2084        /* Only the highest zone is deferred so find it */
2085        for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2086                zone = pgdat->node_zones + zid;
2087                if (first_init_pfn < zone_end_pfn(zone))
2088                        break;
2089        }
2090
2091        /* If the zone is empty somebody else may have cleared out the zone */
2092        if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2093                                                 first_init_pfn))
2094                goto zone_empty;
2095
2096        max_threads = deferred_page_init_max_threads(cpumask);
2097
2098        while (spfn < epfn) {
2099                unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
2100                struct padata_mt_job job = {
2101                        .thread_fn   = deferred_init_memmap_chunk,
2102                        .fn_arg      = zone,
2103                        .start       = spfn,
2104                        .size        = epfn_align - spfn,
2105                        .align       = PAGES_PER_SECTION,
2106                        .min_chunk   = PAGES_PER_SECTION,
2107                        .max_threads = max_threads,
2108                };
2109
2110                padata_do_multithreaded(&job);
2111                deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2112                                                    epfn_align);
2113        }
2114zone_empty:
2115        /* Sanity check that the next zone really is unpopulated */
2116        WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
2117
2118        pr_info("node %d deferred pages initialised in %ums\n",
2119                pgdat->node_id, jiffies_to_msecs(jiffies - start));
2120
2121        pgdat_init_report_one_done();
2122        return 0;
2123}
2124
2125/*
2126 * If this zone has deferred pages, try to grow it by initializing enough
2127 * deferred pages to satisfy the allocation specified by order, rounded up to
2128 * the nearest PAGES_PER_SECTION boundary.  So we're adding memory in increments
2129 * of SECTION_SIZE bytes by initializing struct pages in increments of
2130 * PAGES_PER_SECTION * sizeof(struct page) bytes.
2131 *
2132 * Return true when zone was grown, otherwise return false. We return true even
2133 * when we grow less than requested, to let the caller decide if there are
2134 * enough pages to satisfy the allocation.
2135 *
2136 * Note: We use noinline because this function is needed only during boot, and
2137 * it is called from a __ref function _deferred_grow_zone. This way we are
2138 * making sure that it is not inlined into permanent text section.
2139 */
2140static noinline bool __init
2141deferred_grow_zone(struct zone *zone, unsigned int order)
2142{
2143        unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
2144        pg_data_t *pgdat = zone->zone_pgdat;
2145        unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
2146        unsigned long spfn, epfn, flags;
2147        unsigned long nr_pages = 0;
2148        u64 i;
2149
2150        /* Only the last zone may have deferred pages */
2151        if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
2152                return false;
2153
2154        pgdat_resize_lock(pgdat, &flags);
2155
2156        /*
2157         * If someone grew this zone while we were waiting for spinlock, return
2158         * true, as there might be enough pages already.
2159         */
2160        if (first_deferred_pfn != pgdat->first_deferred_pfn) {
2161                pgdat_resize_unlock(pgdat, &flags);
2162                return true;
2163        }
2164
2165        /* If the zone is empty somebody else may have cleared out the zone */
2166        if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2167                                                 first_deferred_pfn)) {
2168                pgdat->first_deferred_pfn = ULONG_MAX;
2169                pgdat_resize_unlock(pgdat, &flags);
2170                /* Retry only once. */
2171                return first_deferred_pfn != ULONG_MAX;
2172        }
2173
2174        /*
2175         * Initialize and free pages in MAX_ORDER sized increments so
2176         * that we can avoid introducing any issues with the buddy
2177         * allocator.
2178         */
2179        while (spfn < epfn) {
2180                /* update our first deferred PFN for this section */
2181                first_deferred_pfn = spfn;
2182
2183                nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
2184                touch_nmi_watchdog();
2185
2186                /* We should only stop along section boundaries */
2187                if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2188                        continue;
2189
2190                /* If our quota has been met we can stop here */
2191                if (nr_pages >= nr_pages_needed)
2192                        break;
2193        }
2194
2195        pgdat->first_deferred_pfn = spfn;
2196        pgdat_resize_unlock(pgdat, &flags);
2197
2198        return nr_pages > 0;
2199}
2200
2201/*
2202 * deferred_grow_zone() is __init, but it is called from
2203 * get_page_from_freelist() during early boot until deferred_pages permanently
2204 * disables this call. This is why we have refdata wrapper to avoid warning,
2205 * and to ensure that the function body gets unloaded.
2206 */
2207static bool __ref
2208_deferred_grow_zone(struct zone *zone, unsigned int order)
2209{
2210        return deferred_grow_zone(zone, order);
2211}
2212
2213#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
2214
2215void __init page_alloc_init_late(void)
2216{
2217        struct zone *zone;
2218        int nid;
2219
2220#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
2221
2222        /* There will be num_node_state(N_MEMORY) threads */
2223        atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
2224        for_each_node_state(nid, N_MEMORY) {
2225                kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
2226        }
2227
2228        /* Block until all are initialised */
2229        wait_for_completion(&pgdat_init_all_done_comp);
2230
2231        /*
2232         * We initialized the rest of the deferred pages.  Permanently disable
2233         * on-demand struct page initialization.
2234         */
2235        static_branch_disable(&deferred_pages);
2236
2237        /* Reinit limits that are based on free pages after the kernel is up */
2238        files_maxfiles_init();
2239#endif
2240
2241        buffer_init();
2242
2243        /* Discard memblock private memory */
2244        memblock_discard();
2245
2246        for_each_node_state(nid, N_MEMORY)
2247                shuffle_free_memory(NODE_DATA(nid));
2248
2249        for_each_populated_zone(zone)
2250                set_zone_contiguous(zone);
2251}
2252
2253#ifdef CONFIG_CMA
2254/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
2255void __init init_cma_reserved_pageblock(struct page *page)
2256{
2257        unsigned i = pageblock_nr_pages;
2258        struct page *p = page;
2259
2260        do {
2261                __ClearPageReserved(p);
2262                set_page_count(p, 0);
2263        } while (++p, --i);
2264
2265        set_pageblock_migratetype(page, MIGRATE_CMA);
2266
2267        if (pageblock_order >= MAX_ORDER) {
2268                i = pageblock_nr_pages;
2269                p = page;
2270                do {
2271                        set_page_refcounted(p);
2272                        __free_pages(p, MAX_ORDER - 1);
2273                        p += MAX_ORDER_NR_PAGES;
2274                } while (i -= MAX_ORDER_NR_PAGES);
2275        } else {
2276                set_page_refcounted(page);
2277                __free_pages(page, pageblock_order);
2278        }
2279
2280        adjust_managed_page_count(page, pageblock_nr_pages);
2281        page_zone(page)->cma_pages += pageblock_nr_pages;
2282}
2283#endif
2284
2285/*
2286 * The order of subdivision here is critical for the IO subsystem.
2287 * Please do not alter this order without good reasons and regression
2288 * testing. Specifically, as large blocks of memory are subdivided,
2289 * the order in which smaller blocks are delivered depends on the order
2290 * they're subdivided in this function. This is the primary factor
2291 * influencing the order in which pages are delivered to the IO
2292 * subsystem according to empirical testing, and this is also justified
2293 * by considering the behavior of a buddy system containing a single
2294 * large block of memory acted on by a series of small allocations.
2295 * This behavior is a critical factor in sglist merging's success.
2296 *
2297 * -- nyc
2298 */
2299static inline void expand(struct zone *zone, struct page *page,
2300        int low, int high, int migratetype)
2301{
2302        unsigned long size = 1 << high;
2303
2304        while (high > low) {
2305                high--;
2306                size >>= 1;
2307                VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
2308
2309                /*
2310                 * Mark as guard pages (or page), that will allow to
2311                 * merge back to allocator when buddy will be freed.
2312                 * Corresponding page table entries will not be touched,
2313                 * pages will stay not present in virtual address space
2314                 */
2315                if (set_page_guard(zone, &page[size], high, migratetype))
2316                        continue;
2317
2318                add_to_free_list(&page[size], zone, high, migratetype);
2319                set_buddy_order(&page[size], high);
2320        }
2321}
2322
2323static void check_new_page_bad(struct page *page)
2324{
2325        if (unlikely(page->flags & __PG_HWPOISON)) {
2326                /* Don't complain about hwpoisoned pages */
2327                page_mapcount_reset(page); /* remove PageBuddy */
2328                return;
2329        }
2330
2331        bad_page(page,
2332                 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
2333}
2334
2335/*
2336 * This page is about to be returned from the page allocator
2337 */
2338static inline int check_new_page(struct page *page)
2339{
2340        if (likely(page_expected_state(page,
2341                                PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
2342                return 0;
2343
2344        check_new_page_bad(page);
2345        return 1;
2346}
2347
2348#ifdef CONFIG_DEBUG_VM
2349/*
2350 * With DEBUG_VM enabled, order-0 pages are checked for expected state when
2351 * being allocated from pcp lists. With debug_pagealloc also enabled, they are
2352 * also checked when pcp lists are refilled from the free lists.
2353 */
2354static inline bool check_pcp_refill(struct page *page)
2355{
2356        if (debug_pagealloc_enabled_static())
2357                return check_new_page(page);
2358        else
2359                return false;
2360}
2361
2362static inline bool check_new_pcp(struct page *page)
2363{
2364        return check_new_page(page);
2365}
2366#else
2367/*
2368 * With DEBUG_VM disabled, free order-0 pages are checked for expected state
2369 * when pcp lists are being refilled from the free lists. With debug_pagealloc
2370 * enabled, they are also checked when being allocated from the pcp lists.
2371 */
2372static inline bool check_pcp_refill(struct page *page)
2373{
2374        return check_new_page(page);
2375}
2376static inline bool check_new_pcp(struct page *page)
2377{
2378        if (debug_pagealloc_enabled_static())
2379                return check_new_page(page);
2380        else
2381                return false;
2382}
2383#endif /* CONFIG_DEBUG_VM */
2384
2385static bool check_new_pages(struct page *page, unsigned int order)
2386{
2387        int i;
2388        for (i = 0; i < (1 << order); i++) {
2389                struct page *p = page + i;
2390
2391                if (unlikely(check_new_page(p)))
2392                        return true;
2393        }
2394
2395        return false;
2396}
2397
2398inline void post_alloc_hook(struct page *page, unsigned int order,
2399                                gfp_t gfp_flags)
2400{
2401        set_page_private(page, 0);
2402        set_page_refcounted(page);
2403
2404        arch_alloc_page(page, order);
2405        debug_pagealloc_map_pages(page, 1 << order);
2406
2407        /*
2408         * Page unpoisoning must happen before memory initialization.
2409         * Otherwise, the poison pattern will be overwritten for __GFP_ZERO
2410         * allocations and the page unpoisoning code will complain.
2411         */
2412        kernel_unpoison_pages(page, 1 << order);
2413
2414        /*
2415         * As memory initialization might be integrated into KASAN,
2416         * kasan_alloc_pages and kernel_init_free_pages must be
2417         * kept together to avoid discrepancies in behavior.
2418         */
2419        if (kasan_has_integrated_init()) {
2420                kasan_alloc_pages(page, order, gfp_flags);
2421        } else {
2422                bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags);
2423
2424                kasan_unpoison_pages(page, order, init);
2425                if (init)
2426                        kernel_init_free_pages(page, 1 << order,
2427                                               gfp_flags & __GFP_ZEROTAGS);
2428        }
2429
2430        set_page_owner(page, order, gfp_flags);
2431}
2432
2433static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
2434                                                        unsigned int alloc_flags)
2435{
2436        post_alloc_hook(page, order, gfp_flags);
2437
2438        if (order && (gfp_flags & __GFP_COMP))
2439                prep_compound_page(page, order);
2440
2441        /*
2442         * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
2443         * allocate the page. The expectation is that the caller is taking
2444         * steps that will free more memory. The caller should avoid the page
2445         * being used for !PFMEMALLOC purposes.
2446         */
2447        if (alloc_flags & ALLOC_NO_WATERMARKS)
2448                set_page_pfmemalloc(page);
2449        else
2450                clear_page_pfmemalloc(page);
2451}
2452
2453/*
2454 * Go through the free lists for the given migratetype and remove
2455 * the smallest available page from the freelists
2456 */
2457static __always_inline
2458struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
2459                                                int migratetype)
2460{
2461        unsigned int current_order;
2462        struct free_area *area;
2463        struct page *page;
2464
2465        /* Find a page of the appropriate size in the preferred list */
2466        for (current_order = order; current_order < MAX_ORDER; ++current_order) {
2467                area = &(zone->free_area[current_order]);
2468                page = get_page_from_free_area(area, migratetype);
2469                if (!page)
2470                        continue;
2471                del_page_from_free_list(page, zone, current_order);
2472                expand(zone, page, order, current_order, migratetype);
2473                set_pcppage_migratetype(page, migratetype);
2474                return page;
2475        }
2476
2477        return NULL;
2478}
2479
2480
2481/*
2482 * This array describes the order lists are fallen back to when
2483 * the free lists for the desirable migrate type are depleted
2484 */
2485static int fallbacks[MIGRATE_TYPES][3] = {
2486        [MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_TYPES },
2487        [MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
2488        [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_TYPES },
2489#ifdef CONFIG_CMA
2490        [MIGRATE_CMA]         = { MIGRATE_TYPES }, /* Never used */
2491#endif
2492#ifdef CONFIG_MEMORY_ISOLATION
2493        [MIGRATE_ISOLATE]     = { MIGRATE_TYPES }, /* Never used */
2494#endif
2495};
2496
2497#ifdef CONFIG_CMA
2498static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2499                                        unsigned int order)
2500{
2501        return __rmqueue_smallest(zone, order, MIGRATE_CMA);
2502}
2503#else
2504static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
2505                                        unsigned int order) { return NULL; }
2506#endif
2507
2508/*
2509 * Move the free pages in a range to the freelist tail of the requested type.
2510 * Note that start_page and end_pages are not aligned on a pageblock
2511 * boundary. If alignment is required, use move_freepages_block()
2512 */
2513static int move_freepages(struct zone *zone,
2514                          unsigned long start_pfn, unsigned long end_pfn,
2515                          int migratetype, int *num_movable)
2516{
2517        struct page *page;
2518        unsigned long pfn;
2519        unsigned int order;
2520        int pages_moved = 0;
2521
2522        for (pfn = start_pfn; pfn <= end_pfn;) {
2523                if (!pfn_valid_within(pfn)) {
2524                        pfn++;
2525                        continue;
2526                }
2527
2528                page = pfn_to_page(pfn);
2529                if (!PageBuddy(page)) {
2530                        /*
2531                         * We assume that pages that could be isolated for
2532                         * migration are movable. But we don't actually try
2533                         * isolating, as that would be expensive.
2534                         */
2535                        if (num_movable &&
2536                                        (PageLRU(page) || __PageMovable(page)))
2537                                (*num_movable)++;
2538                        pfn++;
2539                        continue;
2540                }
2541
2542                /* Make sure we are not inadvertently changing nodes */
2543                VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
2544                VM_BUG_ON_PAGE(page_zone(page) != zone, page);
2545
2546                order = buddy_order(page);
2547                move_to_free_list(page, zone, order, migratetype);
2548                pfn += 1 << order;
2549                pages_moved += 1 << order;
2550        }
2551
2552        return pages_moved;
2553}
2554
2555int move_freepages_block(struct zone *zone, struct page *page,
2556                                int migratetype, int *num_movable)
2557{
2558        unsigned long start_pfn, end_pfn, pfn;
2559
2560        if (num_movable)
2561                *num_movable = 0;
2562
2563        pfn = page_to_pfn(page);
2564        start_pfn = pfn & ~(pageblock_nr_pages - 1);
2565        end_pfn = start_pfn + pageblock_nr_pages - 1;
2566
2567        /* Do not cross zone boundaries */
2568        if (!zone_spans_pfn(zone, start_pfn))
2569                start_pfn = pfn;
2570        if (!zone_spans_pfn(zone, end_pfn))
2571                return 0;
2572
2573        return move_freepages(zone, start_pfn, end_pfn, migratetype,
2574                                                                num_movable);
2575}
2576
2577static void change_pageblock_range(struct page *pageblock_page,
2578                                        int start_order, int migratetype)
2579{
2580        int nr_pageblocks = 1 << (start_order - pageblock_order);
2581
2582        while (nr_pageblocks--) {
2583                set_pageblock_migratetype(pageblock_page, migratetype);
2584                pageblock_page += pageblock_nr_pages;
2585        }
2586}
2587
2588/*
2589 * When we are falling back to another migratetype during allocation, try to
2590 * steal extra free pages from the same pageblocks to satisfy further
2591 * allocations, instead of polluting multiple pageblocks.
2592 *
2593 * If we are stealing a relatively large buddy page, it is likely there will
2594 * be more free pages in the pageblock, so try to steal them all. For
2595 * reclaimable and unmovable allocations, we steal regardless of page size,
2596 * as fragmentation caused by those allocations polluting movable pageblocks
2597 * is worse than movable allocations stealing from unmovable and reclaimable
2598 * pageblocks.
2599 */
2600static bool can_steal_fallback(unsigned int order, int start_mt)
2601{
2602        /*
2603         * Leaving this order check is intended, although there is
2604         * relaxed order check in next check. The reason is that
2605         * we can actually steal whole pageblock if this condition met,
2606         * but, below check doesn't guarantee it and that is just heuristic
2607         * so could be changed anytime.
2608         */
2609        if (order >= pageblock_order)
2610                return true;
2611
2612        if (order >= pageblock_order / 2 ||
2613                start_mt == MIGRATE_RECLAIMABLE ||
2614                start_mt == MIGRATE_UNMOVABLE ||
2615                page_group_by_mobility_disabled)
2616                return true;
2617
2618        return false;
2619}
2620
2621static inline bool boost_watermark(struct zone *zone)
2622{
2623        unsigned long max_boost;
2624
2625        if (!watermark_boost_factor)
2626                return false;
2627        /*
2628         * Don't bother in zones that are unlikely to produce results.
2629         * On small machines, including kdump capture kernels running
2630         * in a small area, boosting the watermark can cause an out of
2631         * memory situation immediately.
2632         */
2633        if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
2634                return false;
2635
2636        max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2637                        watermark_boost_factor, 10000);
2638
2639        /*
2640         * high watermark may be uninitialised if fragmentation occurs
2641         * very early in boot so do not boost. We do not fall
2642         * through and boost by pageblock_nr_pages as failing
2643         * allocations that early means that reclaim is not going
2644         * to help and it may even be impossible to reclaim the
2645         * boosted watermark resulting in a hang.
2646         */
2647        if (!max_boost)
2648                return false;
2649
2650        max_boost = max(pageblock_nr_pages, max_boost);
2651
2652        zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2653                max_boost);
2654
2655        return true;
2656}
2657
2658/*
2659 * This function implements actual steal behaviour. If order is large enough,
2660 * we can steal whole pageblock. If not, we first move freepages in this
2661 * pageblock to our migratetype and determine how many already-allocated pages
2662 * are there in the pageblock with a compatible migratetype. If at least half
2663 * of pages are free or compatible, we can change migratetype of the pageblock
2664 * itself, so pages freed in the future will be put on the correct free list.
2665 */
2666static void steal_suitable_fallback(struct zone *zone, struct page *page,
2667                unsigned int alloc_flags, int start_type, bool whole_block)
2668{
2669        unsigned int current_order = buddy_order(page);
2670        int free_pages, movable_pages, alike_pages;
2671        int old_block_type;
2672
2673        old_block_type = get_pageblock_migratetype(page);
2674
2675        /*
2676         * This can happen due to races and we want to prevent broken
2677         * highatomic accounting.
2678         */
2679        if (is_migrate_highatomic(old_block_type))
2680                goto single_page;
2681
2682        /* Take ownership for orders >= pageblock_order */
2683        if (current_order >= pageblock_order) {
2684                change_pageblock_range(page, current_order, start_type);
2685                goto single_page;
2686        }
2687
2688        /*
2689         * Boost watermarks to increase reclaim pressure to reduce the
2690         * likelihood of future fallbacks. Wake kswapd now as the node
2691         * may be balanced overall and kswapd will not wake naturally.
2692         */
2693        if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
2694                set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
2695
2696        /* We are not allowed to try stealing from the whole block */
2697        if (!whole_block)
2698                goto single_page;
2699
2700        free_pages = move_freepages_block(zone, page, start_type,
2701                                                &movable_pages);
2702        /*
2703         * Determine how many pages are compatible with our allocation.
2704         * For movable allocation, it's the number of movable pages which
2705         * we just obtained. For other types it's a bit more tricky.
2706         */
2707        if (start_type == MIGRATE_MOVABLE) {
2708                alike_pages = movable_pages;
2709        } else {
2710                /*
2711                 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2712                 * to MOVABLE pageblock, consider all non-movable pages as
2713                 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2714                 * vice versa, be conservative since we can't distinguish the
2715                 * exact migratetype of non-movable pages.
2716                 */
2717                if (old_block_type == MIGRATE_MOVABLE)
2718                        alike_pages = pageblock_nr_pages
2719                                                - (free_pages + movable_pages);
2720                else
2721                        alike_pages = 0;
2722        }
2723
2724        /* moving whole block can fail due to zone boundary conditions */
2725        if (!free_pages)
2726                goto single_page;
2727
2728        /*
2729         * If a sufficient number of pages in the block are either free or of
2730         * comparable migratability as our allocation, claim the whole block.
2731         */
2732        if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2733                        page_group_by_mobility_disabled)
2734                set_pageblock_migratetype(page, start_type);
2735
2736        return;
2737
2738single_page:
2739        move_to_free_list(page, zone, current_order, start_type);
2740}
2741
2742/*
2743 * Check whether there is a suitable fallback freepage with requested order.
2744 * If only_stealable is true, this function returns fallback_mt only if
2745 * we can steal other freepages all together. This would help to reduce
2746 * fragmentation due to mixed migratetype pages in one pageblock.
2747 */
2748int find_suitable_fallback(struct free_area *area, unsigned int order,
2749                        int migratetype, bool only_stealable, bool *can_steal)
2750{
2751        int i;
2752        int fallback_mt;
2753
2754        if (area->nr_free == 0)
2755                return -1;
2756
2757        *can_steal = false;
2758        for (i = 0;; i++) {
2759                fallback_mt = fallbacks[migratetype][i];
2760                if (fallback_mt == MIGRATE_TYPES)
2761                        break;
2762
2763                if (free_area_empty(area, fallback_mt))
2764                        continue;
2765
2766                if (can_steal_fallback(order, migratetype))
2767                        *can_steal = true;
2768
2769                if (!only_stealable)
2770                        return fallback_mt;
2771
2772                if (*can_steal)
2773                        return fallback_mt;
2774        }
2775
2776        return -1;
2777}
2778
2779/*
2780 * Reserve a pageblock for exclusive use of high-order atomic allocations if
2781 * there are no empty page blocks that contain a page with a suitable order
2782 */
2783static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2784                                unsigned int alloc_order)
2785{
2786        int mt;
2787        unsigned long max_managed, flags;
2788
2789        /*
2790         * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2791         * Check is race-prone but harmless.
2792         */
2793        max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
2794        if (zone->nr_reserved_highatomic >= max_managed)
2795                return;
2796
2797        spin_lock_irqsave(&zone->lock, flags);
2798
2799        /* Recheck the nr_reserved_highatomic limit under the lock */
2800        if (zone->nr_reserved_highatomic >= max_managed)
2801                goto out_unlock;
2802
2803        /* Yoink! */
2804        mt = get_pageblock_migratetype(page);
2805        if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2806            && !is_migrate_cma(mt)) {
2807                zone->nr_reserved_highatomic += pageblock_nr_pages;
2808                set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2809                move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2810        }
2811
2812out_unlock:
2813        spin_unlock_irqrestore(&zone->lock, flags);
2814}
2815
2816/*
2817 * Used when an allocation is about to fail under memory pressure. This
2818 * potentially hurts the reliability of high-order allocations when under
2819 * intense memory pressure but failed atomic allocations should be easier
2820 * to recover from than an OOM.
2821 *
2822 * If @force is true, try to unreserve a pageblock even though highatomic
2823 * pageblock is exhausted.
2824 */
2825static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2826                                                bool force)
2827{
2828        struct zonelist *zonelist = ac->zonelist;
2829        unsigned long flags;
2830        struct zoneref *z;
2831        struct zone *zone;
2832        struct page *page;
2833        int order;
2834        bool ret;
2835
2836        for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
2837                                                                ac->nodemask) {
2838                /*
2839                 * Preserve at least one pageblock unless memory pressure
2840                 * is really high.
2841                 */
2842                if (!force && zone->nr_reserved_highatomic <=
2843                                        pageblock_nr_pages)
2844                        continue;
2845
2846                spin_lock_irqsave(&zone->lock, flags);
2847                for (order = 0; order < MAX_ORDER; order++) {
2848                        struct free_area *area = &(zone->free_area[order]);
2849
2850                        page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
2851                        if (!page)
2852                                continue;
2853
2854                        /*
2855                         * In page freeing path, migratetype change is racy so
2856                         * we can counter several free pages in a pageblock
2857                         * in this loop although we changed the pageblock type
2858                         * from highatomic to ac->migratetype. So we should
2859                         * adjust the count once.
2860                         */
2861                        if (is_migrate_highatomic_page(page)) {
2862                                /*
2863                                 * It should never happen but changes to
2864                                 * locking could inadvertently allow a per-cpu
2865                                 * drain to add pages to MIGRATE_HIGHATOMIC
2866                                 * while unreserving so be safe and watch for
2867                                 * underflows.
2868                                 */
2869                                zone->nr_reserved_highatomic -= min(
2870                                                pageblock_nr_pages,
2871                                                zone->nr_reserved_highatomic);
2872                        }
2873
2874                        /*
2875                         * Convert to ac->migratetype and avoid the normal
2876                         * pageblock stealing heuristics. Minimally, the caller
2877                         * is doing the work and needs the pages. More
2878                         * importantly, if the block was always converted to
2879                         * MIGRATE_UNMOVABLE or another type then the number
2880                         * of pageblocks that cannot be completely freed
2881                         * may increase.
2882                         */
2883                        set_pageblock_migratetype(page, ac->migratetype);
2884                        ret = move_freepages_block(zone, page, ac->migratetype,
2885                                                                        NULL);
2886                        if (ret) {
2887                                spin_unlock_irqrestore(&zone->lock, flags);
2888                                return ret;
2889                        }
2890                }
2891                spin_unlock_irqrestore(&zone->lock, flags);
2892        }
2893
2894        return false;
2895}
2896
2897/*
2898 * Try finding a free buddy page on the fallback list and put it on the free
2899 * list of requested migratetype, possibly along with other pages from the same
2900 * block, depending on fragmentation avoidance heuristics. Returns true if
2901 * fallback was found so that __rmqueue_smallest() can grab it.
2902 *
2903 * The use of signed ints for order and current_order is a deliberate
2904 * deviation from the rest of this file, to make the for loop
2905 * condition simpler.
2906 */
2907static __always_inline bool
2908__rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
2909                                                unsigned int alloc_flags)
2910{
2911        struct free_area *area;
2912        int current_order;
2913        int min_order = order;
2914        struct page *page;
2915        int fallback_mt;
2916        bool can_steal;
2917
2918        /*
2919         * Do not steal pages from freelists belonging to other pageblocks
2920         * i.e. orders < pageblock_order. If there are no local zones free,
2921         * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
2922         */
2923        if (alloc_flags & ALLOC_NOFRAGMENT)
2924                min_order = pageblock_order;
2925
2926        /*
2927         * Find the largest available free page in the other list. This roughly
2928         * approximates finding the pageblock with the most free pages, which
2929         * would be too costly to do exactly.
2930         */
2931        for (current_order = MAX_ORDER - 1; current_order >= min_order;
2932                                --current_order) {
2933                area = &(zone->free_area[current_order]);
2934                fallback_mt = find_suitable_fallback(area, current_order,
2935                                start_migratetype, false, &can_steal);
2936                if (fallback_mt == -1)
2937                        continue;
2938
2939                /*
2940                 * We cannot steal all free pages from the pageblock and the
2941                 * requested migratetype is movable. In that case it's better to
2942                 * steal and split the smallest available page instead of the
2943                 * largest available page, because even if the next movable
2944                 * allocation falls back into a different pageblock than this
2945                 * one, it won't cause permanent fragmentation.
2946                 */
2947                if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2948                                        && current_order > order)
2949                        goto find_smallest;
2950
2951                goto do_steal;
2952        }
2953
2954        return false;
2955
2956find_smallest:
2957        for (current_order = order; current_order < MAX_ORDER;
2958                                                        current_order++) {
2959                area = &(zone->free_area[current_order]);
2960                fallback_mt = find_suitable_fallback(area, current_order,
2961                                start_migratetype, false, &can_steal);
2962                if (fallback_mt != -1)
2963                        break;
2964        }
2965
2966        /*
2967         * This should not happen - we already found a suitable fallback
2968         * when looking for the largest page.
2969         */
2970        VM_BUG_ON(current_order == MAX_ORDER);
2971
2972do_steal:
2973        page = get_page_from_free_area(area, fallback_mt);
2974
2975        steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
2976                                                                can_steal);
2977
2978        trace_mm_page_alloc_extfrag(page, order, current_order,
2979                start_migratetype, fallback_mt);
2980
2981        return true;
2982
2983}
2984
2985/*
2986 * Do the hard work of removing an element from the buddy allocator.
2987 * Call me with the zone->lock already held.
2988 */
2989static __always_inline struct page *
2990__rmqueue(struct zone *zone, unsigned int order, int migratetype,
2991                                                unsigned int alloc_flags)
2992{
2993        struct page *page;
2994
2995        if (IS_ENABLED(CONFIG_CMA)) {
2996                /*
2997                 * Balance movable allocations between regular and CMA areas by
2998                 * allocating from CMA when over half of the zone's free memory
2999                 * is in the CMA area.
3000                 */
3001                if (alloc_flags & ALLOC_CMA &&
3002                    zone_page_state(zone, NR_FREE_CMA_PAGES) >
3003                    zone_page_state(zone, NR_FREE_PAGES) / 2) {
3004                        page = __rmqueue_cma_fallback(zone, order);
3005                        if (page)
3006                                goto out;
3007                }
3008        }
3009retry:
3010        page = __rmqueue_smallest(zone, order, migratetype);
3011        if (unlikely(!page)) {
3012                if (alloc_flags & ALLOC_CMA)
3013                        page = __rmqueue_cma_fallback(zone, order);
3014
3015                if (!page && __rmqueue_fallback(zone, order, migratetype,
3016                                                                alloc_flags))
3017                        goto retry;
3018        }
3019out:
3020        if (page)
3021                trace_mm_page_alloc_zone_locked(page, order, migratetype);
3022        return page;
3023}
3024
3025/*
3026 * Obtain a specified number of elements from the buddy allocator, all under
3027 * a single hold of the lock, for efficiency.  Add them to the supplied list.
3028 * Returns the number of new pages which were placed at *list.
3029 */
3030static int rmqueue_bulk(struct zone *zone, unsigned int order,
3031                        unsigned long count, struct list_head *list,
3032                        int migratetype, unsigned int alloc_flags)
3033{
3034        int i, allocated = 0;
3035
3036        /*
3037         * local_lock_irq held so equivalent to spin_lock_irqsave for
3038         * both PREEMPT_RT and non-PREEMPT_RT configurations.
3039         */
3040        spin_lock(&zone->lock);
3041        for (i = 0; i < count; ++i) {
3042                struct page *page = __rmqueue(zone, order, migratetype,
3043                                                                alloc_flags);
3044                if (unlikely(page == NULL))
3045                        break;
3046
3047                if (unlikely(check_pcp_refill(page)))
3048                        continue;
3049
3050                /*
3051                 * Split buddy pages returned by expand() are received here in
3052                 * physical page order. The page is added to the tail of
3053                 * caller's list. From the callers perspective, the linked list
3054                 * is ordered by page number under some conditions. This is
3055                 * useful for IO devices that can forward direction from the
3056                 * head, thus also in the physical page order. This is useful
3057                 * for IO devices that can merge IO requests if the physical
3058                 * pages are ordered properly.
3059                 */
3060                list_add_tail(&page->lru, list);
3061                allocated++;
3062                if (is_migrate_cma(get_pcppage_migratetype(page)))
3063                        __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
3064                                              -(1 << order));
3065        }
3066
3067        /*
3068         * i pages were removed from the buddy list even if some leak due
3069         * to check_pcp_refill failing so adjust NR_FREE_PAGES based
3070         * on i. Do not confuse with 'allocated' which is the number of
3071         * pages added to the pcp list.
3072         */
3073        __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
3074        spin_unlock(&zone->lock);
3075        return allocated;
3076}
3077
3078#ifdef CONFIG_NUMA
3079/*
3080 * Called from the vmstat counter updater to drain pagesets of this
3081 * currently executing processor on remote nodes after they have
3082 * expired.
3083 *
3084 * Note that this function must be called with the thread pinned to
3085 * a single processor.
3086 */
3087void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
3088{
3089        unsigned long flags;
3090        int to_drain, batch;
3091
3092        local_lock_irqsave(&pagesets.lock, flags);
3093        batch = READ_ONCE(pcp->batch);
3094        to_drain = min(pcp->count, batch);
3095        if (to_drain > 0)
3096                free_pcppages_bulk(zone, to_drain, pcp);
3097        local_unlock_irqrestore(&pagesets.lock, flags);
3098}
3099#endif
3100
3101/*
3102 * Drain pcplists of the indicated processor and zone.
3103 *
3104 * The processor must either be the current processor and the
3105 * thread pinned to the current processor or a processor that
3106 * is not online.
3107 */
3108static void drain_pages_zone(unsigned int cpu, struct zone *zone)
3109{
3110        unsigned long flags;
3111        struct per_cpu_pages *pcp;
3112
3113        local_lock_irqsave(&pagesets.lock, flags);
3114
3115        pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
3116        if (pcp->count)
3117                free_pcppages_bulk(zone, pcp->count, pcp);
3118
3119        local_unlock_irqrestore(&pagesets.lock, flags);
3120}
3121
3122/*
3123 * Drain pcplists of all zones on the indicated processor.
3124 *
3125 * The processor must either be the current processor and the
3126 * thread pinned to the current processor or a processor that
3127 * is not online.
3128 */
3129static void drain_pages(unsigned int cpu)
3130{
3131        struct zone *zone;
3132
3133        for_each_populated_zone(zone) {
3134                drain_pages_zone(cpu, zone);
3135        }
3136}
3137
3138/*
3139 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
3140 *
3141 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
3142 * the single zone's pages.
3143 */
3144void drain_local_pages(struct zone *zone)
3145{
3146        int cpu = smp_processor_id();
3147
3148        if (zone)
3149                drain_pages_zone(cpu, zone);
3150        else
3151                drain_pages(cpu);
3152}
3153
3154static void drain_local_pages_wq(struct work_struct *work)
3155{
3156        struct pcpu_drain *drain;
3157
3158        drain = container_of(work, struct pcpu_drain, work);
3159
3160        /*
3161         * drain_all_pages doesn't use proper cpu hotplug protection so
3162         * we can race with cpu offline when the WQ can move this from
3163         * a cpu pinned worker to an unbound one. We can operate on a different
3164         * cpu which is alright but we also have to make sure to not move to
3165         * a different one.
3166         */
3167        preempt_disable();
3168        drain_local_pages(drain->zone);
3169        preempt_enable();
3170}
3171
3172/*
3173 * The implementation of drain_all_pages(), exposing an extra parameter to
3174 * drain on all cpus.
3175 *
3176 * drain_all_pages() is optimized to only execute on cpus where pcplists are
3177 * not empty. The check for non-emptiness can however race with a free to
3178 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers
3179 * that need the guarantee that every CPU has drained can disable the
3180 * optimizing racy check.
3181 */
3182static void __drain_all_pages(struct zone *zone, bool force_all_cpus)
3183{
3184        int cpu;
3185
3186        /*
3187         * Allocate in the BSS so we won't require allocation in
3188         * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
3189         */
3190        static cpumask_t cpus_with_pcps;
3191
3192        /*
3193         * Make sure nobody triggers this path before mm_percpu_wq is fully
3194         * initialized.
3195         */
3196        if (WARN_ON_ONCE(!mm_percpu_wq))
3197                return;
3198
3199        /*
3200         * Do not drain if one is already in progress unless it's specific to
3201         * a zone. Such callers are primarily CMA and memory hotplug and need
3202         * the drain to be complete when the call returns.
3203         */
3204        if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
3205                if (!zone)
3206                        return;
3207                mutex_lock(&pcpu_drain_mutex);
3208        }
3209
3210        /*
3211         * We don't care about racing with CPU hotplug event
3212         * as offline notification will cause the notified
3213         * cpu to drain that CPU pcps and on_each_cpu_mask
3214         * disables preemption as part of its processing
3215         */
3216        for_each_online_cpu(cpu) {
3217                struct per_cpu_pages *pcp;
3218                struct zone *z;
3219                bool has_pcps = false;
3220
3221                if (force_all_cpus) {
3222                        /*
3223                         * The pcp.count check is racy, some callers need a
3224                         * guarantee that no cpu is missed.
3225                         */
3226                        has_pcps = true;
3227                } else if (zone) {
3228                        pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
3229                        if (pcp->count)
3230                                has_pcps = true;
3231                } else {
3232                        for_each_populated_zone(z) {
3233                                pcp = per_cpu_ptr(z->per_cpu_pageset, cpu);
3234                                if (pcp->count) {
3235                                        has_pcps = true;
3236                                        break;
3237                                }
3238                        }
3239                }
3240
3241                if (has_pcps)
3242                        cpumask_set_cpu(cpu, &cpus_with_pcps);
3243                else
3244                        cpumask_clear_cpu(cpu, &cpus_with_pcps);
3245        }
3246
3247        for_each_cpu(cpu, &cpus_with_pcps) {
3248                struct pcpu_drain *drain = per_cpu_ptr(&pcpu_drain, cpu);
3249
3250                drain->zone = zone;
3251                INIT_WORK(&drain->work, drain_local_pages_wq);
3252                queue_work_on(cpu, mm_percpu_wq, &drain->work);
3253        }
3254        for_each_cpu(cpu, &cpus_with_pcps)
3255                flush_work(&per_cpu_ptr(&pcpu_drain, cpu)->work);
3256
3257        mutex_unlock(&pcpu_drain_mutex);
3258}
3259
3260/*
3261 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
3262 *
3263 * When zone parameter is non-NULL, spill just the single zone's pages.
3264 *
3265 * Note that this can be extremely slow as the draining happens in a workqueue.
3266 */
3267void drain_all_pages(struct zone *zone)
3268{
3269        __drain_all_pages(zone, false);
3270}
3271
3272#ifdef CONFIG_HIBERNATION
3273
3274/*
3275 * Touch the watchdog for every WD_PAGE_COUNT pages.
3276 */
3277#define WD_PAGE_COUNT   (128*1024)
3278
3279void mark_free_pages(struct zone *zone)
3280{
3281        unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
3282        unsigned long flags;
3283        unsigned int order, t;
3284        struct page *page;
3285
3286        if (zone_is_empty(zone))
3287                return;
3288
3289        spin_lock_irqsave(&zone->lock, flags);
3290
3291        max_zone_pfn = zone_end_pfn(zone);
3292        for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
3293                if (pfn_valid(pfn)) {
3294                        page = pfn_to_page(pfn);
3295
3296                        if (!--page_count) {
3297                                touch_nmi_watchdog();
3298                                page_count = WD_PAGE_COUNT;
3299                        }
3300
3301                        if (page_zone(page) != zone)
3302                                continue;
3303
3304                        if (!swsusp_page_is_forbidden(page))
3305                                swsusp_unset_page_free(page);
3306                }
3307
3308        for_each_migratetype_order(order, t) {
3309                list_for_each_entry(page,
3310                                &zone->free_area[order].free_list[t], lru) {
3311                        unsigned long i;
3312
3313                        pfn = page_to_pfn(page);
3314                        for (i = 0; i < (1UL << order); i++) {
3315                                if (!--page_count) {
3316                                        touch_nmi_watchdog();
3317                                        page_count = WD_PAGE_COUNT;
3318                                }
3319                                swsusp_set_page_free(pfn_to_page(pfn + i));
3320                        }
3321                }
3322        }
3323        spin_unlock_irqrestore(&zone->lock, flags);
3324}
3325#endif /* CONFIG_PM */
3326
3327static bool free_unref_page_prepare(struct page *page, unsigned long pfn,
3328                                                        unsigned int order)
3329{
3330        int migratetype;
3331
3332        if (!free_pcp_prepare(page, order))
3333                return false;
3334
3335        migratetype = get_pfnblock_migratetype(page, pfn);
3336        set_pcppage_migratetype(page, migratetype);
3337        return true;
3338}
3339
3340static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch)
3341{
3342        int min_nr_free, max_nr_free;
3343
3344        /* Check for PCP disabled or boot pageset */
3345        if (unlikely(high < batch))
3346                return 1;
3347
3348        /* Leave at least pcp->batch pages on the list */
3349        min_nr_free = batch;
3350        max_nr_free = high - batch;
3351
3352        /*
3353         * Double the number of pages freed each time there is subsequent
3354         * freeing of pages without any allocation.
3355         */
3356        batch <<= pcp->free_factor;
3357        if (batch < max_nr_free)
3358                pcp->free_factor++;
3359        batch = clamp(batch, min_nr_free, max_nr_free);
3360
3361        return batch;
3362}
3363
3364static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone)
3365{
3366        int high = READ_ONCE(pcp->high);
3367
3368        if (unlikely(!high))
3369                return 0;
3370
3371        if (!test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags))
3372                return high;
3373
3374        /*
3375         * If reclaim is active, limit the number of pages that can be
3376         * stored on pcp lists
3377         */
3378        return min(READ_ONCE(pcp->batch) << 2, high);
3379}
3380
3381static void free_unref_page_commit(struct page *page, unsigned long pfn,
3382                                   int migratetype, unsigned int order)
3383{
3384        struct zone *zone = page_zone(page);
3385        struct per_cpu_pages *pcp;
3386        int high;
3387        int pindex;
3388
3389        __count_vm_event(PGFREE);
3390        pcp = this_cpu_ptr(zone->per_cpu_pageset);
3391        pindex = order_to_pindex(migratetype, order);
3392        list_add(&page->lru, &pcp->lists[pindex]);
3393        pcp->count += 1 << order;
3394        high = nr_pcp_high(pcp, zone);
3395        if (pcp->count >= high) {
3396                int batch = READ_ONCE(pcp->batch);
3397
3398                free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp);
3399        }
3400}
3401
3402/*
3403 * Free a pcp page
3404 */
3405void free_unref_page(struct page *page, unsigned int order)
3406{
3407        unsigned long flags;
3408        unsigned long pfn = page_to_pfn(page);
3409        int migratetype;
3410
3411        if (!free_unref_page_prepare(page, pfn, order))
3412                return;
3413
3414        /*
3415         * We only track unmovable, reclaimable and movable on pcp lists.
3416         * Place ISOLATE pages on the isolated list because they are being
3417         * offlined but treat HIGHATOMIC as movable pages so we can get those
3418         * areas back if necessary. Otherwise, we may have to free
3419         * excessively into the page allocator
3420         */
3421        migratetype = get_pcppage_migratetype(page);
3422        if (unlikely(migratetype >= MIGRATE_PCPTYPES)) {
3423                if (unlikely(is_migrate_isolate(migratetype))) {
3424                        free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE);
3425                        return;
3426                }
3427                migratetype = MIGRATE_MOVABLE;
3428        }
3429
3430        local_lock_irqsave(&pagesets.lock, flags);
3431        free_unref_page_commit(page, pfn, migratetype, order);
3432        local_unlock_irqrestore(&pagesets.lock, flags);
3433}
3434
3435/*
3436 * Free a list of 0-order pages
3437 */
3438void free_unref_page_list(struct list_head *list)
3439{
3440        struct page *page, *next;
3441        unsigned long flags, pfn;
3442        int batch_count = 0;
3443        int migratetype;
3444
3445        /* Prepare pages for freeing */
3446        list_for_each_entry_safe(page, next, list, lru) {
3447                pfn = page_to_pfn(page);
3448                if (!free_unref_page_prepare(page, pfn, 0))
3449                        list_del(&page->lru);
3450
3451                /*
3452                 * Free isolated pages directly to the allocator, see
3453                 * comment in free_unref_page.
3454                 */
3455                migratetype = get_pcppage_migratetype(page);
3456                if (unlikely(is_migrate_isolate(migratetype))) {
3457                        list_del(&page->lru);
3458                        free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE);
3459                        continue;
3460                }
3461
3462                set_page_private(page, pfn);
3463        }
3464
3465        local_lock_irqsave(&pagesets.lock, flags);
3466        list_for_each_entry_safe(page, next, list, lru) {
3467                pfn = page_private(page);
3468                set_page_private(page, 0);
3469
3470                /*
3471                 * Non-isolated types over MIGRATE_PCPTYPES get added
3472                 * to the MIGRATE_MOVABLE pcp list.
3473                 */
3474                migratetype = get_pcppage_migratetype(page);
3475                if (unlikely(migratetype >= MIGRATE_PCPTYPES))
3476                        migratetype = MIGRATE_MOVABLE;
3477
3478                trace_mm_page_free_batched(page);
3479                free_unref_page_commit(page, pfn, migratetype, 0);
3480
3481                /*
3482                 * Guard against excessive IRQ disabled times when we get
3483                 * a large list of pages to free.
3484                 */
3485                if (++batch_count == SWAP_CLUSTER_MAX) {
3486                        local_unlock_irqrestore(&pagesets.lock, flags);
3487                        batch_count = 0;
3488                        local_lock_irqsave(&pagesets.lock, flags);
3489                }
3490        }
3491        local_unlock_irqrestore(&pagesets.lock, flags);
3492}
3493
3494/*
3495 * split_page takes a non-compound higher-order page, and splits it into
3496 * n (1<<order) sub-pages: page[0..n]
3497 * Each sub-page must be freed individually.
3498 *
3499 * Note: this is probably too low level an operation for use in drivers.
3500 * Please consult with lkml before using this in your driver.
3501 */
3502void split_page(struct page *page, unsigned int order)
3503{
3504        int i;
3505
3506        VM_BUG_ON_PAGE(PageCompound(page), page);
3507        VM_BUG_ON_PAGE(!page_count(page), page);
3508
3509        for (i = 1; i < (1 << order); i++)
3510                set_page_refcounted(page + i);
3511        split_page_owner(page, 1 << order);
3512        split_page_memcg(page, 1 << order);
3513}
3514EXPORT_SYMBOL_GPL(split_page);
3515
3516int __isolate_free_page(struct page *page, unsigned int order)
3517{
3518        unsigned long watermark;
3519        struct zone *zone;
3520        int mt;
3521
3522        BUG_ON(!PageBuddy(page));
3523
3524        zone = page_zone(page);
3525        mt = get_pageblock_migratetype(page);
3526
3527        if (!is_migrate_isolate(mt)) {
3528                /*
3529                 * Obey watermarks as if the page was being allocated. We can
3530                 * emulate a high-order watermark check with a raised order-0
3531                 * watermark, because we already know our high-order page
3532                 * exists.
3533                 */
3534                watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
3535                if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
3536                        return 0;
3537
3538                __mod_zone_freepage_state(zone, -(1UL << order), mt);
3539        }
3540
3541        /* Remove page from free list */
3542
3543        del_page_from_free_list(page, zone, order);
3544
3545        /*
3546         * Set the pageblock if the isolated page is at least half of a
3547         * pageblock
3548         */
3549        if (order >= pageblock_order - 1) {
3550                struct page *endpage = page + (1 << order) - 1;
3551                for (; page < endpage; page += pageblock_nr_pages) {
3552                        int mt = get_pageblock_migratetype(page);
3553                        if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
3554                            && !is_migrate_highatomic(mt))
3555                                set_pageblock_migratetype(page,
3556                                                          MIGRATE_MOVABLE);
3557                }
3558        }
3559
3560
3561        return 1UL << order;
3562}
3563
3564/**
3565 * __putback_isolated_page - Return a now-isolated page back where we got it
3566 * @page: Page that was isolated
3567 * @order: Order of the isolated page
3568 * @mt: The page's pageblock's migratetype
3569 *
3570 * This function is meant to return a page pulled from the free lists via
3571 * __isolate_free_page back to the free lists they were pulled from.
3572 */
3573void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3574{
3575        struct zone *zone = page_zone(page);
3576
3577        /* zone lock should be held when this function is called */
3578        lockdep_assert_held(&zone->lock);
3579
3580        /* Return isolated page to tail of freelist. */
3581        __free_one_page(page, page_to_pfn(page), zone, order, mt,
3582                        FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
3583}
3584
3585/*
3586 * Update NUMA hit/miss statistics
3587 *
3588 * Must be called with interrupts disabled.
3589 */
3590static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,
3591                                   long nr_account)
3592{
3593#ifdef CONFIG_NUMA
3594        enum numa_stat_item local_stat = NUMA_LOCAL;
3595
3596        /* skip numa counters update if numa stats is disabled */
3597        if (!static_branch_likely(&vm_numa_stat_key))
3598                return;
3599
3600        if (zone_to_nid(z) != numa_node_id())
3601                local_stat = NUMA_OTHER;
3602
3603        if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3604                __count_numa_events(z, NUMA_HIT, nr_account);
3605        else {
3606                __count_numa_events(z, NUMA_MISS, nr_account);
3607                __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account);
3608        }
3609        __count_numa_events(z, local_stat, nr_account);
3610#endif
3611}
3612
3613/* Remove page from the per-cpu list, caller must protect the list */
3614static inline
3615struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,
3616                        int migratetype,
3617                        unsigned int alloc_flags,
3618                        struct per_cpu_pages *pcp,
3619                        struct list_head *list)
3620{
3621        struct page *page;
3622
3623        do {
3624                if (list_empty(list)) {
3625                        int batch = READ_ONCE(pcp->batch);
3626                        int alloced;
3627
3628                        /*
3629                         * Scale batch relative to order if batch implies
3630                         * free pages can be stored on the PCP. Batch can
3631                         * be 1 for small zones or for boot pagesets which
3632                         * should never store free pages as the pages may
3633                         * belong to arbitrary zones.
3634                         */
3635                        if (batch > 1)
3636                                batch = max(batch >> order, 2);
3637                        alloced = rmqueue_bulk(zone, order,
3638                                        batch, list,
3639                                        migratetype, alloc_flags);
3640
3641                        pcp->count += alloced << order;
3642                        if (unlikely(list_empty(list)))
3643                                return NULL;
3644                }
3645
3646                page = list_first_entry(list, struct page, lru);
3647                list_del(&page->lru);
3648                pcp->count -= 1 << order;
3649        } while (check_new_pcp(page));
3650
3651        return page;
3652}
3653
3654/* Lock and remove page from the per-cpu list */
3655static struct page *rmqueue_pcplist(struct zone *preferred_zone,
3656                        struct zone *zone, unsigned int order,
3657                        gfp_t gfp_flags, int migratetype,
3658                        unsigned int alloc_flags)
3659{
3660        struct per_cpu_pages *pcp;
3661        struct list_head *list;
3662        struct page *page;
3663        unsigned long flags;
3664
3665        local_lock_irqsave(&pagesets.lock, flags);
3666
3667        /*
3668         * On allocation, reduce the number of pages that are batch freed.
3669         * See nr_pcp_free() where free_factor is increased for subsequent
3670         * frees.
3671         */
3672        pcp = this_cpu_ptr(zone->per_cpu_pageset);
3673        pcp->free_factor >>= 1;
3674        list = &pcp->lists[order_to_pindex(migratetype, order)];
3675        page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list);
3676        local_unlock_irqrestore(&pagesets.lock, flags);
3677        if (page) {
3678                __count_zid_vm_events(PGALLOC, page_zonenum(page), 1);
3679                zone_statistics(preferred_zone, zone, 1);
3680        }
3681        return page;
3682}
3683
3684/*
3685 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
3686 */
3687static inline
3688struct page *rmqueue(struct zone *preferred_zone,
3689                        struct zone *zone, unsigned int order,
3690                        gfp_t gfp_flags, unsigned int alloc_flags,
3691                        int migratetype)
3692{
3693        unsigned long flags;
3694        struct page *page;
3695
3696        if (likely(pcp_allowed_order(order))) {
3697                /*
3698                 * MIGRATE_MOVABLE pcplist could have the pages on CMA area and
3699                 * we need to skip it when CMA area isn't allowed.
3700                 */
3701                if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA ||
3702                                migratetype != MIGRATE_MOVABLE) {
3703                        page = rmqueue_pcplist(preferred_zone, zone, order,
3704                                        gfp_flags, migratetype, alloc_flags);
3705                        goto out;
3706                }
3707        }
3708
3709        /*
3710         * We most definitely don't want callers attempting to
3711         * allocate greater than order-1 page units with __GFP_NOFAIL.
3712         */
3713        WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
3714        spin_lock_irqsave(&zone->lock, flags);
3715
3716        do {
3717                page = NULL;
3718                /*
3719                 * order-0 request can reach here when the pcplist is skipped
3720                 * due to non-CMA allocation context. HIGHATOMIC area is
3721                 * reserved for high-order atomic allocation, so order-0
3722                 * request should skip it.
3723                 */
3724                if (order > 0 && alloc_flags & ALLOC_HARDER) {
3725                        page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3726                        if (page)
3727                                trace_mm_page_alloc_zone_locked(page, order, migratetype);
3728                }
3729                if (!page)
3730                        page = __rmqueue(zone, order, migratetype, alloc_flags);
3731        } while (page && check_new_pages(page, order));
3732        if (!page)
3733                goto failed;
3734
3735        __mod_zone_freepage_state(zone, -(1 << order),
3736                                  get_pcppage_migratetype(page));
3737        spin_unlock_irqrestore(&zone->lock, flags);
3738
3739        __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3740        zone_statistics(preferred_zone, zone, 1);
3741
3742out:
3743        /* Separate test+clear to avoid unnecessary atomics */
3744        if (test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags)) {
3745                clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3746                wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3747        }
3748
3749        VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
3750        return page;
3751
3752failed:
3753        spin_unlock_irqrestore(&zone->lock, flags);
3754        return NULL;
3755}
3756
3757#ifdef CONFIG_FAIL_PAGE_ALLOC
3758
3759static struct {
3760        struct fault_attr attr;
3761
3762        bool ignore_gfp_highmem;
3763        bool ignore_gfp_reclaim;
3764        u32 min_order;
3765} fail_page_alloc = {
3766        .attr = FAULT_ATTR_INITIALIZER,
3767        .ignore_gfp_reclaim = true,
3768        .ignore_gfp_highmem = true,
3769        .min_order = 1,
3770};
3771
3772static int __init setup_fail_page_alloc(char *str)
3773{
3774        return setup_fault_attr(&fail_page_alloc.attr, str);
3775}
3776__setup("fail_page_alloc=", setup_fail_page_alloc);
3777
3778static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3779{
3780        if (order < fail_page_alloc.min_order)
3781                return false;
3782        if (gfp_mask & __GFP_NOFAIL)
3783                return false;
3784        if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
3785                return false;
3786        if (fail_page_alloc.ignore_gfp_reclaim &&
3787                        (gfp_mask & __GFP_DIRECT_RECLAIM))
3788                return false;
3789
3790        return should_fail(&fail_page_alloc.attr, 1 << order);
3791}
3792
3793#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3794
3795static int __init fail_page_alloc_debugfs(void)
3796{
3797        umode_t mode = S_IFREG | 0600;
3798        struct dentry *dir;
3799
3800        dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
3801                                        &fail_page_alloc.attr);
3802
3803        debugfs_create_bool("ignore-gfp-wait", mode, dir,
3804                            &fail_page_alloc.ignore_gfp_reclaim);
3805        debugfs_create_bool("ignore-gfp-highmem", mode, dir,
3806                            &fail_page_alloc.ignore_gfp_highmem);
3807        debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
3808
3809        return 0;
3810}
3811
3812late_initcall(fail_page_alloc_debugfs);
3813
3814#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
3815
3816#else /* CONFIG_FAIL_PAGE_ALLOC */
3817
3818static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3819{
3820        return false;
3821}
3822
3823#endif /* CONFIG_FAIL_PAGE_ALLOC */
3824
3825noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
3826{
3827        return __should_fail_alloc_page(gfp_mask, order);
3828}
3829ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
3830
3831static inline long __zone_watermark_unusable_free(struct zone *z,
3832                                unsigned int order, unsigned int alloc_flags)
3833{
3834        const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3835        long unusable_free = (1 << order) - 1;
3836
3837        /*
3838         * If the caller does not have rights to ALLOC_HARDER then subtract
3839         * the high-atomic reserves. This will over-estimate the size of the
3840         * atomic reserve but it avoids a search.
3841         */
3842        if (likely(!alloc_harder))
3843                unusable_free += z->nr_reserved_highatomic;
3844
3845#ifdef CONFIG_CMA
3846        /* If allocation can't use CMA areas don't use free CMA pages */
3847        if (!(alloc_flags & ALLOC_CMA))
3848                unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
3849#endif
3850
3851        return unusable_free;
3852}
3853
3854/*
3855 * Return true if free base pages are above 'mark'. For high-order checks it
3856 * will return true of the order-0 watermark is reached and there is at least
3857 * one free page of a suitable size. Checking now avoids taking the zone lock
3858 * to check in the allocation paths if no pages are free.
3859 */
3860bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3861                         int highest_zoneidx, unsigned int alloc_flags,
3862                         long free_pages)
3863{
3864        long min = mark;
3865        int o;
3866        const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
3867
3868        /* free_pages may go negative - that's OK */
3869        free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
3870
3871        if (alloc_flags & ALLOC_HIGH)
3872                min -= min / 2;
3873
3874        if (unlikely(alloc_harder)) {
3875                /*
3876                 * OOM victims can try even harder than normal ALLOC_HARDER
3877                 * users on the grounds that it's definitely going to be in
3878                 * the exit path shortly and free memory. Any allocation it
3879                 * makes during the free path will be small and short-lived.
3880                 */
3881                if (alloc_flags & ALLOC_OOM)
3882                        min -= min / 2;
3883                else
3884                        min -= min / 4;
3885        }
3886
3887        /*
3888         * Check watermarks for an order-0 allocation request. If these
3889         * are not met, then a high-order request also cannot go ahead
3890         * even if a suitable page happened to be free.
3891         */
3892        if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
3893                return false;
3894
3895        /* If this is an order-0 request then the watermark is fine */
3896        if (!order)
3897                return true;
3898
3899        /* For a high-order request, check at least one suitable page is free */
3900        for (o = order; o < MAX_ORDER; o++) {
3901                struct free_area *area = &z->free_area[o];
3902                int mt;
3903
3904                if (!area->nr_free)
3905                        continue;
3906
3907                for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3908                        if (!free_area_empty(area, mt))
3909                                return true;
3910                }
3911
3912#ifdef CONFIG_CMA
3913                if ((alloc_flags & ALLOC_CMA) &&
3914                    !free_area_empty(area, MIGRATE_CMA)) {
3915                        return true;
3916                }
3917#endif
3918                if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC))
3919                        return true;
3920        }
3921        return false;
3922}
3923
3924bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3925                      int highest_zoneidx, unsigned int alloc_flags)
3926{
3927        return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3928                                        zone_page_state(z, NR_FREE_PAGES));
3929}
3930
3931static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
3932                                unsigned long mark, int highest_zoneidx,
3933                                unsigned int alloc_flags, gfp_t gfp_mask)
3934{
3935        long free_pages;
3936
3937        free_pages = zone_page_state(z, NR_FREE_PAGES);
3938
3939        /*
3940         * Fast check for order-0 only. If this fails then the reserves
3941         * need to be calculated.
3942         */
3943        if (!order) {
3944                long fast_free;
3945
3946                fast_free = free_pages;
3947                fast_free -= __zone_watermark_unusable_free(z, 0, alloc_flags);
3948                if (fast_free > mark + z->lowmem_reserve[highest_zoneidx])
3949                        return true;
3950        }
3951
3952        if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3953                                        free_pages))
3954                return true;
3955        /*
3956         * Ignore watermark boosting for GFP_ATOMIC order-0 allocations
3957         * when checking the min watermark. The min watermark is the
3958         * point where boosting is ignored so that kswapd is woken up
3959         * when below the low watermark.
3960         */
3961        if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost
3962                && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
3963                mark = z->_watermark[WMARK_MIN];
3964                return __zone_watermark_ok(z, order, mark, highest_zoneidx,
3965                                        alloc_flags, free_pages);
3966        }
3967
3968        return false;
3969}
3970
3971bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
3972                        unsigned long mark, int highest_zoneidx)
3973{
3974        long free_pages = zone_page_state(z, NR_FREE_PAGES);
3975
3976        if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3977                free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3978
3979        return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0,
3980                                                                free_pages);
3981}
3982
3983#ifdef CONFIG_NUMA
3984static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3985{
3986        return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3987                                node_reclaim_distance;
3988}
3989#else   /* CONFIG_NUMA */
3990static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3991{
3992        return true;
3993}
3994#endif  /* CONFIG_NUMA */
3995
3996/*
3997 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3998 * fragmentation is subtle. If the preferred zone was HIGHMEM then
3999 * premature use of a lower zone may cause lowmem pressure problems that
4000 * are worse than fragmentation. If the next zone is ZONE_DMA then it is
4001 * probably too small. It only makes sense to spread allocations to avoid
4002 * fragmentation between the Normal and DMA32 zones.
4003 */
4004static inline unsigned int
4005alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
4006{
4007        unsigned int alloc_flags;
4008
4009        /*
4010         * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
4011         * to save a branch.
4012         */
4013        alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
4014
4015#ifdef CONFIG_ZONE_DMA32
4016        if (!zone)
4017                return alloc_flags;
4018
4019        if (zone_idx(zone) != ZONE_NORMAL)
4020                return alloc_flags;
4021
4022        /*
4023         * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
4024         * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
4025         * on UMA that if Normal is populated then so is DMA32.
4026         */
4027        BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
4028        if (nr_online_nodes > 1 && !populated_zone(--zone))
4029                return alloc_flags;
4030
4031        alloc_flags |= ALLOC_NOFRAGMENT;
4032#endif /* CONFIG_ZONE_DMA32 */
4033        return alloc_flags;
4034}
4035
4036/* Must be called after current_gfp_context() which can change gfp_mask */
4037static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask,
4038                                                  unsigned int alloc_flags)
4039{
4040#ifdef CONFIG_CMA
4041        if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
4042                alloc_flags |= ALLOC_CMA;
4043#endif
4044        return alloc_flags;
4045}
4046
4047/*
4048 * get_page_from_freelist goes through the zonelist trying to allocate
4049 * a page.
4050 */
4051static struct page *
4052get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
4053                                                const struct alloc_context *ac)
4054{
4055        struct zoneref *z;
4056        struct zone *zone;
4057        struct pglist_data *last_pgdat_dirty_limit = NULL;
4058        bool no_fallback;
4059
4060retry:
4061        /*
4062         * Scan zonelist, looking for a zone with enough free.
4063         * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
4064         */
4065        no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
4066        z = ac->preferred_zoneref;
4067        for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
4068                                        ac->nodemask) {
4069                struct page *page;
4070                unsigned long mark;
4071
4072                if (cpusets_enabled() &&
4073                        (alloc_flags & ALLOC_CPUSET) &&
4074                        !__cpuset_zone_allowed(zone, gfp_mask))
4075                                continue;
4076                /*
4077                 * When allocating a page cache page for writing, we
4078                 * want to get it from a node that is within its dirty
4079                 * limit, such that no single node holds more than its
4080                 * proportional share of globally allowed dirty pages.
4081                 * The dirty limits take into account the node's
4082                 * lowmem reserves and high watermark so that kswapd
4083                 * should be able to balance it without having to
4084                 * write pages from its LRU list.
4085                 *
4086                 * XXX: For now, allow allocations to potentially
4087                 * exceed the per-node dirty limit in the slowpath
4088                 * (spread_dirty_pages unset) before going into reclaim,
4089                 * which is important when on a NUMA setup the allowed
4090                 * nodes are together not big enough to reach the
4091                 * global limit.  The proper fix for these situations
4092                 * will require awareness of nodes in the
4093                 * dirty-throttling and the flusher threads.
4094                 */
4095                if (ac->spread_dirty_pages) {
4096                        if (last_pgdat_dirty_limit == zone->zone_pgdat)
4097                                continue;
4098
4099                        if (!node_dirty_ok(zone->zone_pgdat)) {
4100                                last_pgdat_dirty_limit = zone->zone_pgdat;
4101                                continue;
4102                        }
4103                }
4104
4105                if (no_fallback && nr_online_nodes > 1 &&
4106                    zone != ac->preferred_zoneref->zone) {
4107                        int local_nid;
4108
4109                        /*
4110                         * If moving to a remote node, retry but allow
4111                         * fragmenting fallbacks. Locality is more important
4112                         * than fragmentation avoidance.
4113                         */
4114                        local_nid = zone_to_nid(ac->preferred_zoneref->zone);
4115                        if (zone_to_nid(zone) != local_nid) {
4116                                alloc_flags &= ~ALLOC_NOFRAGMENT;
4117                                goto retry;
4118                        }
4119                }
4120
4121                mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
4122                if (!zone_watermark_fast(zone, order, mark,
4123                                       ac->highest_zoneidx, alloc_flags,
4124                                       gfp_mask)) {
4125                        int ret;
4126
4127#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
4128                        /*
4129                         * Watermark failed for this zone, but see if we can
4130                         * grow this zone if it contains deferred pages.
4131                         */
4132                        if (static_branch_unlikely(&deferred_pages)) {
4133                                if (_deferred_grow_zone(zone, order))
4134                                        goto try_this_zone;
4135                        }
4136#endif
4137                        /* Checked here to keep the fast path fast */
4138                        BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
4139                        if (alloc_flags & ALLOC_NO_WATERMARKS)
4140                                goto try_this_zone;
4141
4142                        if (!node_reclaim_enabled() ||
4143                            !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
4144                                continue;
4145
4146                        ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
4147                        switch (ret) {
4148                        case NODE_RECLAIM_NOSCAN:
4149                                /* did not scan */
4150                                continue;
4151                        case NODE_RECLAIM_FULL:
4152                                /* scanned but unreclaimable */
4153                                continue;
4154                        default:
4155                                /* did we reclaim enough */
4156                                if (zone_watermark_ok(zone, order, mark,
4157                                        ac->highest_zoneidx, alloc_flags))
4158                                        goto try_this_zone;
4159
4160                                continue;
4161                        }
4162                }
4163
4164try_this_zone:
4165                page = rmqueue(ac->preferred_zoneref->zone, zone, order,
4166                                gfp_mask, alloc_flags, ac->migratetype);
4167                if (page) {
4168                        prep_new_page(page, order, gfp_mask, alloc_flags);
4169
4170                        /*
4171                         * If this is a high-order atomic allocation then check
4172                         * if the pageblock should be reserved for the future
4173                         */
4174                        if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
4175                                reserve_highatomic_pageblock(page, zone, order);
4176
4177                        return page;
4178                } else {
4179#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
4180                        /* Try again if zone has deferred pages */
4181                        if (static_branch_unlikely(&deferred_pages)) {
4182                                if (_deferred_grow_zone(zone, order))
4183                                        goto try_this_zone;
4184                        }
4185#endif
4186                }
4187        }
4188
4189        /*
4190         * It's possible on a UMA machine to get through all zones that are
4191         * fragmented. If avoiding fragmentation, reset and try again.
4192         */
4193        if (no_fallback) {
4194                alloc_flags &= ~ALLOC_NOFRAGMENT;
4195                goto retry;
4196        }
4197
4198        return NULL;
4199}
4200
4201static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
4202{
4203        unsigned int filter = SHOW_MEM_FILTER_NODES;
4204
4205        /*
4206         * This documents exceptions given to allocations in certain
4207         * contexts that are allowed to allocate outside current's set
4208         * of allowed nodes.
4209         */
4210        if (!(gfp_mask & __GFP_NOMEMALLOC))
4211                if (tsk_is_oom_victim(current) ||
4212                    (current->flags & (PF_MEMALLOC | PF_EXITING)))
4213                        filter &= ~SHOW_MEM_FILTER_NODES;
4214        if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
4215                filter &= ~SHOW_MEM_FILTER_NODES;
4216
4217        show_mem(filter, nodemask);
4218}
4219
4220void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
4221{
4222        struct va_format vaf;
4223        va_list args;
4224        static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
4225
4226        if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
4227                return;
4228
4229        va_start(args, fmt);
4230        vaf.fmt = fmt;
4231        vaf.va = &args;
4232        pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
4233                        current->comm, &vaf, gfp_mask, &gfp_mask,
4234                        nodemask_pr_args(nodemask));
4235        va_end(args);
4236
4237        cpuset_print_current_mems_allowed();
4238        pr_cont("\n");
4239        dump_stack();
4240        warn_alloc_show_mem(gfp_mask, nodemask);
4241}
4242
4243static inline struct page *
4244__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
4245                              unsigned int alloc_flags,
4246                              const struct alloc_context *ac)
4247{
4248        struct page *page;
4249
4250        page = get_page_from_freelist(gfp_mask, order,
4251                        alloc_flags|ALLOC_CPUSET, ac);
4252        /*
4253         * fallback to ignore cpuset restriction if our nodes
4254         * are depleted
4255         */
4256        if (!page)
4257                page = get_page_from_freelist(gfp_mask, order,
4258                                alloc_flags, ac);
4259
4260        return page;
4261}
4262
4263static inline struct page *
4264__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
4265        const struct alloc_context *ac, unsigned long *did_some_progress)
4266{
4267        struct oom_control oc = {
4268                .zonelist = ac->zonelist,
4269                .nodemask = ac->nodemask,
4270                .memcg = NULL,
4271                .gfp_mask = gfp_mask,
4272                .order = order,
4273        };
4274        struct page *page;
4275
4276        *did_some_progress = 0;
4277
4278        /*
4279         * Acquire the oom lock.  If that fails, somebody else is
4280         * making progress for us.
4281         */
4282        if (!mutex_trylock(&oom_lock)) {
4283                *did_some_progress = 1;
4284                schedule_timeout_uninterruptible(1);
4285                return NULL;
4286        }
4287
4288        /*
4289         * Go through the zonelist yet one more time, keep very high watermark
4290         * here, this is only to catch a parallel oom killing, we must fail if
4291         * we're still under heavy pressure. But make sure that this reclaim
4292         * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
4293         * allocation which will never fail due to oom_lock already held.
4294         */
4295        page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
4296                                      ~__GFP_DIRECT_RECLAIM, order,
4297                                      ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
4298        if (page)
4299                goto out;
4300
4301        /* Coredumps can quickly deplete all memory reserves */
4302        if (current->flags & PF_DUMPCORE)
4303                goto out;
4304        /* The OOM killer will not help higher order allocs */
4305        if (order > PAGE_ALLOC_COSTLY_ORDER)
4306                goto out;
4307        /*
4308         * We have already exhausted all our reclaim opportunities without any
4309         * success so it is time to admit defeat. We will skip the OOM killer
4310         * because it is very likely that the caller has a more reasonable
4311         * fallback than shooting a random task.
4312         *
4313         * The OOM killer may not free memory on a specific node.
4314         */
4315        if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))
4316                goto out;
4317        /* The OOM killer does not needlessly kill tasks for lowmem */
4318        if (ac->highest_zoneidx < ZONE_NORMAL)
4319                goto out;
4320        if (pm_suspended_storage())
4321                goto out;
4322        /*
4323         * XXX: GFP_NOFS allocations should rather fail than rely on
4324         * other request to make a forward progress.
4325         * We are in an unfortunate situation where out_of_memory cannot
4326         * do much for this context but let's try it to at least get
4327         * access to memory reserved if the current task is killed (see
4328         * out_of_memory). Once filesystems are ready to handle allocation
4329         * failures more gracefully we should just bail out here.
4330         */
4331
4332        /* Exhausted what can be done so it's blame time */
4333        if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
4334                *did_some_progress = 1;
4335
4336                /*
4337                 * Help non-failing allocations by giving them access to memory
4338                 * reserves
4339                 */
4340                if (gfp_mask & __GFP_NOFAIL)
4341                        page = __alloc_pages_cpuset_fallback(gfp_mask, order,
4342                                        ALLOC_NO_WATERMARKS, ac);
4343        }
4344out:
4345        mutex_unlock(&oom_lock);
4346        return page;
4347}
4348
4349/*
4350 * Maximum number of compaction retries with a progress before OOM
4351 * killer is consider as the only way to move forward.
4352 */
4353#define MAX_COMPACT_RETRIES 16
4354
4355#ifdef CONFIG_COMPACTION
4356/* Try memory compaction for high-order allocations before reclaim */
4357static struct page *
4358__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
4359                unsigned int alloc_flags, const struct alloc_context *ac,
4360                enum compact_priority prio, enum compact_result *compact_result)
4361{
4362        struct page *page = NULL;
4363        unsigned long pflags;
4364        unsigned int noreclaim_flag;
4365
4366        if (!order)
4367                return NULL;
4368
4369        psi_memstall_enter(&pflags);
4370        noreclaim_flag = memalloc_noreclaim_save();
4371
4372        *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
4373                                                                prio, &page);
4374
4375        memalloc_noreclaim_restore(noreclaim_flag);
4376        psi_memstall_leave(&pflags);
4377
4378        if (*compact_result == COMPACT_SKIPPED)
4379                return NULL;
4380        /*
4381         * At least in one zone compaction wasn't deferred or skipped, so let's
4382         * count a compaction stall
4383         */
4384        count_vm_event(COMPACTSTALL);
4385
4386        /* Prep a captured page if available */
4387        if (page)
4388                prep_new_page(page, order, gfp_mask, alloc_flags);
4389
4390        /* Try get a page from the freelist if available */
4391        if (!page)
4392                page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4393
4394        if (page) {
4395                struct zone *zone = page_zone(page);
4396
4397                zone->compact_blockskip_flush = false;
4398                compaction_defer_reset(zone, order, true);
4399                count_vm_event(COMPACTSUCCESS);
4400                return page;
4401        }
4402
4403        /*
4404         * It's bad if compaction run occurs and fails. The most likely reason
4405         * is that pages exist, but not enough to satisfy watermarks.
4406         */
4407        count_vm_event(COMPACTFAIL);
4408
4409        cond_resched();
4410
4411        return NULL;
4412}
4413
4414static inline bool
4415should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
4416                     enum compact_result compact_result,
4417                     enum compact_priority *compact_priority,
4418                     int *compaction_retries)
4419{
4420        int max_retries = MAX_COMPACT_RETRIES;
4421        int min_priority;
4422        bool ret = false;
4423        int retries = *compaction_retries;
4424        enum compact_priority priority = *compact_priority;
4425
4426        if (!order)
4427                return false;
4428
4429        if (fatal_signal_pending(current))
4430                return false;
4431
4432        if (compaction_made_progress(compact_result))
4433                (*compaction_retries)++;
4434
4435        /*
4436         * compaction considers all the zone as desperately out of memory
4437         * so it doesn't really make much sense to retry except when the
4438         * failure could be caused by insufficient priority
4439         */
4440        if (compaction_failed(compact_result))
4441                goto check_priority;
4442
4443        /*
4444         * compaction was skipped because there are not enough order-0 pages
4445         * to work with, so we retry only if it looks like reclaim can help.
4446         */
4447        if (compaction_needs_reclaim(compact_result)) {
4448                ret = compaction_zonelist_suitable(ac, order, alloc_flags);
4449                goto out;
4450        }
4451
4452        /*
4453         * make sure the compaction wasn't deferred or didn't bail out early
4454         * due to locks contention before we declare that we should give up.
4455         * But the next retry should use a higher priority if allowed, so
4456         * we don't just keep bailing out endlessly.
4457         */
4458        if (compaction_withdrawn(compact_result)) {
4459                goto check_priority;
4460        }
4461
4462        /*
4463         * !costly requests are much more important than __GFP_RETRY_MAYFAIL
4464         * costly ones because they are de facto nofail and invoke OOM
4465         * killer to move on while costly can fail and users are ready
4466         * to cope with that. 1/4 retries is rather arbitrary but we
4467         * would need much more detailed feedback from compaction to
4468         * make a better decision.
4469         */
4470        if (order > PAGE_ALLOC_COSTLY_ORDER)
4471                max_retries /= 4;
4472        if (*compaction_retries <= max_retries) {
4473                ret = true;
4474                goto out;
4475        }
4476
4477        /*
4478         * Make sure there are attempts at the highest priority if we exhausted
4479         * all retries or failed at the lower priorities.
4480         */
4481check_priority:
4482        min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
4483                        MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
4484
4485        if (*compact_priority > min_priority) {
4486                (*compact_priority)--;
4487                *compaction_retries = 0;
4488                ret = true;
4489        }
4490out:
4491        trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
4492        return ret;
4493}
4494#else
4495static inline struct page *
4496__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
4497                unsigned int alloc_flags, const struct alloc_context *ac,
4498                enum compact_priority prio, enum compact_result *compact_result)
4499{
4500        *compact_result = COMPACT_SKIPPED;
4501        return NULL;
4502}
4503
4504static inline bool
4505should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
4506                     enum compact_result compact_result,
4507                     enum compact_priority *compact_priority,
4508                     int *compaction_retries)
4509{
4510        struct zone *zone;
4511        struct zoneref *z;
4512
4513        if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
4514                return false;
4515
4516        /*
4517         * There are setups with compaction disabled which would prefer to loop
4518         * inside the allocator rather than hit the oom killer prematurely.
4519         * Let's give them a good hope and keep retrying while the order-0
4520         * watermarks are OK.
4521         */
4522        for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4523                                ac->highest_zoneidx, ac->nodemask) {
4524                if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
4525                                        ac->highest_zoneidx, alloc_flags))
4526                        return true;
4527        }
4528        return false;
4529}
4530#endif /* CONFIG_COMPACTION */
4531
4532#ifdef CONFIG_LOCKDEP
4533static struct lockdep_map __fs_reclaim_map =
4534        STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
4535
4536static bool __need_reclaim(gfp_t gfp_mask)
4537{
4538        /* no reclaim without waiting on it */
4539        if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
4540                return false;
4541
4542        /* this guy won't enter reclaim */
4543        if (current->flags & PF_MEMALLOC)
4544                return false;
4545
4546        if (gfp_mask & __GFP_NOLOCKDEP)
4547                return false;
4548
4549        return true;
4550}
4551
4552void __fs_reclaim_acquire(void)
4553{
4554        lock_map_acquire(&__fs_reclaim_map);
4555}
4556
4557void __fs_reclaim_release(void)
4558{
4559        lock_map_release(&__fs_reclaim_map);
4560}
4561
4562void fs_reclaim_acquire(gfp_t gfp_mask)
4563{
4564        gfp_mask = current_gfp_context(gfp_mask);
4565
4566        if (__need_reclaim(gfp_mask)) {
4567                if (gfp_mask & __GFP_FS)
4568                        __fs_reclaim_acquire();
4569
4570#ifdef CONFIG_MMU_NOTIFIER
4571                lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
4572                lock_map_release(&__mmu_notifier_invalidate_range_start_map);
4573#endif
4574
4575        }
4576}
4577EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
4578
4579void fs_reclaim_release(gfp_t gfp_mask)
4580{
4581        gfp_mask = current_gfp_context(gfp_mask);
4582
4583        if (__need_reclaim(gfp_mask)) {
4584                if (gfp_mask & __GFP_FS)
4585                        __fs_reclaim_release();
4586        }
4587}
4588EXPORT_SYMBOL_GPL(fs_reclaim_release);
4589#endif
4590
4591/* Perform direct synchronous page reclaim */
4592static unsigned long
4593__perform_reclaim(gfp_t gfp_mask, unsigned int order,
4594                                        const struct alloc_context *ac)
4595{
4596        unsigned int noreclaim_flag;
4597        unsigned long pflags, progress;
4598
4599        cond_resched();
4600
4601        /* We now go into synchronous reclaim */
4602        cpuset_memory_pressure_bump();
4603        psi_memstall_enter(&pflags);
4604        fs_reclaim_acquire(gfp_mask);
4605        noreclaim_flag = memalloc_noreclaim_save();
4606
4607        progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
4608                                                                ac->nodemask);
4609
4610        memalloc_noreclaim_restore(noreclaim_flag);
4611        fs_reclaim_release(gfp_mask);
4612        psi_memstall_leave(&pflags);
4613
4614        cond_resched();
4615
4616        return progress;
4617}
4618
4619/* The really slow allocator path where we enter direct reclaim */
4620static inline struct page *
4621__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
4622                unsigned int alloc_flags, const struct alloc_context *ac,
4623                unsigned long *did_some_progress)
4624{
4625        struct page *page = NULL;
4626        bool drained = false;
4627
4628        *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
4629        if (unlikely(!(*did_some_progress)))
4630                return NULL;
4631
4632retry:
4633        page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4634
4635        /*
4636         * If an allocation failed after direct reclaim, it could be because
4637         * pages are pinned on the per-cpu lists or in high alloc reserves.
4638         * Shrink them and try again
4639         */
4640        if (!page && !drained) {
4641                unreserve_highatomic_pageblock(ac, false);
4642                drain_all_pages(NULL);
4643                drained = true;
4644                goto retry;
4645        }
4646
4647        return page;
4648}
4649
4650static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4651                             const struct alloc_context *ac)
4652{
4653        struct zoneref *z;
4654        struct zone *zone;
4655        pg_data_t *last_pgdat = NULL;
4656        enum zone_type highest_zoneidx = ac->highest_zoneidx;
4657
4658        for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,
4659                                        ac->nodemask) {
4660                if (last_pgdat != zone->zone_pgdat)
4661                        wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);
4662                last_pgdat = zone->zone_pgdat;
4663        }
4664}
4665
4666static inline unsigned int
4667gfp_to_alloc_flags(gfp_t gfp_mask)
4668{
4669        unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
4670
4671        /*
4672         * __GFP_HIGH is assumed to be the same as ALLOC_HIGH
4673         * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
4674         * to save two branches.
4675         */
4676        BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
4677        BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
4678
4679        /*
4680         * The caller may dip into page reserves a bit more if the caller
4681         * cannot run direct reclaim, or if the caller has realtime scheduling
4682         * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
4683         * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
4684         */
4685        alloc_flags |= (__force int)
4686                (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
4687
4688        if (gfp_mask & __GFP_ATOMIC) {
4689                /*
4690                 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
4691                 * if it can't schedule.
4692                 */
4693                if (!(gfp_mask & __GFP_NOMEMALLOC))
4694                        alloc_flags |= ALLOC_HARDER;
4695                /*
4696                 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
4697                 * comment for __cpuset_node_allowed().
4698                 */
4699                alloc_flags &= ~ALLOC_CPUSET;
4700        } else if (unlikely(rt_task(current)) && !in_interrupt())
4701                alloc_flags |= ALLOC_HARDER;
4702
4703        alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);
4704
4705        return alloc_flags;
4706}
4707
4708static bool oom_reserves_allowed(struct task_struct *tsk)
4709{
4710        if (!tsk_is_oom_victim(tsk))
4711                return false;
4712
4713        /*
4714         * !MMU doesn't have oom reaper so give access to memory reserves
4715         * only to the thread with TIF_MEMDIE set
4716         */
4717        if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4718                return false;
4719
4720        return true;
4721}
4722
4723/*
4724 * Distinguish requests which really need access to full memory
4725 * reserves from oom victims which can live with a portion of it
4726 */
4727static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4728{
4729        if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4730                return 0;
4731        if (gfp_mask & __GFP_MEMALLOC)
4732                return ALLOC_NO_WATERMARKS;
4733        if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4734                return ALLOC_NO_WATERMARKS;
4735        if (!in_interrupt()) {
4736                if (current->flags & PF_MEMALLOC)
4737                        return ALLOC_NO_WATERMARKS;
4738                else if (oom_reserves_allowed(current))
4739                        return ALLOC_OOM;
4740        }
4741
4742        return 0;
4743}
4744
4745bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4746{
4747        return !!__gfp_pfmemalloc_flags(gfp_mask);
4748}
4749
4750/*
4751 * Checks whether it makes sense to retry the reclaim to make a forward progress
4752 * for the given allocation request.
4753 *
4754 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
4755 * without success, or when we couldn't even meet the watermark if we
4756 * reclaimed all remaining pages on the LRU lists.
4757 *
4758 * Returns true if a retry is viable or false to enter the oom path.
4759 */
4760static inline bool
4761should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4762                     struct alloc_context *ac, int alloc_flags,
4763                     bool did_some_progress, int *no_progress_loops)
4764{
4765        struct zone *zone;
4766        struct zoneref *z;
4767        bool ret = false;
4768
4769        /*
4770         * Costly allocations might have made a progress but this doesn't mean
4771         * their order will become available due to high fragmentation so
4772         * always increment the no progress counter for them
4773         */
4774        if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4775                *no_progress_loops = 0;
4776        else
4777                (*no_progress_loops)++;
4778
4779        /*
4780         * Make sure we converge to OOM if we cannot make any progress
4781         * several times in the row.
4782         */
4783        if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
4784                /* Before OOM, exhaust highatomic_reserve */
4785                return unreserve_highatomic_pageblock(ac, true);
4786        }
4787
4788        /*
4789         * Keep reclaiming pages while there is a chance this will lead
4790         * somewhere.  If none of the target zones can satisfy our allocation
4791         * request even if all reclaimable pages are considered then we are
4792         * screwed and have to go OOM.
4793         */
4794        for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4795                                ac->highest_zoneidx, ac->nodemask) {
4796                unsigned long available;
4797                unsigned long reclaimable;
4798                unsigned long min_wmark = min_wmark_pages(zone);
4799                bool wmark;
4800
4801                available = reclaimable = zone_reclaimable_pages(zone);
4802                available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
4803
4804                /*
4805                 * Would the allocation succeed if we reclaimed all
4806                 * reclaimable pages?
4807                 */
4808                wmark = __zone_watermark_ok(zone, order, min_wmark,
4809                                ac->highest_zoneidx, alloc_flags, available);
4810                trace_reclaim_retry_zone(z, order, reclaimable,
4811                                available, min_wmark, *no_progress_loops, wmark);
4812                if (wmark) {
4813                        /*
4814                         * If we didn't make any progress and have a lot of
4815                         * dirty + writeback pages then we should wait for
4816                         * an IO to complete to slow down the reclaim and
4817                         * prevent from pre mature OOM
4818                         */
4819                        if (!did_some_progress) {
4820                                unsigned long write_pending;
4821
4822                                write_pending = zone_page_state_snapshot(zone,
4823                                                        NR_ZONE_WRITE_PENDING);
4824
4825                                if (2 * write_pending > reclaimable) {
4826                                        congestion_wait(BLK_RW_ASYNC, HZ/10);
4827                                        return true;
4828                                }
4829                        }
4830
4831                        ret = true;
4832                        goto out;
4833                }
4834        }
4835
4836out:
4837        /*
4838         * Memory allocation/reclaim might be called from a WQ context and the
4839         * current implementation of the WQ concurrency control doesn't
4840         * recognize that a particular WQ is congested if the worker thread is
4841         * looping without ever sleeping. Therefore we have to do a short sleep
4842         * here rather than calling cond_resched().
4843         */
4844        if (current->flags & PF_WQ_WORKER)
4845                schedule_timeout_uninterruptible(1);
4846        else
4847                cond_resched();
4848        return ret;
4849}
4850
4851static inline bool
4852check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4853{
4854        /*
4855         * It's possible that cpuset's mems_allowed and the nodemask from
4856         * mempolicy don't intersect. This should be normally dealt with by
4857         * policy_nodemask(), but it's possible to race with cpuset update in
4858         * such a way the check therein was true, and then it became false
4859         * before we got our cpuset_mems_cookie here.
4860         * This assumes that for all allocations, ac->nodemask can come only
4861         * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
4862         * when it does not intersect with the cpuset restrictions) or the
4863         * caller can deal with a violated nodemask.
4864         */
4865        if (cpusets_enabled() && ac->nodemask &&
4866                        !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4867                ac->nodemask = NULL;
4868                return true;
4869        }
4870
4871        /*
4872         * When updating a task's mems_allowed or mempolicy nodemask, it is
4873         * possible to race with parallel threads in such a way that our
4874         * allocation can fail while the mask is being updated. If we are about
4875         * to fail, check if the cpuset changed during allocation and if so,
4876         * retry.
4877         */
4878        if (read_mems_allowed_retry(cpuset_mems_cookie))
4879                return true;
4880
4881        return false;
4882}
4883
4884static inline struct page *
4885__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
4886                                                struct alloc_context *ac)
4887{
4888        bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
4889        const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
4890        struct page *page = NULL;
4891        unsigned int alloc_flags;
4892        unsigned long did_some_progress;
4893        enum compact_priority compact_priority;
4894        enum compact_result compact_result;
4895        int compaction_retries;
4896        int no_progress_loops;
4897        unsigned int cpuset_mems_cookie;
4898        int reserve_flags;
4899
4900        /*
4901         * We also sanity check to catch abuse of atomic reserves being used by
4902         * callers that are not in atomic context.
4903         */
4904        if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
4905                                (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
4906                gfp_mask &= ~__GFP_ATOMIC;
4907
4908retry_cpuset:
4909        compaction_retries = 0;
4910        no_progress_loops = 0;
4911        compact_priority = DEF_COMPACT_PRIORITY;
4912        cpuset_mems_cookie = read_mems_allowed_begin();
4913
4914        /*
4915         * The fast path uses conservative alloc_flags to succeed only until
4916         * kswapd needs to be woken up, and to avoid the cost of setting up
4917         * alloc_flags precisely. So we do that now.
4918         */
4919        alloc_flags = gfp_to_alloc_flags(gfp_mask);
4920
4921        /*
4922         * We need to recalculate the starting point for the zonelist iterator
4923         * because we might have used different nodemask in the fast path, or
4924         * there was a cpuset modification and we are retrying - otherwise we
4925         * could end up iterating over non-eligible zones endlessly.
4926         */
4927        ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4928                                        ac->highest_zoneidx, ac->nodemask);
4929        if (!ac->preferred_zoneref->zone)
4930                goto nopage;
4931
4932        if (alloc_flags & ALLOC_KSWAPD)
4933                wake_all_kswapds(order, gfp_mask, ac);
4934
4935        /*
4936         * The adjusted alloc_flags might result in immediate success, so try
4937         * that first
4938         */
4939        page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4940        if (page)
4941                goto got_pg;
4942
4943        /*
4944         * For costly allocations, try direct compaction first, as it's likely
4945         * that we have enough base pages and don't need to reclaim. For non-
4946         * movable high-order allocations, do that as well, as compaction will
4947         * try prevent permanent fragmentation by migrating from blocks of the
4948         * same migratetype.
4949         * Don't try this for allocations that are allowed to ignore
4950         * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
4951         */
4952        if (can_direct_reclaim &&
4953                        (costly_order ||
4954                           (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4955                        && !gfp_pfmemalloc_allowed(gfp_mask)) {
4956                page = __alloc_pages_direct_compact(gfp_mask, order,
4957                                                alloc_flags, ac,
4958                                                INIT_COMPACT_PRIORITY,
4959                                                &compact_result);
4960                if (page)
4961                        goto got_pg;
4962
4963                /*
4964                 * Checks for costly allocations with __GFP_NORETRY, which
4965                 * includes some THP page fault allocations
4966                 */
4967                if (costly_order && (gfp_mask & __GFP_NORETRY)) {
4968                        /*
4969                         * If allocating entire pageblock(s) and compaction
4970                         * failed because all zones are below low watermarks
4971                         * or is prohibited because it recently failed at this
4972                         * order, fail immediately unless the allocator has
4973                         * requested compaction and reclaim retry.
4974                         *
4975                         * Reclaim is
4976                         *  - potentially very expensive because zones are far
4977                         *    below their low watermarks or this is part of very
4978                         *    bursty high order allocations,
4979                         *  - not guaranteed to help because isolate_freepages()
4980                         *    may not iterate over freed pages as part of its
4981                         *    linear scan, and
4982                         *  - unlikely to make entire pageblocks free on its
4983                         *    own.
4984                         */
4985                        if (compact_result == COMPACT_SKIPPED ||
4986                            compact_result == COMPACT_DEFERRED)
4987                                goto nopage;
4988
4989                        /*
4990                         * Looks like reclaim/compaction is worth trying, but
4991                         * sync compaction could be very expensive, so keep
4992                         * using async compaction.
4993                         */
4994                        compact_priority = INIT_COMPACT_PRIORITY;
4995                }
4996        }
4997
4998retry:
4999        /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
5000        if (alloc_flags & ALLOC_KSWAPD)
5001                wake_all_kswapds(order, gfp_mask, ac);
5002
5003        reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
5004        if (reserve_flags)
5005                alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags);
5006
5007        /*
5008         * Reset the nodemask and zonelist iterators if memory policies can be
5009         * ignored. These allocations are high priority and system rather than
5010         * user oriented.
5011         */
5012        if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
5013                ac->nodemask = NULL;
5014                ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
5015                                        ac->highest_zoneidx, ac->nodemask);
5016        }
5017
5018        /* Attempt with potentially adjusted zonelist and alloc_flags */
5019        page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
5020        if (page)
5021                goto got_pg;
5022
5023        /* Caller is not willing to reclaim, we can't balance anything */
5024        if (!can_direct_reclaim)
5025                goto nopage;
5026
5027        /* Avoid recursion of direct reclaim */
5028        if (current->flags & PF_MEMALLOC)
5029                goto nopage;
5030
5031        /* Try direct reclaim and then allocating */
5032        page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
5033                                                        &did_some_progress);
5034        if (page)
5035                goto got_pg;
5036
5037        /* Try direct compaction and then allocating */
5038        page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
5039                                        compact_priority, &compact_result);
5040        if (page)
5041                goto got_pg;
5042
5043        /* Do not loop if specifically requested */
5044        if (gfp_mask & __GFP_NORETRY)
5045                goto nopage;
5046
5047        /*
5048         * Do not retry costly high order allocations unless they are
5049         * __GFP_RETRY_MAYFAIL
5050         */
5051        if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
5052                goto nopage;
5053
5054        if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
5055                                 did_some_progress > 0, &no_progress_loops))
5056                goto retry;
5057
5058        /*
5059         * It doesn't make any sense to retry for the compaction if the order-0
5060         * reclaim is not able to make any progress because the current
5061         * implementation of the compaction depends on the sufficient amount
5062         * of free memory (see __compaction_suitable)
5063         */
5064        if (did_some_progress > 0 &&
5065                        should_compact_retry(ac, order, alloc_flags,
5066                                compact_result, &compact_priority,
5067                                &compaction_retries))
5068                goto retry;
5069
5070
5071        /* Deal with possible cpuset update races before we start OOM killing */
5072        if (check_retry_cpuset(cpuset_mems_cookie, ac))
5073                goto retry_cpuset;
5074
5075        /* Reclaim has failed us, start killing