linux/mm/vmscan.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   4 *
   5 *  Swap reorganised 29.12.95, Stephen Tweedie.
   6 *  kswapd added: 7.1.96  sct
   7 *  Removed kswapd_ctl limits, and swap out as many pages as needed
   8 *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
   9 *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
  10 *  Multiqueue VM started 5.8.00, Rik van Riel.
  11 */
  12
  13#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  14
  15#include <linux/mm.h>
  16#include <linux/sched/mm.h>
  17#include <linux/module.h>
  18#include <linux/gfp.h>
  19#include <linux/kernel_stat.h>
  20#include <linux/swap.h>
  21#include <linux/pagemap.h>
  22#include <linux/init.h>
  23#include <linux/highmem.h>
  24#include <linux/vmpressure.h>
  25#include <linux/vmstat.h>
  26#include <linux/file.h>
  27#include <linux/writeback.h>
  28#include <linux/blkdev.h>
  29#include <linux/buffer_head.h>  /* for try_to_release_page(),
  30                                        buffer_heads_over_limit */
  31#include <linux/mm_inline.h>
  32#include <linux/backing-dev.h>
  33#include <linux/rmap.h>
  34#include <linux/topology.h>
  35#include <linux/cpu.h>
  36#include <linux/cpuset.h>
  37#include <linux/compaction.h>
  38#include <linux/notifier.h>
  39#include <linux/rwsem.h>
  40#include <linux/delay.h>
  41#include <linux/kthread.h>
  42#include <linux/freezer.h>
  43#include <linux/memcontrol.h>
  44#include <linux/delayacct.h>
  45#include <linux/sysctl.h>
  46#include <linux/oom.h>
  47#include <linux/pagevec.h>
  48#include <linux/prefetch.h>
  49#include <linux/printk.h>
  50#include <linux/dax.h>
  51#include <linux/psi.h>
  52
  53#include <asm/tlbflush.h>
  54#include <asm/div64.h>
  55
  56#include <linux/swapops.h>
  57#include <linux/balloon_compaction.h>
  58
  59#include "internal.h"
  60
  61#define CREATE_TRACE_POINTS
  62#include <trace/events/vmscan.h>
  63
  64struct scan_control {
  65        /* How many pages shrink_list() should reclaim */
  66        unsigned long nr_to_reclaim;
  67
  68        /*
  69         * Nodemask of nodes allowed by the caller. If NULL, all nodes
  70         * are scanned.
  71         */
  72        nodemask_t      *nodemask;
  73
  74        /*
  75         * The memory cgroup that hit its limit and as a result is the
  76         * primary target of this reclaim invocation.
  77         */
  78        struct mem_cgroup *target_mem_cgroup;
  79
  80        /*
  81         * Scan pressure balancing between anon and file LRUs
  82         */
  83        unsigned long   anon_cost;
  84        unsigned long   file_cost;
  85
  86        /* Can active pages be deactivated as part of reclaim? */
  87#define DEACTIVATE_ANON 1
  88#define DEACTIVATE_FILE 2
  89        unsigned int may_deactivate:2;
  90        unsigned int force_deactivate:1;
  91        unsigned int skipped_deactivate:1;
  92
  93        /* Writepage batching in laptop mode; RECLAIM_WRITE */
  94        unsigned int may_writepage:1;
  95
  96        /* Can mapped pages be reclaimed? */
  97        unsigned int may_unmap:1;
  98
  99        /* Can pages be swapped as part of reclaim? */
 100        unsigned int may_swap:1;
 101
 102        /*
 103         * Cgroup memory below memory.low is protected as long as we
 104         * don't threaten to OOM. If any cgroup is reclaimed at
 105         * reduced force or passed over entirely due to its memory.low
 106         * setting (memcg_low_skipped), and nothing is reclaimed as a
 107         * result, then go back for one more cycle that reclaims the protected
 108         * memory (memcg_low_reclaim) to avert OOM.
 109         */
 110        unsigned int memcg_low_reclaim:1;
 111        unsigned int memcg_low_skipped:1;
 112
 113        unsigned int hibernation_mode:1;
 114
 115        /* One of the zones is ready for compaction */
 116        unsigned int compaction_ready:1;
 117
 118        /* There is easily reclaimable cold cache in the current node */
 119        unsigned int cache_trim_mode:1;
 120
 121        /* The file pages on the current node are dangerously low */
 122        unsigned int file_is_tiny:1;
 123
 124        /* Allocation order */
 125        s8 order;
 126
 127        /* Scan (total_size >> priority) pages at once */
 128        s8 priority;
 129
 130        /* The highest zone to isolate pages for reclaim from */
 131        s8 reclaim_idx;
 132
 133        /* This context's GFP mask */
 134        gfp_t gfp_mask;
 135
 136        /* Incremented by the number of inactive pages that were scanned */
 137        unsigned long nr_scanned;
 138
 139        /* Number of pages freed so far during a call to shrink_zones() */
 140        unsigned long nr_reclaimed;
 141
 142        struct {
 143                unsigned int dirty;
 144                unsigned int unqueued_dirty;
 145                unsigned int congested;
 146                unsigned int writeback;
 147                unsigned int immediate;
 148                unsigned int file_taken;
 149                unsigned int taken;
 150        } nr;
 151
 152        /* for recording the reclaimed slab by now */
 153        struct reclaim_state reclaim_state;
 154};
 155
 156#ifdef ARCH_HAS_PREFETCHW
 157#define prefetchw_prev_lru_page(_page, _base, _field)                   \
 158        do {                                                            \
 159                if ((_page)->lru.prev != _base) {                       \
 160                        struct page *prev;                              \
 161                                                                        \
 162                        prev = lru_to_page(&(_page->lru));              \
 163                        prefetchw(&prev->_field);                       \
 164                }                                                       \
 165        } while (0)
 166#else
 167#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
 168#endif
 169
 170/*
 171 * From 0 .. 200.  Higher means more swappy.
 172 */
 173int vm_swappiness = 60;
 174
 175static void set_task_reclaim_state(struct task_struct *task,
 176                                   struct reclaim_state *rs)
 177{
 178        /* Check for an overwrite */
 179        WARN_ON_ONCE(rs && task->reclaim_state);
 180
 181        /* Check for the nulling of an already-nulled member */
 182        WARN_ON_ONCE(!rs && !task->reclaim_state);
 183
 184        task->reclaim_state = rs;
 185}
 186
 187static LIST_HEAD(shrinker_list);
 188static DECLARE_RWSEM(shrinker_rwsem);
 189
 190#ifdef CONFIG_MEMCG
 191static int shrinker_nr_max;
 192
 193/* The shrinker_info is expanded in a batch of BITS_PER_LONG */
 194static inline int shrinker_map_size(int nr_items)
 195{
 196        return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long));
 197}
 198
 199static inline int shrinker_defer_size(int nr_items)
 200{
 201        return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t));
 202}
 203
 204static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
 205                                                     int nid)
 206{
 207        return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,
 208                                         lockdep_is_held(&shrinker_rwsem));
 209}
 210
 211static int expand_one_shrinker_info(struct mem_cgroup *memcg,
 212                                    int map_size, int defer_size,
 213                                    int old_map_size, int old_defer_size)
 214{
 215        struct shrinker_info *new, *old;
 216        struct mem_cgroup_per_node *pn;
 217        int nid;
 218        int size = map_size + defer_size;
 219
 220        for_each_node(nid) {
 221                pn = memcg->nodeinfo[nid];
 222                old = shrinker_info_protected(memcg, nid);
 223                /* Not yet online memcg */
 224                if (!old)
 225                        return 0;
 226
 227                new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
 228                if (!new)
 229                        return -ENOMEM;
 230
 231                new->nr_deferred = (atomic_long_t *)(new + 1);
 232                new->map = (void *)new->nr_deferred + defer_size;
 233
 234                /* map: set all old bits, clear all new bits */
 235                memset(new->map, (int)0xff, old_map_size);
 236                memset((void *)new->map + old_map_size, 0, map_size - old_map_size);
 237                /* nr_deferred: copy old values, clear all new values */
 238                memcpy(new->nr_deferred, old->nr_deferred, old_defer_size);
 239                memset((void *)new->nr_deferred + old_defer_size, 0,
 240                       defer_size - old_defer_size);
 241
 242                rcu_assign_pointer(pn->shrinker_info, new);
 243                kvfree_rcu(old, rcu);
 244        }
 245
 246        return 0;
 247}
 248
 249void free_shrinker_info(struct mem_cgroup *memcg)
 250{
 251        struct mem_cgroup_per_node *pn;
 252        struct shrinker_info *info;
 253        int nid;
 254
 255        for_each_node(nid) {
 256                pn = memcg->nodeinfo[nid];
 257                info = rcu_dereference_protected(pn->shrinker_info, true);
 258                kvfree(info);
 259                rcu_assign_pointer(pn->shrinker_info, NULL);
 260        }
 261}
 262
 263int alloc_shrinker_info(struct mem_cgroup *memcg)
 264{
 265        struct shrinker_info *info;
 266        int nid, size, ret = 0;
 267        int map_size, defer_size = 0;
 268
 269        down_write(&shrinker_rwsem);
 270        map_size = shrinker_map_size(shrinker_nr_max);
 271        defer_size = shrinker_defer_size(shrinker_nr_max);
 272        size = map_size + defer_size;
 273        for_each_node(nid) {
 274                info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid);
 275                if (!info) {
 276                        free_shrinker_info(memcg);
 277                        ret = -ENOMEM;
 278                        break;
 279                }
 280                info->nr_deferred = (atomic_long_t *)(info + 1);
 281                info->map = (void *)info->nr_deferred + defer_size;
 282                rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
 283        }
 284        up_write(&shrinker_rwsem);
 285
 286        return ret;
 287}
 288
 289static inline bool need_expand(int nr_max)
 290{
 291        return round_up(nr_max, BITS_PER_LONG) >
 292               round_up(shrinker_nr_max, BITS_PER_LONG);
 293}
 294
 295static int expand_shrinker_info(int new_id)
 296{
 297        int ret = 0;
 298        int new_nr_max = new_id + 1;
 299        int map_size, defer_size = 0;
 300        int old_map_size, old_defer_size = 0;
 301        struct mem_cgroup *memcg;
 302
 303        if (!need_expand(new_nr_max))
 304                goto out;
 305
 306        if (!root_mem_cgroup)
 307                goto out;
 308
 309        lockdep_assert_held(&shrinker_rwsem);
 310
 311        map_size = shrinker_map_size(new_nr_max);
 312        defer_size = shrinker_defer_size(new_nr_max);
 313        old_map_size = shrinker_map_size(shrinker_nr_max);
 314        old_defer_size = shrinker_defer_size(shrinker_nr_max);
 315
 316        memcg = mem_cgroup_iter(NULL, NULL, NULL);
 317        do {
 318                ret = expand_one_shrinker_info(memcg, map_size, defer_size,
 319                                               old_map_size, old_defer_size);
 320                if (ret) {
 321                        mem_cgroup_iter_break(NULL, memcg);
 322                        goto out;
 323                }
 324        } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
 325out:
 326        if (!ret)
 327                shrinker_nr_max = new_nr_max;
 328
 329        return ret;
 330}
 331
 332void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
 333{
 334        if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
 335                struct shrinker_info *info;
 336
 337                rcu_read_lock();
 338                info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
 339                /* Pairs with smp mb in shrink_slab() */
 340                smp_mb__before_atomic();
 341                set_bit(shrinker_id, info->map);
 342                rcu_read_unlock();
 343        }
 344}
 345
 346static DEFINE_IDR(shrinker_idr);
 347
 348static int prealloc_memcg_shrinker(struct shrinker *shrinker)
 349{
 350        int id, ret = -ENOMEM;
 351
 352        if (mem_cgroup_disabled())
 353                return -ENOSYS;
 354
 355        down_write(&shrinker_rwsem);
 356        /* This may call shrinker, so it must use down_read_trylock() */
 357        id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
 358        if (id < 0)
 359                goto unlock;
 360
 361        if (id >= shrinker_nr_max) {
 362                if (expand_shrinker_info(id)) {
 363                        idr_remove(&shrinker_idr, id);
 364                        goto unlock;
 365                }
 366        }
 367        shrinker->id = id;
 368        ret = 0;
 369unlock:
 370        up_write(&shrinker_rwsem);
 371        return ret;
 372}
 373
 374static void unregister_memcg_shrinker(struct shrinker *shrinker)
 375{
 376        int id = shrinker->id;
 377
 378        BUG_ON(id < 0);
 379
 380        lockdep_assert_held(&shrinker_rwsem);
 381
 382        idr_remove(&shrinker_idr, id);
 383}
 384
 385static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
 386                                   struct mem_cgroup *memcg)
 387{
 388        struct shrinker_info *info;
 389
 390        info = shrinker_info_protected(memcg, nid);
 391        return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0);
 392}
 393
 394static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
 395                                  struct mem_cgroup *memcg)
 396{
 397        struct shrinker_info *info;
 398
 399        info = shrinker_info_protected(memcg, nid);
 400        return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]);
 401}
 402
 403void reparent_shrinker_deferred(struct mem_cgroup *memcg)
 404{
 405        int i, nid;
 406        long nr;
 407        struct mem_cgroup *parent;
 408        struct shrinker_info *child_info, *parent_info;
 409
 410        parent = parent_mem_cgroup(memcg);
 411        if (!parent)
 412                parent = root_mem_cgroup;
 413
 414        /* Prevent from concurrent shrinker_info expand */
 415        down_read(&shrinker_rwsem);
 416        for_each_node(nid) {
 417                child_info = shrinker_info_protected(memcg, nid);
 418                parent_info = shrinker_info_protected(parent, nid);
 419                for (i = 0; i < shrinker_nr_max; i++) {
 420                        nr = atomic_long_read(&child_info->nr_deferred[i]);
 421                        atomic_long_add(nr, &parent_info->nr_deferred[i]);
 422                }
 423        }
 424        up_read(&shrinker_rwsem);
 425}
 426
 427static bool cgroup_reclaim(struct scan_control *sc)
 428{
 429        return sc->target_mem_cgroup;
 430}
 431
 432/**
 433 * writeback_throttling_sane - is the usual dirty throttling mechanism available?
 434 * @sc: scan_control in question
 435 *
 436 * The normal page dirty throttling mechanism in balance_dirty_pages() is
 437 * completely broken with the legacy memcg and direct stalling in
 438 * shrink_page_list() is used for throttling instead, which lacks all the
 439 * niceties such as fairness, adaptive pausing, bandwidth proportional
 440 * allocation and configurability.
 441 *
 442 * This function tests whether the vmscan currently in progress can assume
 443 * that the normal dirty throttling mechanism is operational.
 444 */
 445static bool writeback_throttling_sane(struct scan_control *sc)
 446{
 447        if (!cgroup_reclaim(sc))
 448                return true;
 449#ifdef CONFIG_CGROUP_WRITEBACK
 450        if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
 451                return true;
 452#endif
 453        return false;
 454}
 455#else
 456static int prealloc_memcg_shrinker(struct shrinker *shrinker)
 457{
 458        return -ENOSYS;
 459}
 460
 461static void unregister_memcg_shrinker(struct shrinker *shrinker)
 462{
 463}
 464
 465static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
 466                                   struct mem_cgroup *memcg)
 467{
 468        return 0;
 469}
 470
 471static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
 472                                  struct mem_cgroup *memcg)
 473{
 474        return 0;
 475}
 476
 477static bool cgroup_reclaim(struct scan_control *sc)
 478{
 479        return false;
 480}
 481
 482static bool writeback_throttling_sane(struct scan_control *sc)
 483{
 484        return true;
 485}
 486#endif
 487
 488static long xchg_nr_deferred(struct shrinker *shrinker,
 489                             struct shrink_control *sc)
 490{
 491        int nid = sc->nid;
 492
 493        if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
 494                nid = 0;
 495
 496        if (sc->memcg &&
 497            (shrinker->flags & SHRINKER_MEMCG_AWARE))
 498                return xchg_nr_deferred_memcg(nid, shrinker,
 499                                              sc->memcg);
 500
 501        return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
 502}
 503
 504
 505static long add_nr_deferred(long nr, struct shrinker *shrinker,
 506                            struct shrink_control *sc)
 507{
 508        int nid = sc->nid;
 509
 510        if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
 511                nid = 0;
 512
 513        if (sc->memcg &&
 514            (shrinker->flags & SHRINKER_MEMCG_AWARE))
 515                return add_nr_deferred_memcg(nr, nid, shrinker,
 516                                             sc->memcg);
 517
 518        return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);
 519}
 520
 521/*
 522 * This misses isolated pages which are not accounted for to save counters.
 523 * As the data only determines if reclaim or compaction continues, it is
 524 * not expected that isolated pages will be a dominating factor.
 525 */
 526unsigned long zone_reclaimable_pages(struct zone *zone)
 527{
 528        unsigned long nr;
 529
 530        nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
 531                zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
 532        if (get_nr_swap_pages() > 0)
 533                nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
 534                        zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
 535
 536        return nr;
 537}
 538
 539/**
 540 * lruvec_lru_size -  Returns the number of pages on the given LRU list.
 541 * @lruvec: lru vector
 542 * @lru: lru to use
 543 * @zone_idx: zones to consider (use MAX_NR_ZONES for the whole LRU list)
 544 */
 545static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
 546                                     int zone_idx)
 547{
 548        unsigned long size = 0;
 549        int zid;
 550
 551        for (zid = 0; zid <= zone_idx && zid < MAX_NR_ZONES; zid++) {
 552                struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
 553
 554                if (!managed_zone(zone))
 555                        continue;
 556
 557                if (!mem_cgroup_disabled())
 558                        size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
 559                else
 560                        size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
 561        }
 562        return size;
 563}
 564
 565/*
 566 * Add a shrinker callback to be called from the vm.
 567 */
 568int prealloc_shrinker(struct shrinker *shrinker)
 569{
 570        unsigned int size;
 571        int err;
 572
 573        if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
 574                err = prealloc_memcg_shrinker(shrinker);
 575                if (err != -ENOSYS)
 576                        return err;
 577
 578                shrinker->flags &= ~SHRINKER_MEMCG_AWARE;
 579        }
 580
 581        size = sizeof(*shrinker->nr_deferred);
 582        if (shrinker->flags & SHRINKER_NUMA_AWARE)
 583                size *= nr_node_ids;
 584
 585        shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
 586        if (!shrinker->nr_deferred)
 587                return -ENOMEM;
 588
 589        return 0;
 590}
 591
 592void free_prealloced_shrinker(struct shrinker *shrinker)
 593{
 594        if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
 595                down_write(&shrinker_rwsem);
 596                unregister_memcg_shrinker(shrinker);
 597                up_write(&shrinker_rwsem);
 598                return;
 599        }
 600
 601        kfree(shrinker->nr_deferred);
 602        shrinker->nr_deferred = NULL;
 603}
 604
 605void register_shrinker_prepared(struct shrinker *shrinker)
 606{
 607        down_write(&shrinker_rwsem);
 608        list_add_tail(&shrinker->list, &shrinker_list);
 609        shrinker->flags |= SHRINKER_REGISTERED;
 610        up_write(&shrinker_rwsem);
 611}
 612
 613int register_shrinker(struct shrinker *shrinker)
 614{
 615        int err = prealloc_shrinker(shrinker);
 616
 617        if (err)
 618                return err;
 619        register_shrinker_prepared(shrinker);
 620        return 0;
 621}
 622EXPORT_SYMBOL(register_shrinker);
 623
 624/*
 625 * Remove one
 626 */
 627void unregister_shrinker(struct shrinker *shrinker)
 628{
 629        if (!(shrinker->flags & SHRINKER_REGISTERED))
 630                return;
 631
 632        down_write(&shrinker_rwsem);
 633        list_del(&shrinker->list);
 634        shrinker->flags &= ~SHRINKER_REGISTERED;
 635        if (shrinker->flags & SHRINKER_MEMCG_AWARE)
 636                unregister_memcg_shrinker(shrinker);
 637        up_write(&shrinker_rwsem);
 638
 639        kfree(shrinker->nr_deferred);
 640        shrinker->nr_deferred = NULL;
 641}
 642EXPORT_SYMBOL(unregister_shrinker);
 643
 644#define SHRINK_BATCH 128
 645
 646static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
 647                                    struct shrinker *shrinker, int priority)
 648{
 649        unsigned long freed = 0;
 650        unsigned long long delta;
 651        long total_scan;
 652        long freeable;
 653        long nr;
 654        long new_nr;
 655        long batch_size = shrinker->batch ? shrinker->batch
 656                                          : SHRINK_BATCH;
 657        long scanned = 0, next_deferred;
 658
 659        freeable = shrinker->count_objects(shrinker, shrinkctl);
 660        if (freeable == 0 || freeable == SHRINK_EMPTY)
 661                return freeable;
 662
 663        /*
 664         * copy the current shrinker scan count into a local variable
 665         * and zero it so that other concurrent shrinker invocations
 666         * don't also do this scanning work.
 667         */
 668        nr = xchg_nr_deferred(shrinker, shrinkctl);
 669
 670        if (shrinker->seeks) {
 671                delta = freeable >> priority;
 672                delta *= 4;
 673                do_div(delta, shrinker->seeks);
 674        } else {
 675                /*
 676                 * These objects don't require any IO to create. Trim
 677                 * them aggressively under memory pressure to keep
 678                 * them from causing refetches in the IO caches.
 679                 */
 680                delta = freeable / 2;
 681        }
 682
 683        total_scan = nr >> priority;
 684        total_scan += delta;
 685        total_scan = min(total_scan, (2 * freeable));
 686
 687        trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
 688                                   freeable, delta, total_scan, priority);
 689
 690        /*
 691         * Normally, we should not scan less than batch_size objects in one
 692         * pass to avoid too frequent shrinker calls, but if the slab has less
 693         * than batch_size objects in total and we are really tight on memory,
 694         * we will try to reclaim all available objects, otherwise we can end
 695         * up failing allocations although there are plenty of reclaimable
 696         * objects spread over several slabs with usage less than the
 697         * batch_size.
 698         *
 699         * We detect the "tight on memory" situations by looking at the total
 700         * number of objects we want to scan (total_scan). If it is greater
 701         * than the total number of objects on slab (freeable), we must be
 702         * scanning at high prio and therefore should try to reclaim as much as
 703         * possible.
 704         */
 705        while (total_scan >= batch_size ||
 706               total_scan >= freeable) {
 707                unsigned long ret;
 708                unsigned long nr_to_scan = min(batch_size, total_scan);
 709
 710                shrinkctl->nr_to_scan = nr_to_scan;
 711                shrinkctl->nr_scanned = nr_to_scan;
 712                ret = shrinker->scan_objects(shrinker, shrinkctl);
 713                if (ret == SHRINK_STOP)
 714                        break;
 715                freed += ret;
 716
 717                count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
 718                total_scan -= shrinkctl->nr_scanned;
 719                scanned += shrinkctl->nr_scanned;
 720
 721                cond_resched();
 722        }
 723
 724        /*
 725         * The deferred work is increased by any new work (delta) that wasn't
 726         * done, decreased by old deferred work that was done now.
 727         *
 728         * And it is capped to two times of the freeable items.
 729         */
 730        next_deferred = max_t(long, (nr + delta - scanned), 0);
 731        next_deferred = min(next_deferred, (2 * freeable));
 732
 733        /*
 734         * move the unused scan count back into the shrinker in a
 735         * manner that handles concurrent updates.
 736         */
 737        new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl);
 738
 739        trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan);
 740        return freed;
 741}
 742
 743#ifdef CONFIG_MEMCG
 744static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
 745                        struct mem_cgroup *memcg, int priority)
 746{
 747        struct shrinker_info *info;
 748        unsigned long ret, freed = 0;
 749        int i;
 750
 751        if (!mem_cgroup_online(memcg))
 752                return 0;
 753
 754        if (!down_read_trylock(&shrinker_rwsem))
 755                return 0;
 756
 757        info = shrinker_info_protected(memcg, nid);
 758        if (unlikely(!info))
 759                goto unlock;
 760
 761        for_each_set_bit(i, info->map, shrinker_nr_max) {
 762                struct shrink_control sc = {
 763                        .gfp_mask = gfp_mask,
 764                        .nid = nid,
 765                        .memcg = memcg,
 766                };
 767                struct shrinker *shrinker;
 768
 769                shrinker = idr_find(&shrinker_idr, i);
 770                if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) {
 771                        if (!shrinker)
 772                                clear_bit(i, info->map);
 773                        continue;
 774                }
 775
 776                /* Call non-slab shrinkers even though kmem is disabled */
 777                if (!memcg_kmem_enabled() &&
 778                    !(shrinker->flags & SHRINKER_NONSLAB))
 779                        continue;
 780
 781                ret = do_shrink_slab(&sc, shrinker, priority);
 782                if (ret == SHRINK_EMPTY) {
 783                        clear_bit(i, info->map);
 784                        /*
 785                         * After the shrinker reported that it had no objects to
 786                         * free, but before we cleared the corresponding bit in
 787                         * the memcg shrinker map, a new object might have been
 788                         * added. To make sure, we have the bit set in this
 789                         * case, we invoke the shrinker one more time and reset
 790                         * the bit if it reports that it is not empty anymore.
 791                         * The memory barrier here pairs with the barrier in
 792                         * set_shrinker_bit():
 793                         *
 794                         * list_lru_add()     shrink_slab_memcg()
 795                         *   list_add_tail()    clear_bit()
 796                         *   <MB>               <MB>
 797                         *   set_bit()          do_shrink_slab()
 798                         */
 799                        smp_mb__after_atomic();
 800                        ret = do_shrink_slab(&sc, shrinker, priority);
 801                        if (ret == SHRINK_EMPTY)
 802                                ret = 0;
 803                        else
 804                                set_shrinker_bit(memcg, nid, i);
 805                }
 806                freed += ret;
 807
 808                if (rwsem_is_contended(&shrinker_rwsem)) {
 809                        freed = freed ? : 1;
 810                        break;
 811                }
 812        }
 813unlock:
 814        up_read(&shrinker_rwsem);
 815        return freed;
 816}
 817#else /* CONFIG_MEMCG */
 818static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
 819                        struct mem_cgroup *memcg, int priority)
 820{
 821        return 0;
 822}
 823#endif /* CONFIG_MEMCG */
 824
 825/**
 826 * shrink_slab - shrink slab caches
 827 * @gfp_mask: allocation context
 828 * @nid: node whose slab caches to target
 829 * @memcg: memory cgroup whose slab caches to target
 830 * @priority: the reclaim priority
 831 *
 832 * Call the shrink functions to age shrinkable caches.
 833 *
 834 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
 835 * unaware shrinkers will receive a node id of 0 instead.
 836 *
 837 * @memcg specifies the memory cgroup to target. Unaware shrinkers
 838 * are called only if it is the root cgroup.
 839 *
 840 * @priority is sc->priority, we take the number of objects and >> by priority
 841 * in order to get the scan target.
 842 *
 843 * Returns the number of reclaimed slab objects.
 844 */
 845static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
 846                                 struct mem_cgroup *memcg,
 847                                 int priority)
 848{
 849        unsigned long ret, freed = 0;
 850        struct shrinker *shrinker;
 851
 852        /*
 853         * The root memcg might be allocated even though memcg is disabled
 854         * via "cgroup_disable=memory" boot parameter.  This could make
 855         * mem_cgroup_is_root() return false, then just run memcg slab
 856         * shrink, but skip global shrink.  This may result in premature
 857         * oom.
 858         */
 859        if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
 860                return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
 861
 862        if (!down_read_trylock(&shrinker_rwsem))
 863                goto out;
 864
 865        list_for_each_entry(shrinker, &shrinker_list, list) {
 866                struct shrink_control sc = {
 867                        .gfp_mask = gfp_mask,
 868                        .nid = nid,
 869                        .memcg = memcg,
 870                };
 871
 872                ret = do_shrink_slab(&sc, shrinker, priority);
 873                if (ret == SHRINK_EMPTY)
 874                        ret = 0;
 875                freed += ret;
 876                /*
 877                 * Bail out if someone want to register a new shrinker to
 878                 * prevent the registration from being stalled for long periods
 879                 * by parallel ongoing shrinking.
 880                 */
 881                if (rwsem_is_contended(&shrinker_rwsem)) {
 882                        freed = freed ? : 1;
 883                        break;
 884                }
 885        }
 886
 887        up_read(&shrinker_rwsem);
 888out:
 889        cond_resched();
 890        return freed;
 891}
 892
 893void drop_slab_node(int nid)
 894{
 895        unsigned long freed;
 896
 897        do {
 898                struct mem_cgroup *memcg = NULL;
 899
 900                if (fatal_signal_pending(current))
 901                        return;
 902
 903                freed = 0;
 904                memcg = mem_cgroup_iter(NULL, NULL, NULL);
 905                do {
 906                        freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
 907                } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
 908        } while (freed > 10);
 909}
 910
 911void drop_slab(void)
 912{
 913        int nid;
 914
 915        for_each_online_node(nid)
 916                drop_slab_node(nid);
 917}
 918
 919static inline int is_page_cache_freeable(struct page *page)
 920{
 921        /*
 922         * A freeable page cache page is referenced only by the caller
 923         * that isolated the page, the page cache and optional buffer
 924         * heads at page->private.
 925         */
 926        int page_cache_pins = thp_nr_pages(page);
 927        return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
 928}
 929
 930static int may_write_to_inode(struct inode *inode)
 931{
 932        if (current->flags & PF_SWAPWRITE)
 933                return 1;
 934        if (!inode_write_congested(inode))
 935                return 1;
 936        if (inode_to_bdi(inode) == current->backing_dev_info)
 937                return 1;
 938        return 0;
 939}
 940
 941/*
 942 * We detected a synchronous write error writing a page out.  Probably
 943 * -ENOSPC.  We need to propagate that into the address_space for a subsequent
 944 * fsync(), msync() or close().
 945 *
 946 * The tricky part is that after writepage we cannot touch the mapping: nothing
 947 * prevents it from being freed up.  But we have a ref on the page and once
 948 * that page is locked, the mapping is pinned.
 949 *
 950 * We're allowed to run sleeping lock_page() here because we know the caller has
 951 * __GFP_FS.
 952 */
 953static void handle_write_error(struct address_space *mapping,
 954                                struct page *page, int error)
 955{
 956        lock_page(page);
 957        if (page_mapping(page) == mapping)
 958                mapping_set_error(mapping, error);
 959        unlock_page(page);
 960}
 961
 962/* possible outcome of pageout() */
 963typedef enum {
 964        /* failed to write page out, page is locked */
 965        PAGE_KEEP,
 966        /* move page to the active list, page is locked */
 967        PAGE_ACTIVATE,
 968        /* page has been sent to the disk successfully, page is unlocked */
 969        PAGE_SUCCESS,
 970        /* page is clean and locked */
 971        PAGE_CLEAN,
 972} pageout_t;
 973
 974/*
 975 * pageout is called by shrink_page_list() for each dirty page.
 976 * Calls ->writepage().
 977 */
 978static pageout_t pageout(struct page *page, struct address_space *mapping)
 979{
 980        /*
 981         * If the page is dirty, only perform writeback if that write
 982         * will be non-blocking.  To prevent this allocation from being
 983         * stalled by pagecache activity.  But note that there may be
 984         * stalls if we need to run get_block().  We could test
 985         * PagePrivate for that.
 986         *
 987         * If this process is currently in __generic_file_write_iter() against
 988         * this page's queue, we can perform writeback even if that
 989         * will block.
 990         *
 991         * If the page is swapcache, write it back even if that would
 992         * block, for some throttling. This happens by accident, because
 993         * swap_backing_dev_info is bust: it doesn't reflect the
 994         * congestion state of the swapdevs.  Easy to fix, if needed.
 995         */
 996        if (!is_page_cache_freeable(page))
 997                return PAGE_KEEP;
 998        if (!mapping) {
 999                /*
1000                 * Some data journaling orphaned pages can have
1001                 * page->mapping == NULL while being dirty with clean buffers.
1002                 */
1003                if (page_has_private(page)) {
1004                        if (try_to_free_buffers(page)) {
1005                                ClearPageDirty(page);
1006                                pr_info("%s: orphaned page\n", __func__);
1007                                return PAGE_CLEAN;
1008                        }
1009                }
1010                return PAGE_KEEP;
1011        }
1012        if (mapping->a_ops->writepage == NULL)
1013                return PAGE_ACTIVATE;
1014        if (!may_write_to_inode(mapping->host))
1015                return PAGE_KEEP;
1016
1017        if (clear_page_dirty_for_io(page)) {
1018                int res;
1019                struct writeback_control wbc = {
1020                        .sync_mode = WB_SYNC_NONE,
1021                        .nr_to_write = SWAP_CLUSTER_MAX,
1022                        .range_start = 0,
1023                        .range_end = LLONG_MAX,
1024                        .for_reclaim = 1,
1025                };
1026
1027                SetPageReclaim(page);
1028                res = mapping->a_ops->writepage(page, &wbc);
1029                if (res < 0)
1030                        handle_write_error(mapping, page, res);
1031                if (res == AOP_WRITEPAGE_ACTIVATE) {
1032                        ClearPageReclaim(page);
1033                        return PAGE_ACTIVATE;
1034                }
1035
1036                if (!PageWriteback(page)) {
1037                        /* synchronous write or broken a_ops? */
1038                        ClearPageReclaim(page);
1039                }
1040                trace_mm_vmscan_writepage(page);
1041                inc_node_page_state(page, NR_VMSCAN_WRITE);
1042                return PAGE_SUCCESS;
1043        }
1044
1045        return PAGE_CLEAN;
1046}
1047
1048/*
1049 * Same as remove_mapping, but if the page is removed from the mapping, it
1050 * gets returned with a refcount of 0.
1051 */
1052static int __remove_mapping(struct address_space *mapping, struct page *page,
1053                            bool reclaimed, struct mem_cgroup *target_memcg)
1054{
1055        unsigned long flags;
1056        int refcount;
1057        void *shadow = NULL;
1058
1059        BUG_ON(!PageLocked(page));
1060        BUG_ON(mapping != page_mapping(page));
1061
1062        xa_lock_irqsave(&mapping->i_pages, flags);
1063        /*
1064         * The non racy check for a busy page.
1065         *
1066         * Must be careful with the order of the tests. When someone has
1067         * a ref to the page, it may be possible that they dirty it then
1068         * drop the reference. So if PageDirty is tested before page_count
1069         * here, then the following race may occur:
1070         *
1071         * get_user_pages(&page);
1072         * [user mapping goes away]
1073         * write_to(page);
1074         *                              !PageDirty(page)    [good]
1075         * SetPageDirty(page);
1076         * put_page(page);
1077         *                              !page_count(page)   [good, discard it]
1078         *
1079         * [oops, our write_to data is lost]
1080         *
1081         * Reversing the order of the tests ensures such a situation cannot
1082         * escape unnoticed. The smp_rmb is needed to ensure the page->flags
1083         * load is not satisfied before that of page->_refcount.
1084         *
1085         * Note that if SetPageDirty is always performed via set_page_dirty,
1086         * and thus under the i_pages lock, then this ordering is not required.
1087         */
1088        refcount = 1 + compound_nr(page);
1089        if (!page_ref_freeze(page, refcount))
1090                goto cannot_free;
1091        /* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
1092        if (unlikely(PageDirty(page))) {
1093                page_ref_unfreeze(page, refcount);
1094                goto cannot_free;
1095        }
1096
1097        if (PageSwapCache(page)) {
1098                swp_entry_t swap = { .val = page_private(page) };
1099                mem_cgroup_swapout(page, swap);
1100                if (reclaimed && !mapping_exiting(mapping))
1101                        shadow = workingset_eviction(page, target_memcg);
1102                __delete_from_swap_cache(page, swap, shadow);
1103                xa_unlock_irqrestore(&mapping->i_pages, flags);
1104                put_swap_page(page, swap);
1105        } else {
1106                void (*freepage)(struct page *);
1107
1108                freepage = mapping->a_ops->freepage;
1109                /*
1110                 * Remember a shadow entry for reclaimed file cache in
1111                 * order to detect refaults, thus thrashing, later on.
1112                 *
1113                 * But don't store shadows in an address space that is
1114                 * already exiting.  This is not just an optimization,
1115                 * inode reclaim needs to empty out the radix tree or
1116                 * the nodes are lost.  Don't plant shadows behind its
1117                 * back.
1118                 *
1119                 * We also don't store shadows for DAX mappings because the
1120                 * only page cache pages found in these are zero pages
1121                 * covering holes, and because we don't want to mix DAX
1122                 * exceptional entries and shadow exceptional entries in the
1123                 * same address_space.
1124                 */
1125                if (reclaimed && page_is_file_lru(page) &&
1126                    !mapping_exiting(mapping) && !dax_mapping(mapping))
1127                        shadow = workingset_eviction(page, target_memcg);
1128                __delete_from_page_cache(page, shadow);
1129                xa_unlock_irqrestore(&mapping->i_pages, flags);
1130
1131                if (freepage != NULL)
1132                        freepage(page);
1133        }
1134
1135        return 1;
1136
1137cannot_free:
1138        xa_unlock_irqrestore(&mapping->i_pages, flags);
1139        return 0;
1140}
1141
1142/*
1143 * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
1144 * someone else has a ref on the page, abort and return 0.  If it was
1145 * successfully detached, return 1.  Assumes the caller has a single ref on
1146 * this page.
1147 */
1148int remove_mapping(struct address_space *mapping, struct page *page)
1149{
1150        if (__remove_mapping(mapping, page, false, NULL)) {
1151                /*
1152                 * Unfreezing the refcount with 1 rather than 2 effectively
1153                 * drops the pagecache ref for us without requiring another
1154                 * atomic operation.
1155                 */
1156                page_ref_unfreeze(page, 1);
1157                return 1;
1158        }
1159        return 0;
1160}
1161
1162/**
1163 * putback_lru_page - put previously isolated page onto appropriate LRU list
1164 * @page: page to be put back to appropriate lru list
1165 *
1166 * Add previously isolated @page to appropriate LRU list.
1167 * Page may still be unevictable for other reasons.
1168 *
1169 * lru_lock must not be held, interrupts must be enabled.
1170 */
1171void putback_lru_page(struct page *page)
1172{
1173        lru_cache_add(page);
1174        put_page(page);         /* drop ref from isolate */
1175}
1176
1177enum page_references {
1178        PAGEREF_RECLAIM,
1179        PAGEREF_RECLAIM_CLEAN,
1180        PAGEREF_KEEP,
1181        PAGEREF_ACTIVATE,
1182};
1183
1184static enum page_references page_check_references(struct page *page,
1185                                                  struct scan_control *sc)
1186{
1187        int referenced_ptes, referenced_page;
1188        unsigned long vm_flags;
1189
1190        referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
1191                                          &vm_flags);
1192        referenced_page = TestClearPageReferenced(page);
1193
1194        /*
1195         * Mlock lost the isolation race with us.  Let try_to_unmap()
1196         * move the page to the unevictable list.
1197         */
1198        if (vm_flags & VM_LOCKED)
1199                return PAGEREF_RECLAIM;
1200
1201        if (referenced_ptes) {
1202                /*
1203                 * All mapped pages start out with page table
1204                 * references from the instantiating fault, so we need
1205                 * to look twice if a mapped file page is used more
1206                 * than once.
1207                 *
1208                 * Mark it and spare it for another trip around the
1209                 * inactive list.  Another page table reference will
1210                 * lead to its activation.
1211                 *
1212                 * Note: the mark is set for activated pages as well
1213                 * so that recently deactivated but used pages are
1214                 * quickly recovered.
1215                 */
1216                SetPageReferenced(page);
1217
1218                if (referenced_page || referenced_ptes > 1)
1219                        return PAGEREF_ACTIVATE;
1220
1221                /*
1222                 * Activate file-backed executable pages after first usage.
1223                 */
1224                if ((vm_flags & VM_EXEC) && !PageSwapBacked(page))
1225                        return PAGEREF_ACTIVATE;
1226
1227                return PAGEREF_KEEP;
1228        }
1229
1230        /* Reclaim if clean, defer dirty pages to writeback */
1231        if (referenced_page && !PageSwapBacked(page))
1232                return PAGEREF_RECLAIM_CLEAN;
1233
1234        return PAGEREF_RECLAIM;
1235}
1236
1237/* Check if a page is dirty or under writeback */
1238static void page_check_dirty_writeback(struct page *page,
1239                                       bool *dirty, bool *writeback)
1240{
1241        struct address_space *mapping;
1242
1243        /*
1244         * Anonymous pages are not handled by flushers and must be written
1245         * from reclaim context. Do not stall reclaim based on them
1246         */
1247        if (!page_is_file_lru(page) ||
1248            (PageAnon(page) && !PageSwapBacked(page))) {
1249                *dirty = false;
1250                *writeback = false;
1251                return;
1252        }
1253
1254        /* By default assume that the page flags are accurate */
1255        *dirty = PageDirty(page);
1256        *writeback = PageWriteback(page);
1257
1258        /* Verify dirty/writeback state if the filesystem supports it */
1259        if (!page_has_private(page))
1260                return;
1261
1262        mapping = page_mapping(page);
1263        if (mapping && mapping->a_ops->is_dirty_writeback)
1264                mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
1265}
1266
1267/*
1268 * shrink_page_list() returns the number of reclaimed pages
1269 */
1270static unsigned int shrink_page_list(struct list_head *page_list,
1271                                     struct pglist_data *pgdat,
1272                                     struct scan_control *sc,
1273                                     struct reclaim_stat *stat,
1274                                     bool ignore_references)
1275{
1276        LIST_HEAD(ret_pages);
1277        LIST_HEAD(free_pages);
1278        unsigned int nr_reclaimed = 0;
1279        unsigned int pgactivate = 0;
1280
1281        memset(stat, 0, sizeof(*stat));
1282        cond_resched();
1283
1284        while (!list_empty(page_list)) {
1285                struct address_space *mapping;
1286                struct page *page;
1287                enum page_references references = PAGEREF_RECLAIM;
1288                bool dirty, writeback, may_enter_fs;
1289                unsigned int nr_pages;
1290
1291                cond_resched();
1292
1293                page = lru_to_page(page_list);
1294                list_del(&page->lru);
1295
1296                if (!trylock_page(page))
1297                        goto keep;
1298
1299                VM_BUG_ON_PAGE(PageActive(page), page);
1300
1301                nr_pages = compound_nr(page);
1302
1303                /* Account the number of base pages even though THP */
1304                sc->nr_scanned += nr_pages;
1305
1306                if (unlikely(!page_evictable(page)))
1307                        goto activate_locked;
1308
1309                if (!sc->may_unmap && page_mapped(page))
1310                        goto keep_locked;
1311
1312                may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
1313                        (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
1314
1315                /*
1316                 * The number of dirty pages determines if a node is marked
1317                 * reclaim_congested which affects wait_iff_congested. kswapd
1318                 * will stall and start writing pages if the tail of the LRU
1319                 * is all dirty unqueued pages.
1320                 */
1321                page_check_dirty_writeback(page, &dirty, &writeback);
1322                if (dirty || writeback)
1323                        stat->nr_dirty++;
1324
1325                if (dirty && !writeback)
1326                        stat->nr_unqueued_dirty++;
1327
1328                /*
1329                 * Treat this page as congested if the underlying BDI is or if
1330                 * pages are cycling through the LRU so quickly that the
1331                 * pages marked for immediate reclaim are making it to the
1332                 * end of the LRU a second time.
1333                 */
1334                mapping = page_mapping(page);
1335                if (((dirty || writeback) && mapping &&
1336                     inode_write_congested(mapping->host)) ||
1337                    (writeback && PageReclaim(page)))
1338                        stat->nr_congested++;
1339
1340                /*
1341                 * If a page at the tail of the LRU is under writeback, there
1342                 * are three cases to consider.
1343                 *
1344                 * 1) If reclaim is encountering an excessive number of pages
1345                 *    under writeback and this page is both under writeback and
1346                 *    PageReclaim then it indicates that pages are being queued
1347                 *    for IO but are being recycled through the LRU before the
1348                 *    IO can complete. Waiting on the page itself risks an
1349                 *    indefinite stall if it is impossible to writeback the
1350                 *    page due to IO error or disconnected storage so instead
1351                 *    note that the LRU is being scanned too quickly and the
1352                 *    caller can stall after page list has been processed.
1353                 *
1354                 * 2) Global or new memcg reclaim encounters a page that is
1355                 *    not marked for immediate reclaim, or the caller does not
1356                 *    have __GFP_FS (or __GFP_IO if it's simply going to swap,
1357                 *    not to fs). In this case mark the page for immediate
1358                 *    reclaim and continue scanning.
1359                 *
1360                 *    Require may_enter_fs because we would wait on fs, which
1361                 *    may not have submitted IO yet. And the loop driver might
1362                 *    enter reclaim, and deadlock if it waits on a page for
1363                 *    which it is needed to do the write (loop masks off
1364                 *    __GFP_IO|__GFP_FS for this reason); but more thought
1365                 *    would probably show more reasons.
1366                 *
1367                 * 3) Legacy memcg encounters a page that is already marked
1368                 *    PageReclaim. memcg does not have any dirty pages
1369                 *    throttling so we could easily OOM just because too many
1370                 *    pages are in writeback and there is nothing else to
1371                 *    reclaim. Wait for the writeback to complete.
1372                 *
1373                 * In cases 1) and 2) we activate the pages to get them out of
1374                 * the way while we continue scanning for clean pages on the
1375                 * inactive list and refilling from the active list. The
1376                 * observation here is that waiting for disk writes is more
1377                 * expensive than potentially causing reloads down the line.
1378                 * Since they're marked for immediate reclaim, they won't put
1379                 * memory pressure on the cache working set any longer than it
1380                 * takes to write them to disk.
1381                 */
1382                if (PageWriteback(page)) {
1383                        /* Case 1 above */
1384                        if (current_is_kswapd() &&
1385                            PageReclaim(page) &&
1386                            test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1387                                stat->nr_immediate++;
1388                                goto activate_locked;
1389
1390                        /* Case 2 above */
1391                        } else if (writeback_throttling_sane(sc) ||
1392                            !PageReclaim(page) || !may_enter_fs) {
1393                                /*
1394                                 * This is slightly racy - end_page_writeback()
1395                                 * might have just cleared PageReclaim, then
1396                                 * setting PageReclaim here end up interpreted
1397                                 * as PageReadahead - but that does not matter
1398                                 * enough to care.  What we do want is for this
1399                                 * page to have PageReclaim set next time memcg
1400                                 * reclaim reaches the tests above, so it will
1401                                 * then wait_on_page_writeback() to avoid OOM;
1402                                 * and it's also appropriate in global reclaim.
1403                                 */
1404                                SetPageReclaim(page);
1405                                stat->nr_writeback++;
1406                                goto activate_locked;
1407
1408                        /* Case 3 above */
1409                        } else {
1410                                unlock_page(page);
1411                                wait_on_page_writeback(page);
1412                                /* then go back and try same page again */
1413                                list_add_tail(&page->lru, page_list);
1414                                continue;
1415                        }
1416                }
1417
1418                if (!ignore_references)
1419                        references = page_check_references(page, sc);
1420
1421                switch (references) {
1422                case PAGEREF_ACTIVATE:
1423                        goto activate_locked;
1424                case PAGEREF_KEEP:
1425                        stat->nr_ref_keep += nr_pages;
1426                        goto keep_locked;
1427                case PAGEREF_RECLAIM:
1428                case PAGEREF_RECLAIM_CLEAN:
1429                        ; /* try to reclaim the page below */
1430                }
1431
1432                /*
1433                 * Anonymous process memory has backing store?
1434                 * Try to allocate it some swap space here.
1435                 * Lazyfree page could be freed directly
1436                 */
1437                if (PageAnon(page) && PageSwapBacked(page)) {
1438                        if (!PageSwapCache(page)) {
1439                                if (!(sc->gfp_mask & __GFP_IO))
1440                                        goto keep_locked;
1441                                if (page_maybe_dma_pinned(page))
1442                                        goto keep_locked;
1443                                if (PageTransHuge(page)) {
1444                                        /* cannot split THP, skip it */
1445                                        if (!can_split_huge_page(page, NULL))
1446                                                goto activate_locked;
1447                                        /*
1448                                         * Split pages without a PMD map right
1449                                         * away. Chances are some or all of the
1450                                         * tail pages can be freed without IO.
1451                                         */
1452                                        if (!compound_mapcount(page) &&
1453                                            split_huge_page_to_list(page,
1454                                                                    page_list))
1455                                                goto activate_locked;
1456                                }
1457                                if (!add_to_swap(page)) {
1458                                        if (!PageTransHuge(page))
1459                                                goto activate_locked_split;
1460                                        /* Fallback to swap normal pages */
1461                                        if (split_huge_page_to_list(page,
1462                                                                    page_list))
1463                                                goto activate_locked;
1464#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1465                                        count_vm_event(THP_SWPOUT_FALLBACK);
1466#endif
1467                                        if (!add_to_swap(page))
1468                                                goto activate_locked_split;
1469                                }
1470
1471                                may_enter_fs = true;
1472
1473                                /* Adding to swap updated mapping */
1474                                mapping = page_mapping(page);
1475                        }
1476                } else if (unlikely(PageTransHuge(page))) {
1477                        /* Split file THP */
1478                        if (split_huge_page_to_list(page, page_list))
1479                                goto keep_locked;
1480                }
1481
1482                /*
1483                 * THP may get split above, need minus tail pages and update
1484                 * nr_pages to avoid accounting tail pages twice.
1485                 *
1486                 * The tail pages that are added into swap cache successfully
1487                 * reach here.
1488                 */
1489                if ((nr_pages > 1) && !PageTransHuge(page)) {
1490                        sc->nr_scanned -= (nr_pages - 1);
1491                        nr_pages = 1;
1492                }
1493
1494                /*
1495                 * The page is mapped into the page tables of one or more
1496                 * processes. Try to unmap it here.
1497                 */
1498                if (page_mapped(page)) {
1499                        enum ttu_flags flags = TTU_BATCH_FLUSH;
1500                        bool was_swapbacked = PageSwapBacked(page);
1501
1502                        if (unlikely(PageTransHuge(page)))
1503                                flags |= TTU_SPLIT_HUGE_PMD;
1504
1505                        try_to_unmap(page, flags);
1506                        if (page_mapped(page)) {
1507                                stat->nr_unmap_fail += nr_pages;
1508                                if (!was_swapbacked && PageSwapBacked(page))
1509                                        stat->nr_lazyfree_fail += nr_pages;
1510                                goto activate_locked;
1511                        }
1512                }
1513
1514                if (PageDirty(page)) {
1515                        /*
1516                         * Only kswapd can writeback filesystem pages
1517                         * to avoid risk of stack overflow. But avoid
1518                         * injecting inefficient single-page IO into
1519                         * flusher writeback as much as possible: only
1520                         * write pages when we've encountered many
1521                         * dirty pages, and when we've already scanned
1522                         * the rest of the LRU for clean pages and see
1523                         * the same dirty pages again (PageReclaim).
1524                         */
1525                        if (page_is_file_lru(page) &&
1526                            (!current_is_kswapd() || !PageReclaim(page) ||
1527                             !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1528                                /*
1529                                 * Immediately reclaim when written back.
1530                                 * Similar in principal to deactivate_page()
1531                                 * except we already have the page isolated
1532                                 * and know it's dirty
1533                                 */
1534                                inc_node_page_state(page, NR_VMSCAN_IMMEDIATE);
1535                                SetPageReclaim(page);
1536
1537                                goto activate_locked;
1538                        }
1539
1540                        if (references == PAGEREF_RECLAIM_CLEAN)
1541                                goto keep_locked;
1542                        if (!may_enter_fs)
1543                                goto keep_locked;
1544                        if (!sc->may_writepage)
1545                                goto keep_locked;
1546
1547                        /*
1548                         * Page is dirty. Flush the TLB if a writable entry
1549                         * potentially exists to avoid CPU writes after IO
1550                         * starts and then write it out here.
1551                         */
1552                        try_to_unmap_flush_dirty();
1553                        switch (pageout(page, mapping)) {
1554                        case PAGE_KEEP:
1555                                goto keep_locked;
1556                        case PAGE_ACTIVATE:
1557                                goto activate_locked;
1558                        case PAGE_SUCCESS:
1559                                stat->nr_pageout += thp_nr_pages(page);
1560
1561                                if (PageWriteback(page))
1562                                        goto keep;
1563                                if (PageDirty(page))
1564                                        goto keep;
1565
1566                                /*
1567                                 * A synchronous write - probably a ramdisk.  Go
1568                                 * ahead and try to reclaim the page.
1569                                 */
1570                                if (!trylock_page(page))
1571                                        goto keep;
1572                                if (PageDirty(page) || PageWriteback(page))
1573                                        goto keep_locked;
1574                                mapping = page_mapping(page);
1575                                fallthrough;
1576                        case PAGE_CLEAN:
1577                                ; /* try to free the page below */
1578                        }
1579                }
1580
1581                /*
1582                 * If the page has buffers, try to free the buffer mappings
1583                 * associated with this page. If we succeed we try to free
1584                 * the page as well.
1585                 *
1586                 * We do this even if the page is PageDirty().
1587                 * try_to_release_page() does not perform I/O, but it is
1588                 * possible for a page to have PageDirty set, but it is actually
1589                 * clean (all its buffers are clean).  This happens if the
1590                 * buffers were written out directly, with submit_bh(). ext3
1591                 * will do this, as well as the blockdev mapping.
1592                 * try_to_release_page() will discover that cleanness and will
1593                 * drop the buffers and mark the page clean - it can be freed.
1594                 *
1595                 * Rarely, pages can have buffers and no ->mapping.  These are
1596                 * the pages which were not successfully invalidated in
1597                 * truncate_cleanup_page().  We try to drop those buffers here
1598                 * and if that worked, and the page is no longer mapped into
1599                 * process address space (page_count == 1) it can be freed.
1600                 * Otherwise, leave the page on the LRU so it is swappable.
1601                 */
1602                if (page_has_private(page)) {
1603                        if (!try_to_release_page(page, sc->gfp_mask))
1604                                goto activate_locked;
1605                        if (!mapping && page_count(page) == 1) {
1606                                unlock_page(page);
1607                                if (put_page_testzero(page))
1608                                        goto free_it;
1609                                else {
1610                                        /*
1611                                         * rare race with speculative reference.
1612                                         * the speculative reference will free
1613                                         * this page shortly, so we may
1614                                         * increment nr_reclaimed here (and
1615                                         * leave it off the LRU).
1616                                         */
1617                                        nr_reclaimed++;
1618                                        continue;
1619                                }
1620                        }
1621                }
1622
1623                if (PageAnon(page) && !PageSwapBacked(page)) {
1624                        /* follow __remove_mapping for reference */
1625                        if (!page_ref_freeze(page, 1))
1626                                goto keep_locked;
1627                        if (PageDirty(page)) {
1628                                page_ref_unfreeze(page, 1);
1629                                goto keep_locked;
1630                        }
1631
1632                        count_vm_event(PGLAZYFREED);
1633                        count_memcg_page_event(page, PGLAZYFREED);
1634                } else if (!mapping || !__remove_mapping(mapping, page, true,
1635                                                         sc->target_mem_cgroup))
1636                        goto keep_locked;
1637
1638                unlock_page(page);
1639free_it:
1640                /*
1641                 * THP may get swapped out in a whole, need account
1642                 * all base pages.
1643                 */
1644                nr_reclaimed += nr_pages;
1645
1646                /*
1647                 * Is there need to periodically free_page_list? It would
1648                 * appear not as the counts should be low
1649                 */
1650                if (unlikely(PageTransHuge(page)))
1651                        destroy_compound_page(page);
1652                else
1653                        list_add(&page->lru, &free_pages);
1654                continue;
1655
1656activate_locked_split:
1657                /*
1658                 * The tail pages that are failed to add into swap cache
1659                 * reach here.  Fixup nr_scanned and nr_pages.
1660                 */
1661                if (nr_pages > 1) {
1662                        sc->nr_scanned -= (nr_pages - 1);
1663                        nr_pages = 1;
1664                }
1665activate_locked:
1666                /* Not a candidate for swapping, so reclaim swap space. */
1667                if (PageSwapCache(page) && (mem_cgroup_swap_full(page) ||
1668                                                PageMlocked(page)))
1669                        try_to_free_swap(page);
1670                VM_BUG_ON_PAGE(PageActive(page), page);
1671                if (!PageMlocked(page)) {
1672                        int type = page_is_file_lru(page);
1673                        SetPageActive(page);
1674                        stat->nr_activate[type] += nr_pages;
1675                        count_memcg_page_event(page, PGACTIVATE);
1676                }
1677keep_locked:
1678                unlock_page(page);
1679keep:
1680                list_add(&page->lru, &ret_pages);
1681                VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
1682        }
1683
1684        pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
1685
1686        mem_cgroup_uncharge_list(&free_pages);
1687        try_to_unmap_flush();
1688        free_unref_page_list(&free_pages);
1689
1690        list_splice(&ret_pages, page_list);
1691        count_vm_events(PGACTIVATE, pgactivate);
1692
1693        return nr_reclaimed;
1694}
1695
1696unsigned int reclaim_clean_pages_from_list(struct zone *zone,
1697                                            struct list_head *page_list)
1698{
1699        struct scan_control sc = {
1700                .gfp_mask = GFP_KERNEL,
1701                .priority = DEF_PRIORITY,
1702                .may_unmap = 1,
1703        };
1704        struct reclaim_stat stat;
1705        unsigned int nr_reclaimed;
1706        struct page *page, *next;
1707        LIST_HEAD(clean_pages);
1708        unsigned int noreclaim_flag;
1709
1710        list_for_each_entry_safe(page, next, page_list, lru) {
1711                if (!PageHuge(page) && page_is_file_lru(page) &&
1712                    !PageDirty(page) && !__PageMovable(page) &&
1713                    !PageUnevictable(page)) {
1714                        ClearPageActive(page);
1715                        list_move(&page->lru, &clean_pages);
1716                }
1717        }
1718
1719        /*
1720         * We should be safe here since we are only dealing with file pages and
1721         * we are not kswapd and therefore cannot write dirty file pages. But
1722         * call memalloc_noreclaim_save() anyway, just in case these conditions
1723         * change in the future.
1724         */
1725        noreclaim_flag = memalloc_noreclaim_save();
1726        nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
1727                                        &stat, true);
1728        memalloc_noreclaim_restore(noreclaim_flag);
1729
1730        list_splice(&clean_pages, page_list);
1731        mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
1732                            -(long)nr_reclaimed);
1733        /*
1734         * Since lazyfree pages are isolated from file LRU from the beginning,
1735         * they will rotate back to anonymous LRU in the end if it failed to
1736         * discard so isolated count will be mismatched.
1737         * Compensate the isolated count for both LRU lists.
1738         */
1739        mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON,
1740                            stat.nr_lazyfree_fail);
1741        mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
1742                            -(long)stat.nr_lazyfree_fail);
1743        return nr_reclaimed;
1744}
1745
1746/*
1747 * Attempt to remove the specified page from its LRU.  Only take this page
1748 * if it is of the appropriate PageActive status.  Pages which are being
1749 * freed elsewhere are also ignored.
1750 *
1751 * page:        page to consider
1752 * mode:        one of the LRU isolation modes defined above
1753 *
1754 * returns true on success, false on failure.
1755 */
1756bool __isolate_lru_page_prepare(struct page *page, isolate_mode_t mode)
1757{
1758        /* Only take pages on the LRU. */
1759        if (!PageLRU(page))
1760                return false;
1761
1762        /* Compaction should not handle unevictable pages but CMA can do so */
1763        if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
1764                return false;
1765
1766        /*
1767         * To minimise LRU disruption, the caller can indicate that it only
1768         * wants to isolate pages it will be able to operate on without
1769         * blocking - clean pages for the most part.
1770         *
1771         * ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
1772         * that it is possible to migrate without blocking
1773         */
1774        if (mode & ISOLATE_ASYNC_MIGRATE) {
1775                /* All the caller can do on PageWriteback is block */
1776                if (PageWriteback(page))
1777                        return false;
1778
1779                if (PageDirty(page)) {
1780                        struct address_space *mapping;
1781                        bool migrate_dirty;
1782
1783                        /*
1784                         * Only pages without mappings or that have a
1785                         * ->migratepage callback are possible to migrate
1786                         * without blocking. However, we can be racing with
1787                         * truncation so it's necessary to lock the page
1788                         * to stabilise the mapping as truncation holds
1789                         * the page lock until after the page is removed
1790                         * from the page cache.
1791                         */
1792                        if (!trylock_page(page))
1793                                return false;
1794
1795                        mapping = page_mapping(page);
1796                        migrate_dirty = !mapping || mapping->a_ops->migratepage;
1797                        unlock_page(page);
1798                        if (!migrate_dirty)
1799                                return false;
1800                }
1801        }
1802
1803        if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
1804                return false;
1805
1806        return true;
1807}
1808
1809/*
1810 * Update LRU sizes after isolating pages. The LRU size updates must
1811 * be complete before mem_cgroup_update_lru_size due to a sanity check.
1812 */
1813static __always_inline void update_lru_sizes(struct lruvec *lruvec,
1814                        enum lru_list lru, unsigned long *nr_zone_taken)
1815{
1816        int zid;
1817
1818        for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1819                if (!nr_zone_taken[zid])
1820                        continue;
1821
1822                update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1823        }
1824
1825}
1826
1827/*
1828 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
1829 *
1830 * lruvec->lru_lock is heavily contended.  Some of the functions that
1831 * shrink the lists perform better by taking out a batch of pages
1832 * and working on them outside the LRU lock.
1833 *
1834 * For pagecache intensive workloads, this function is the hottest
1835 * spot in the kernel (apart from copy_*_user functions).
1836 *
1837 * Lru_lock must be held before calling this function.
1838 *
1839 * @nr_to_scan: The number of eligible pages to look through on the list.
1840 * @lruvec:     The LRU vector to pull pages from.
1841 * @dst:        The temp list to put pages on to.
1842 * @nr_scanned: The number of pages that were scanned.
1843 * @sc:         The scan_control struct for this reclaim session
1844 * @lru:        LRU list id for isolating
1845 *
1846 * returns how many pages were moved onto *@dst.
1847 */
1848static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1849                struct lruvec *lruvec, struct list_head *dst,
1850                unsigned long *nr_scanned, struct scan_control *sc,
1851                enum lru_list lru)
1852{
1853        struct list_head *src = &lruvec->lists[lru];
1854        unsigned long nr_taken = 0;
1855        unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
1856        unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
1857        unsigned long skipped = 0;
1858        unsigned long scan, total_scan, nr_pages;
1859        LIST_HEAD(pages_skipped);
1860        isolate_mode_t mode = (sc->may_unmap ? 0 : ISOLATE_UNMAPPED);
1861
1862        total_scan = 0;
1863        scan = 0;
1864        while (scan < nr_to_scan && !list_empty(src)) {
1865                struct page *page;
1866
1867                page = lru_to_page(src);
1868                prefetchw_prev_lru_page(page, src, flags);
1869
1870                nr_pages = compound_nr(page);
1871                total_scan += nr_pages;
1872
1873                if (page_zonenum(page) > sc->reclaim_idx) {
1874                        list_move(&page->lru, &pages_skipped);
1875                        nr_skipped[page_zonenum(page)] += nr_pages;
1876                        continue;
1877                }
1878
1879                /*
1880                 * Do not count skipped pages because that makes the function
1881                 * return with no isolated pages if the LRU mostly contains
1882                 * ineligible pages.  This causes the VM to not reclaim any
1883                 * pages, triggering a premature OOM.
1884                 *
1885                 * Account all tail pages of THP.  This would not cause
1886                 * premature OOM since __isolate_lru_page() returns -EBUSY
1887                 * only when the page is being freed somewhere else.
1888                 */
1889                scan += nr_pages;
1890                if (!__isolate_lru_page_prepare(page, mode)) {
1891                        /* It is being freed elsewhere */
1892                        list_move(&page->lru, src);
1893                        continue;
1894                }
1895                /*
1896                 * Be careful not to clear PageLRU until after we're
1897                 * sure the page is not being freed elsewhere -- the
1898                 * page release code relies on it.
1899                 */
1900                if (unlikely(!get_page_unless_zero(page))) {
1901                        list_move(&page->lru, src);
1902                        continue;
1903                }
1904
1905                if (!TestClearPageLRU(page)) {
1906                        /* Another thread is already isolating this page */
1907                        put_page(page);
1908                        list_move(&page->lru, src);
1909                        continue;
1910                }
1911
1912                nr_taken += nr_pages;
1913                nr_zone_taken[page_zonenum(page)] += nr_pages;
1914                list_move(&page->lru, dst);
1915        }
1916
1917        /*
1918         * Splice any skipped pages to the start of the LRU list. Note that
1919         * this disrupts the LRU order when reclaiming for lower zones but
1920         * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
1921         * scanning would soon rescan the same pages to skip and put the
1922         * system at risk of premature OOM.
1923         */
1924        if (!list_empty(&pages_skipped)) {
1925                int zid;
1926
1927                list_splice(&pages_skipped, src);
1928                for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1929                        if (!nr_skipped[zid])
1930                                continue;
1931
1932                        __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
1933                        skipped += nr_skipped[zid];
1934                }
1935        }
1936        *nr_scanned = total_scan;
1937        trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
1938                                    total_scan, skipped, nr_taken, mode, lru);
1939        update_lru_sizes(lruvec, lru, nr_zone_taken);
1940        return nr_taken;
1941}
1942
1943/**
1944 * isolate_lru_page - tries to isolate a page from its LRU list
1945 * @page: page to isolate from its LRU list
1946 *
1947 * Isolates a @page from an LRU list, clears PageLRU and adjusts the
1948 * vmstat statistic corresponding to whatever LRU list the page was on.
1949 *
1950 * Returns 0 if the page was removed from an LRU list.
1951 * Returns -EBUSY if the page was not on an LRU list.
1952 *
1953 * The returned page will have PageLRU() cleared.  If it was found on
1954 * the active list, it will have PageActive set.  If it was found on
1955 * the unevictable list, it will have the PageUnevictable bit set. That flag
1956 * may need to be cleared by the caller before letting the page go.
1957 *
1958 * The vmstat statistic corresponding to the list on which the page was
1959 * found will be decremented.
1960 *
1961 * Restrictions:
1962 *
1963 * (1) Must be called with an elevated refcount on the page. This is a
1964 *     fundamental difference from isolate_lru_pages (which is called
1965 *     without a stable reference).
1966 * (2) the lru_lock must not be held.
1967 * (3) interrupts must be enabled.
1968 */
1969int isolate_lru_page(struct page *page)
1970{
1971        int ret = -EBUSY;
1972
1973        VM_BUG_ON_PAGE(!page_count(page), page);
1974        WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
1975
1976        if (TestClearPageLRU(page)) {
1977                struct lruvec *lruvec;
1978
1979                get_page(page);
1980                lruvec = lock_page_lruvec_irq(page);
1981                del_page_from_lru_list(page, lruvec);
1982                unlock_page_lruvec_irq(lruvec);
1983                ret = 0;
1984        }
1985
1986        return ret;
1987}
1988
1989/*
1990 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1991 * then get rescheduled. When there are massive number of tasks doing page
1992 * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1993 * the LRU list will go small and be scanned faster than necessary, leading to
1994 * unnecessary swapping, thrashing and OOM.
1995 */
1996static int too_many_isolated(struct pglist_data *pgdat, int file,
1997                struct scan_control *sc)
1998{
1999        unsigned long inactive, isolated;
2000
2001        if (current_is_kswapd())
2002                return 0;
2003
2004        if (!writeback_throttling_sane(sc))
2005                return 0;
2006
2007        if (file) {
2008                inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
2009                isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
2010        } else {
2011                inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
2012                isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
2013        }
2014
2015        /*
2016         * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
2017         * won't get blocked by normal direct-reclaimers, forming a circular
2018         * deadlock.
2019         */
2020        if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
2021                inactive >>= 3;
2022
2023        return isolated > inactive;
2024}
2025
2026/*
2027 * move_pages_to_lru() moves pages from private @list to appropriate LRU list.
2028 * On return, @list is reused as a list of pages to be freed by the caller.
2029 *
2030 * Returns the number of pages moved to the given lruvec.
2031 */
2032static unsigned int move_pages_to_lru(struct lruvec *lruvec,
2033                                      struct list_head *list)
2034{
2035        int nr_pages, nr_moved = 0;
2036        LIST_HEAD(pages_to_free);
2037        struct page *page;
2038
2039        while (!list_empty(list)) {
2040                page = lru_to_page(list);
2041                VM_BUG_ON_PAGE(PageLRU(page), page);
2042                list_del(&page->lru);
2043                if (unlikely(!page_evictable(page))) {
2044                        spin_unlock_irq(&lruvec->lru_lock);
2045                        putback_lru_page(page);
2046                        spin_lock_irq(&lruvec->lru_lock);
2047                        continue;
2048                }
2049
2050                /*
2051                 * The SetPageLRU needs to be kept here for list integrity.
2052                 * Otherwise:
2053                 *   #0 move_pages_to_lru             #1 release_pages
2054                 *   if !put_page_testzero
2055                 *                                    if (put_page_testzero())
2056                 *                                      !PageLRU //skip lru_lock
2057                 *     SetPageLRU()
2058                 *     list_add(&page->lru,)
2059                 *                                        list_add(&page->lru,)
2060                 */
2061                SetPageLRU(page);
2062
2063                if (unlikely(put_page_testzero(page))) {
2064                        __clear_page_lru_flags(page);
2065
2066                        if (unlikely(PageCompound(page))) {
2067                                spin_unlock_irq(&lruvec->lru_lock);
2068                                destroy_compound_page(page);
2069                                spin_lock_irq(&lruvec->lru_lock);
2070                        } else
2071                                list_add(&page->lru, &pages_to_free);
2072
2073                        continue;
2074                }
2075
2076                /*
2077                 * All pages were isolated from the same lruvec (and isolation
2078                 * inhibits memcg migration).
2079                 */
2080                VM_BUG_ON_PAGE(!page_matches_lruvec(page, lruvec), page);
2081                add_page_to_lru_list(page, lruvec);
2082                nr_pages = thp_nr_pages(page);
2083                nr_moved += nr_pages;
2084                if (PageActive(page))
2085                        workingset_age_nonresident(lruvec, nr_pages);
2086        }
2087
2088        /*
2089         * To save our caller's stack, now use input list for pages to free.
2090         */
2091        list_splice(&pages_to_free, list);
2092
2093        return nr_moved;
2094}
2095
2096/*
2097 * If a kernel thread (such as nfsd for loop-back mounts) services
2098 * a backing device by writing to the page cache it sets PF_LOCAL_THROTTLE.
2099 * In that case we should only throttle if the backing device it is
2100 * writing to is congested.  In other cases it is safe to throttle.
2101 */
2102static int current_may_throttle(void)
2103{
2104        return !(current->flags & PF_LOCAL_THROTTLE) ||
2105                current->backing_dev_info == NULL ||
2106                bdi_write_congested(current->backing_dev_info);
2107}
2108
2109/*
2110 * shrink_inactive_list() is a helper for shrink_node().  It returns the number
2111 * of reclaimed pages
2112 */
2113static unsigned long
2114shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
2115                     struct scan_control *sc, enum lru_list lru)
2116{
2117        LIST_HEAD(page_list);
2118        unsigned long nr_scanned;
2119        unsigned int nr_reclaimed = 0;
2120        unsigned long nr_taken;
2121        struct reclaim_stat stat;
2122        bool file = is_file_lru(lru);
2123        enum vm_event_item item;
2124        struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2125        bool stalled = false;
2126
2127        while (unlikely(too_many_isolated(pgdat, file, sc))) {
2128                if (stalled)
2129                        return 0;
2130
2131                /* wait a bit for the reclaimer. */
2132                msleep(100);
2133                stalled = true;
2134
2135                /* We are about to die and free our memory. Return now. */
2136                if (fatal_signal_pending(current))
2137                        return SWAP_CLUSTER_MAX;
2138        }
2139
2140        lru_add_drain();
2141
2142        spin_lock_irq(&lruvec->lru_lock);
2143
2144        nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
2145                                     &nr_scanned, sc, lru);
2146
2147        __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2148        item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
2149        if (!cgroup_reclaim(sc))
2150                __count_vm_events(item, nr_scanned);
2151        __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
2152        __count_vm_events(PGSCAN_ANON + file, nr_scanned);
2153
2154        spin_unlock_irq(&lruvec->lru_lock);
2155
2156        if (nr_taken == 0)
2157                return 0;
2158
2159        nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false);
2160
2161        spin_lock_irq(&lruvec->lru_lock);
2162        move_pages_to_lru(lruvec, &page_list);
2163
2164        __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2165        item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
2166        if (!cgroup_reclaim(sc))
2167                __count_vm_events(item, nr_reclaimed);
2168        __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
2169        __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed);
2170        spin_unlock_irq(&lruvec->lru_lock);
2171
2172        lru_note_cost(lruvec, file, stat.nr_pageout);
2173        mem_cgroup_uncharge_list(&page_list);
2174        free_unref_page_list(&page_list);
2175
2176        /*
2177         * If dirty pages are scanned that are not queued for IO, it
2178         * implies that flushers are not doing their job. This can
2179         * happen when memory pressure pushes dirty pages to the end of
2180         * the LRU before the dirty limits are breached and the dirty
2181         * data has expired. It can also happen when the proportion of
2182         * dirty pages grows not through writes but through memory
2183         * pressure reclaiming all the clean cache. And in some cases,
2184         * the flushers simply cannot keep up with the allocation
2185         * rate. Nudge the flusher threads in case they are asleep.
2186         */
2187        if (stat.nr_unqueued_dirty == nr_taken)
2188                wakeup_flusher_threads(WB_REASON_VMSCAN);
2189
2190        sc->nr.dirty += stat.nr_dirty;
2191        sc->nr.congested += stat.nr_congested;
2192        sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
2193        sc->nr.writeback += stat.nr_writeback;
2194        sc->nr.immediate += stat.nr_immediate;
2195        sc->nr.taken += nr_taken;
2196        if (file)
2197                sc->nr.file_taken += nr_taken;
2198
2199        trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
2200                        nr_scanned, nr_reclaimed, &stat, sc->priority, file);
2201        return nr_reclaimed;
2202}
2203
2204/*
2205 * shrink_active_list() moves pages from the active LRU to the inactive LRU.
2206 *
2207 * We move them the other way if the page is referenced by one or more
2208 * processes.
2209 *
2210 * If the pages are mostly unmapped, the processing is fast and it is
2211 * appropriate to hold lru_lock across the whole operation.  But if
2212 * the pages are mapped, the processing is slow (page_referenced()), so
2213 * we should drop lru_lock around each page.  It's impossible to balance
2214 * this, so instead we remove the pages from the LRU while processing them.
2215 * It is safe to rely on PG_active against the non-LRU pages in here because
2216 * nobody will play with that bit on a non-LRU page.
2217 *
2218 * The downside is that we have to touch page->_refcount against each page.
2219 * But we had to alter page->flags anyway.
2220 */
2221static void shrink_active_list(unsigned long nr_to_scan,
2222                               struct lruvec *lruvec,
2223                               struct scan_control *sc,
2224                               enum lru_list lru)
2225{
2226        unsigned long nr_taken;
2227        unsigned long nr_scanned;
2228        unsigned long vm_flags;
2229        LIST_HEAD(l_hold);      /* The pages which were snipped off */
2230        LIST_HEAD(l_active);
2231        LIST_HEAD(l_inactive);
2232        struct page *page;
2233        unsigned nr_deactivate, nr_activate;
2234        unsigned nr_rotated = 0;
2235        int file = is_file_lru(lru);
2236        struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2237
2238        lru_add_drain();
2239
2240        spin_lock_irq(&lruvec->lru_lock);
2241
2242        nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
2243                                     &nr_scanned, sc, lru);
2244
2245        __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2246
2247        if (!cgroup_reclaim(sc))
2248                __count_vm_events(PGREFILL, nr_scanned);
2249        __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2250
2251        spin_unlock_irq(&lruvec->lru_lock);
2252
2253        while (!list_empty(&l_hold)) {
2254                cond_resched();
2255                page = lru_to_page(&l_hold);
2256                list_del(&page->lru);
2257
2258                if (unlikely(!page_evictable(page))) {
2259                        putback_lru_page(page);
2260                        continue;
2261                }
2262
2263                if (unlikely(buffer_heads_over_limit)) {
2264                        if (page_has_private(page) && trylock_page(page)) {
2265                                if (page_has_private(page))
2266                                        try_to_release_page(page, 0);
2267                                unlock_page(page);
2268                        }
2269                }
2270
2271                if (page_referenced(page, 0, sc->target_mem_cgroup,
2272                                    &vm_flags)) {
2273                        /*
2274                         * Identify referenced, file-backed active pages and
2275                         * give them one more trip around the active list. So
2276                         * that executable code get better chances to stay in
2277                         * memory under moderate memory pressure.  Anon pages
2278                         * are not likely to be evicted by use-once streaming
2279                         * IO, plus JVM can create lots of anon VM_EXEC pages,
2280                         * so we ignore them here.
2281                         */
2282                        if ((vm_flags & VM_EXEC) && page_is_file_lru(page)) {
2283                                nr_rotated += thp_nr_pages(page);
2284                                list_add(&page->lru, &l_active);
2285                                continue;
2286                        }
2287                }
2288
2289                ClearPageActive(page);  /* we are de-activating */
2290                SetPageWorkingset(page);
2291                list_add(&page->lru, &l_inactive);
2292        }
2293
2294        /*
2295         * Move pages back to the lru list.
2296         */
2297        spin_lock_irq(&lruvec->lru_lock);
2298
2299        nr_activate = move_pages_to_lru(lruvec, &l_active);
2300        nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);
2301        /* Keep all free pages in l_active list */
2302        list_splice(&l_inactive, &l_active);
2303
2304        __count_vm_events(PGDEACTIVATE, nr_deactivate);
2305        __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
2306
2307        __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2308        spin_unlock_irq(&lruvec->lru_lock);
2309
2310        mem_cgroup_uncharge_list(&l_active);
2311        free_unref_page_list(&l_active);
2312        trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2313                        nr_deactivate, nr_rotated, sc->priority, file);
2314}
2315
2316unsigned long reclaim_pages(struct list_head *page_list)
2317{
2318        int nid = NUMA_NO_NODE;
2319        unsigned int nr_reclaimed = 0;
2320        LIST_HEAD(node_page_list);
2321        struct reclaim_stat dummy_stat;
2322        struct page *page;
2323        unsigned int noreclaim_flag;
2324        struct scan_control sc = {
2325                .gfp_mask = GFP_KERNEL,
2326                .priority = DEF_PRIORITY,
2327                .may_writepage = 1,
2328                .may_unmap = 1,
2329                .may_swap = 1,
2330        };
2331
2332        noreclaim_flag = memalloc_noreclaim_save();
2333
2334        while (!list_empty(page_list)) {
2335                page = lru_to_page(page_list);
2336                if (nid == NUMA_NO_NODE) {
2337                        nid = page_to_nid(page);
2338                        INIT_LIST_HEAD(&node_page_list);
2339                }
2340
2341                if (nid == page_to_nid(page)) {
2342                        ClearPageActive(page);
2343                        list_move(&page->lru, &node_page_list);
2344                        continue;
2345                }
2346
2347                nr_reclaimed += shrink_page_list(&node_page_list,
2348                                                NODE_DATA(nid),
2349                                                &sc, &dummy_stat, false);
2350                while (!list_empty(&node_page_list)) {
2351                        page = lru_to_page(&node_page_list);
2352                        list_del(&page->lru);
2353                        putback_lru_page(page);
2354                }
2355
2356                nid = NUMA_NO_NODE;
2357        }
2358
2359        if (!list_empty(&node_page_list)) {
2360                nr_reclaimed += shrink_page_list(&node_page_list,
2361                                                NODE_DATA(nid),
2362                                                &sc, &dummy_stat, false);
2363                while (!list_empty(&node_page_list)) {
2364                        page = lru_to_page(&node_page_list);
2365                        list_del(&page->lru);
2366                        putback_lru_page(page);
2367                }
2368        }
2369
2370        memalloc_noreclaim_restore(noreclaim_flag);
2371
2372        return nr_reclaimed;
2373}
2374
2375static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2376                                 struct lruvec *lruvec, struct scan_control *sc)
2377{
2378        if (is_active_lru(lru)) {
2379                if (sc->may_deactivate & (1 << is_file_lru(lru)))
2380                        shrink_active_list(nr_to_scan, lruvec, sc, lru);
2381                else
2382                        sc->skipped_deactivate = 1;
2383                return 0;
2384        }
2385
2386        return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2387}
2388
2389/*
2390 * The inactive anon list should be small enough that the VM never has
2391 * to do too much work.
2392 *
2393 * The inactive file list should be small enough to leave most memory
2394 * to the established workingset on the scan-resistant active list,
2395 * but large enough to avoid thrashing the aggregate readahead window.
2396 *
2397 * Both inactive lists should also be large enough that each inactive
2398 * page has a chance to be referenced again before it is reclaimed.
2399 *
2400 * If that fails and refaulting is observed, the inactive list grows.
2401 *
2402 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
2403 * on this LRU, maintained by the pageout code. An inactive_ratio
2404 * of 3 means 3:1 or 25% of the pages are kept on the inactive list.
2405 *
2406 * total     target    max
2407 * memory    ratio     inactive
2408 * -------------------------------------
2409 *   10MB       1         5MB
2410 *  100MB       1        50MB
2411 *    1GB       3       250MB
2412 *   10GB      10       0.9GB
2413 *  100GB      31         3GB
2414 *    1TB     101        10GB
2415 *   10TB     320        32GB
2416 */
2417static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2418{
2419        enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2420        unsigned long inactive, active;
2421        unsigned long inactive_ratio;
2422        unsigned long gb;
2423
2424        inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
2425        active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2426
2427        gb = (inactive + active) >> (30 - PAGE_SHIFT);
2428        if (gb)
2429                inactive_ratio = int_sqrt(10 * gb);
2430        else
2431                inactive_ratio = 1;
2432
2433        return inactive * inactive_ratio < active;
2434}
2435
2436enum scan_balance {
2437        SCAN_EQUAL,
2438        SCAN_FRACT,
2439        SCAN_ANON,
2440        SCAN_FILE,
2441};
2442
2443/*
2444 * Determine how aggressively the anon and file LRU lists should be
2445 * scanned.  The relative value of each set of LRU lists is determined
2446 * by looking at the fraction of the pages scanned we did rotate back
2447 * onto the active list instead of evict.
2448 *
2449 * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2450 * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2451 */
2452static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
2453                           unsigned long *nr)
2454{
2455        struct mem_cgroup *memcg = lruvec_memcg(lruvec);
2456        unsigned long anon_cost, file_cost, total_cost;
2457        int swappiness = mem_cgroup_swappiness(memcg);
2458        u64 fraction[ANON_AND_FILE];
2459        u64 denominator = 0;    /* gcc */
2460        enum scan_balance scan_balance;
2461        unsigned long ap, fp;
2462        enum lru_list lru;
2463
2464        /* If we have no swap space, do not bother scanning anon pages. */
2465        if (!sc->may_swap || mem_cgroup_get_nr_swap_pages(memcg) <= 0) {
2466                scan_balance = SCAN_FILE;
2467                goto out;
2468        }
2469
2470        /*
2471         * Global reclaim will swap to prevent OOM even with no
2472         * swappiness, but memcg users want to use this knob to
2473         * disable swapping for individual groups completely when
2474         * using the memory controller's swap limit feature would be
2475         * too expensive.
2476         */
2477        if (cgroup_reclaim(sc) && !swappiness) {
2478                scan_balance = SCAN_FILE;
2479                goto out;
2480        }
2481
2482        /*
2483         * Do not apply any pressure balancing cleverness when the
2484         * system is close to OOM, scan both anon and file equally
2485         * (unless the swappiness setting disagrees with swapping).
2486         */
2487        if (!sc->priority && swappiness) {
2488                scan_balance = SCAN_EQUAL;
2489                goto out;
2490        }
2491
2492        /*
2493         * If the system is almost out of file pages, force-scan anon.
2494         */
2495        if (sc->file_is_tiny) {
2496                scan_balance = SCAN_ANON;
2497                goto out;
2498        }
2499
2500        /*
2501         * If there is enough inactive page cache, we do not reclaim
2502         * anything from the anonymous working right now.
2503         */
2504        if (sc->cache_trim_mode) {
2505                scan_balance = SCAN_FILE;
2506                goto out;
2507        }
2508
2509        scan_balance = SCAN_FRACT;
2510        /*
2511         * Calculate the pressure balance between anon and file pages.
2512         *
2513         * The amount of pressure we put on each LRU is inversely
2514         * proportional to the cost of reclaiming each list, as
2515         * determined by the share of pages that are refaulting, times
2516         * the relative IO cost of bringing back a swapped out
2517         * anonymous page vs reloading a filesystem page (swappiness).
2518         *
2519         * Although we limit that influence to ensure no list gets
2520         * left behind completely: at least a third of the pressure is
2521         * applied, before swappiness.
2522         *
2523         * With swappiness at 100, anon and file have equal IO cost.
2524         */
2525        total_cost = sc->anon_cost + sc->file_cost;
2526        anon_cost = total_cost + sc->anon_cost;
2527        file_cost = total_cost + sc->file_cost;
2528        total_cost = anon_cost + file_cost;
2529
2530        ap = swappiness * (total_cost + 1);
2531        ap /= anon_cost + 1;
2532
2533        fp = (200 - swappiness) * (total_cost + 1);
2534        fp /= file_cost + 1;
2535
2536        fraction[0] = ap;
2537        fraction[1] = fp;
2538        denominator = ap + fp;
2539out:
2540        for_each_evictable_lru(lru) {
2541                int file = is_file_lru(lru);
2542                unsigned long lruvec_size;
2543                unsigned long low, min;
2544                unsigned long scan;
2545
2546                lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2547                mem_cgroup_protection(sc->target_mem_cgroup, memcg,
2548                                      &min, &low);
2549
2550                if (min || low) {
2551                        /*
2552                         * Scale a cgroup's reclaim pressure by proportioning
2553                         * its current usage to its memory.low or memory.min
2554                         * setting.
2555                         *
2556                         * This is important, as otherwise scanning aggression
2557                         * becomes extremely binary -- from nothing as we
2558                         * approach the memory protection threshold, to totally
2559                         * nominal as we exceed it.  This results in requiring
2560                         * setting extremely liberal protection thresholds. It
2561                         * also means we simply get no protection at all if we
2562                         * set it too low, which is not ideal.
2563                         *
2564                         * If there is any protection in place, we reduce scan
2565                         * pressure by how much of the total memory used is
2566                         * within protection thresholds.
2567                         *
2568                         * There is one special case: in the first reclaim pass,
2569                         * we skip over all groups that are within their low
2570                         * protection. If that fails to reclaim enough pages to
2571                         * satisfy the reclaim goal, we come back and override
2572                         * the best-effort low protection. However, we still
2573                         * ideally want to honor how well-behaved groups are in
2574                         * that case instead of simply punishing them all
2575                         * equally. As such, we reclaim them based on how much
2576                         * memory they are using, reducing the scan pressure
2577                         * again by how much of the total memory used is under
2578                         * hard protection.
2579                         */
2580                        unsigned long cgroup_size = mem_cgroup_size(memcg);
2581                        unsigned long protection;
2582
2583                        /* memory.low scaling, make sure we retry before OOM */
2584                        if (!sc->memcg_low_reclaim && low > min) {
2585                                protection = low;
2586                                sc->memcg_low_skipped = 1;
2587                        } else {
2588                                protection = min;
2589                        }
2590
2591                        /* Avoid TOCTOU with earlier protection check */
2592                        cgroup_size = max(cgroup_size, protection);
2593
2594                        scan = lruvec_size - lruvec_size * protection /
2595                                cgroup_size;
2596
2597                        /*
2598                         * Minimally target SWAP_CLUSTER_MAX pages to keep
2599                         * reclaim moving forwards, avoiding decrementing
2600                         * sc->priority further than desirable.
2601                         */
2602                        scan = max(scan, SWAP_CLUSTER_MAX);
2603                } else {
2604                        scan = lruvec_size;
2605                }
2606
2607                scan >>= sc->priority;
2608
2609                /*
2610                 * If the cgroup's already been deleted, make sure to
2611                 * scrape out the remaining cache.
2612                 */
2613                if (!scan && !mem_cgroup_online(memcg))
2614                        scan = min(lruvec_size, SWAP_CLUSTER_MAX);
2615
2616                switch (scan_balance) {
2617                case SCAN_EQUAL:
2618                        /* Scan lists relative to size */
2619                        break;
2620                case SCAN_FRACT:
2621                        /*
2622                         * Scan types proportional to swappiness and
2623                         * their relative recent reclaim efficiency.
2624                         * Make sure we don't miss the last page on
2625                         * the offlined memory cgroups because of a
2626                         * round-off error.
2627                         */
2628                        scan = mem_cgroup_online(memcg) ?
2629                               div64_u64(scan * fraction[file], denominator) :
2630                               DIV64_U64_ROUND_UP(scan * fraction[file],
2631                                                  denominator);
2632                        break;
2633                case SCAN_FILE:
2634                case SCAN_ANON:
2635                        /* Scan one type exclusively */
2636                        if ((scan_balance == SCAN_FILE) != file)
2637                                scan = 0;
2638                        break;
2639                default:
2640                        /* Look ma, no brain */
2641                        BUG();
2642                }
2643
2644                nr[lru] = scan;
2645        }
2646}
2647
2648static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
2649{
2650        unsigned long nr[NR_LRU_LISTS];
2651        unsigned long targets[NR_LRU_LISTS];
2652        unsigned long nr_to_scan;
2653        enum lru_list lru;
2654        unsigned long nr_reclaimed = 0;
2655        unsigned long nr_to_reclaim = sc->nr_to_reclaim;
2656        struct blk_plug plug;
2657        bool scan_adjusted;
2658
2659        get_scan_count(lruvec, sc, nr);
2660
2661        /* Record the original scan target for proportional adjustments later */
2662        memcpy(targets, nr, sizeof(nr));
2663
2664        /*
2665         * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
2666         * event that can occur when there is little memory pressure e.g.
2667         * multiple streaming readers/writers. Hence, we do not abort scanning
2668         * when the requested number of pages are reclaimed when scanning at
2669         * DEF_PRIORITY on the assumption that the fact we are direct
2670         * reclaiming implies that kswapd is not keeping up and it is best to
2671         * do a batch of work at once. For memcg reclaim one check is made to
2672         * abort proportional reclaim if either the file or anon lru has already
2673         * dropped to zero at the first pass.
2674         */
2675        scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
2676                         sc->priority == DEF_PRIORITY);
2677
2678        blk_start_plug(&plug);
2679        while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
2680                                        nr[LRU_INACTIVE_FILE]) {
2681                unsigned long nr_anon, nr_file, percentage;
2682                unsigned long nr_scanned;
2683
2684                for_each_evictable_lru(lru) {
2685                        if (nr[lru]) {
2686                                nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
2687                                nr[lru] -= nr_to_scan;
2688
2689                                nr_reclaimed += shrink_list(lru, nr_to_scan,
2690                                                            lruvec, sc);
2691                        }
2692                }
2693
2694                cond_resched();
2695
2696                if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
2697                        continue;
2698
2699                /*
2700                 * For kswapd and memcg, reclaim at least the number of pages
2701                 * requested. Ensure that the anon and file LRUs are scanned
2702                 * proportionally what was requested by get_scan_count(). We
2703                 * stop reclaiming one LRU and reduce the amount scanning
2704                 * proportional to the original scan target.
2705                 */
2706                nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
2707                nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
2708
2709                /*
2710                 * It's just vindictive to attack the larger once the smaller
2711                 * has gone to zero.  And given the way we stop scanning the
2712                 * smaller below, this makes sure that we only make one nudge
2713                 * towards proportionality once we've got nr_to_reclaim.
2714                 */
2715                if (!nr_file || !nr_anon)
2716                        break;
2717
2718                if (nr_file > nr_anon) {
2719                        unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
2720                                                targets[LRU_ACTIVE_ANON] + 1;
2721                        lru = LRU_BASE;
2722                        percentage = nr_anon * 100 / scan_target;
2723                } else {
2724                        unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
2725                                                targets[LRU_ACTIVE_FILE] + 1;
2726                        lru = LRU_FILE;
2727                        percentage = nr_file * 100 / scan_target;
2728                }
2729
2730                /* Stop scanning the smaller of the LRU */
2731                nr[lru] = 0;
2732                nr[lru + LRU_ACTIVE] = 0;
2733
2734                /*
2735                 * Recalculate the other LRU scan count based on its original
2736                 * scan target and the percentage scanning already complete
2737                 */
2738                lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
2739                nr_scanned = targets[lru] - nr[lru];
2740                nr[lru] = targets[lru] * (100 - percentage) / 100;
2741                nr[lru] -= min(nr[lru], nr_scanned);
2742
2743                lru += LRU_ACTIVE;
2744                nr_scanned = targets[lru] - nr[lru];
2745                nr[lru] = targets[lru] * (100 - percentage) / 100;
2746                nr[lru] -= min(nr[lru], nr_scanned);
2747
2748                scan_adjusted = true;
2749        }
2750        blk_finish_plug(&plug);
2751        sc->nr_reclaimed += nr_reclaimed;
2752
2753        /*
2754         * Even if we did not try to evict anon pages at all, we want to
2755         * rebalance the anon lru active/inactive ratio.
2756         */
2757        if (total_swap_pages && inactive_is_low(lruvec, LRU_INACTIVE_ANON))
2758                shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2759                                   sc, LRU_ACTIVE_ANON);
2760}
2761
2762/* Use reclaim/compaction for costly allocs or under memory pressure */
2763static bool in_reclaim_compaction(struct scan_control *sc)
2764{
2765        if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
2766                        (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
2767                         sc->priority < DEF_PRIORITY - 2))
2768                return true;
2769
2770        return false;
2771}
2772
2773/*
2774 * Reclaim/compaction is used for high-order allocation requests. It reclaims
2775 * order-0 pages before compacting the zone. should_continue_reclaim() returns
2776 * true if more pages should be reclaimed such that when the page allocator
2777 * calls try_to_compact_pages() that it will have enough free pages to succeed.
2778 * It will give up earlier than that if there is difficulty reclaiming pages.
2779 */
2780static inline bool should_continue_reclaim(struct pglist_data *pgdat,
2781                                        unsigned long nr_reclaimed,
2782                                        struct scan_control *sc)
2783{
2784        unsigned long pages_for_compaction;
2785        unsigned long inactive_lru_pages;
2786        int z;
2787
2788        /* If not in reclaim/compaction mode, stop */
2789        if (!in_reclaim_compaction(sc))
2790                return false;
2791
2792        /*
2793         * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
2794         * number of pages that were scanned. This will return to the caller
2795         * with the risk reclaim/compaction and the resulting allocation attempt
2796         * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
2797         * allocations through requiring that the full LRU list has been scanned
2798         * first, by assuming that zero delta of sc->nr_scanned means full LRU
2799         * scan, but that approximation was wrong, and there were corner cases
2800         * where always a non-zero amount of pages were scanned.
2801         */
2802        if (!nr_reclaimed)
2803                return false;
2804
2805        /* If compaction would go ahead or the allocation would succeed, stop */
2806        for (z = 0; z <= sc->reclaim_idx; z++) {
2807                struct zone *zone = &pgdat->node_zones[z];
2808                if (!managed_zone(zone))
2809                        continue;
2810
2811                switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
2812                case COMPACT_SUCCESS:
2813                case COMPACT_CONTINUE:
2814                        return false;
2815                default:
2816                        /* check next zone */
2817                        ;
2818                }
2819        }
2820
2821        /*
2822         * If we have not reclaimed enough pages for compaction and the
2823         * inactive lists are large enough, continue reclaiming
2824         */
2825        pages_for_compaction = compact_gap(sc->order);
2826        inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
2827        if (get_nr_swap_pages() > 0)
2828                inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
2829
2830        return inactive_lru_pages > pages_for_compaction;
2831}
2832
2833static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
2834{
2835        struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
2836        struct mem_cgroup *memcg;
2837
2838        memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
2839        do {
2840                struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
2841                unsigned long reclaimed;
2842                unsigned long scanned;
2843
2844                /*
2845                 * This loop can become CPU-bound when target memcgs
2846                 * aren't eligible for reclaim - either because they
2847                 * don't have any reclaimable pages, or because their
2848                 * memory is explicitly protected. Avoid soft lockups.
2849                 */
2850                cond_resched();
2851
2852                mem_cgroup_calculate_protection(target_memcg, memcg);
2853
2854                if (mem_cgroup_below_min(memcg)) {
2855                        /*
2856                         * Hard protection.
2857                         * If there is no reclaimable memory, OOM.
2858                         */
2859                        continue;
2860                } else if (mem_cgroup_below_low(memcg)) {
2861                        /*
2862                         * Soft protection.
2863                         * Respect the protection only as long as
2864                         * there is an unprotected supply
2865                         * of reclaimable memory from other cgroups.
2866                         */
2867                        if (!sc->memcg_low_reclaim) {
2868                                sc->memcg_low_skipped = 1;
2869                                continue;
2870                        }
2871                        memcg_memory_event(memcg, MEMCG_LOW);
2872                }
2873
2874                reclaimed = sc->nr_reclaimed;
2875                scanned = sc->nr_scanned;
2876
2877                shrink_lruvec(lruvec, sc);
2878
2879                shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
2880                            sc->priority);
2881
2882                /* Record the group's reclaim efficiency */
2883                vmpressure(sc->gfp_mask, memcg, false,
2884                           sc->nr_scanned - scanned,
2885                           sc->nr_reclaimed - reclaimed);
2886
2887        } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
2888}
2889
2890static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
2891{
2892        struct reclaim_state *reclaim_state = current->reclaim_state;
2893        unsigned long nr_reclaimed, nr_scanned;
2894        struct lruvec *target_lruvec;
2895        bool reclaimable = false;
2896        unsigned long file;
2897
2898        target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
2899
2900again:
2901        memset(&sc->nr, 0, sizeof(sc->nr));
2902
2903        nr_reclaimed = sc->nr_reclaimed;
2904        nr_scanned = sc->nr_scanned;
2905
2906        /*
2907         * Determine the scan balance between anon and file LRUs.
2908         */
2909        spin_lock_irq(&target_lruvec->lru_lock);
2910        sc->anon_cost = target_lruvec->anon_cost;
2911        sc->file_cost = target_lruvec->file_cost;
2912        spin_unlock_irq(&target_lruvec->lru_lock);
2913
2914        /*
2915         * Target desirable inactive:active list ratios for the anon
2916         * and file LRU lists.
2917         */
2918        if (!sc->force_deactivate) {
2919                unsigned long refaults;
2920
2921                refaults = lruvec_page_state(target_lruvec,
2922                                WORKINGSET_ACTIVATE_ANON);
2923                if (refaults != target_lruvec->refaults[0] ||
2924                        inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
2925                        sc->may_deactivate |= DEACTIVATE_ANON;
2926                else
2927                        sc->may_deactivate &= ~DEACTIVATE_ANON;
2928
2929                /*
2930                 * When refaults are being observed, it means a new
2931                 * workingset is being established. Deactivate to get
2932                 * rid of any stale active pages quickly.
2933                 */
2934                refaults = lruvec_page_state(target_lruvec,
2935                                WORKINGSET_ACTIVATE_FILE);
2936                if (refaults != target_lruvec->refaults[1] ||
2937                    inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
2938                        sc->may_deactivate |= DEACTIVATE_FILE;
2939                else
2940                        sc->may_deactivate &= ~DEACTIVATE_FILE;
2941        } else
2942                sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
2943
2944        /*
2945         * If we have plenty of inactive file pages that aren't
2946         * thrashing, try to reclaim those first before touching
2947         * anonymous pages.
2948         */
2949        file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
2950        if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
2951                sc->cache_trim_mode = 1;
2952        else
2953                sc->cache_trim_mode = 0;
2954
2955        /*
2956         * Prevent the reclaimer from falling into the cache trap: as
2957         * cache pages start out inactive, every cache fault will tip
2958         * the scan balance towards the file LRU.  And as the file LRU
2959         * shrinks, so does the window for rotation from references.
2960         * This means we have a runaway feedback loop where a tiny
2961         * thrashing file LRU becomes infinitely more attractive than
2962         * anon pages.  Try to detect this based on file LRU size.
2963         */
2964        if (!cgroup_reclaim(sc)) {
2965                unsigned long total_high_wmark = 0;
2966                unsigned long free, anon;
2967                int z;
2968
2969                free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2970                file = node_page_state(pgdat, NR_ACTIVE_FILE) +
2971                           node_page_state(pgdat, NR_INACTIVE_FILE);
2972
2973                for (z = 0; z < MAX_NR_ZONES; z++) {
2974                        struct zone *zone = &pgdat->node_zones[z];
2975                        if (!managed_zone(zone))
2976                                continue;
2977
2978                        total_high_wmark += high_wmark_pages(zone);
2979                }
2980
2981                /*
2982                 * Consider anon: if that's low too, this isn't a
2983                 * runaway file reclaim problem, but rather just
2984                 * extreme pressure. Reclaim as per usual then.
2985                 */
2986                anon = node_page_state(pgdat, NR_INACTIVE_ANON);
2987
2988                sc->file_is_tiny =
2989                        file + free <= total_high_wmark &&
2990                        !(sc->may_deactivate & DEACTIVATE_ANON) &&
2991                        anon >> sc->priority;
2992        }
2993
2994        shrink_node_memcgs(pgdat, sc);
2995
2996        if (reclaim_state) {
2997                sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2998                reclaim_state->reclaimed_slab = 0;
2999        }
3000
3001        /* Record the subtree's reclaim efficiency */
3002        vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
3003                   sc->nr_scanned - nr_scanned,
3004                   sc->nr_reclaimed - nr_reclaimed);
3005
3006        if (sc->nr_reclaimed - nr_reclaimed)
3007                reclaimable = true;
3008
3009        if (current_is_kswapd()) {
3010                /*
3011                 * If reclaim is isolating dirty pages under writeback,
3012                 * it implies that the long-lived page allocation rate
3013                 * is exceeding the page laundering rate. Either the
3014                 * global limits are not being effective at throttling
3015                 * processes due to the page distribution throughout
3016                 * zones or there is heavy usage of a slow backing
3017                 * device. The only option is to throttle from reclaim
3018                 * context which is not ideal as there is no guarantee
3019                 * the dirtying process is throttled in the same way
3020                 * balance_dirty_pages() manages.
3021                 *
3022                 * Once a node is flagged PGDAT_WRITEBACK, kswapd will
3023                 * count the number of pages under pages flagged for
3024                 * immediate reclaim and stall if any are encountered
3025                 * in the nr_immediate check below.
3026                 */
3027                if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
3028                        set_bit(PGDAT_WRITEBACK, &pgdat->flags);
3029
3030                /* Allow kswapd to start writing pages during reclaim.*/
3031                if (sc->nr.unqueued_dirty == sc->nr.file_taken)
3032                        set_bit(PGDAT_DIRTY, &pgdat->flags);
3033
3034                /*
3035                 * If kswapd scans pages marked for immediate
3036                 * reclaim and under writeback (nr_immediate), it
3037                 * implies that pages are cycling through the LRU
3038                 * faster than they are written so also forcibly stall.
3039                 */
3040                if (sc->nr.immediate)
3041                        congestion_wait(BLK_RW_ASYNC, HZ/10);
3042        }
3043
3044        /*
3045         * Tag a node/memcg as congested if all the dirty pages
3046         * scanned were backed by a congested BDI and
3047         * wait_iff_congested will stall.
3048         *
3049         * Legacy memcg will stall in page writeback so avoid forcibly
3050         * stalling in wait_iff_congested().
3051         */
3052        if ((current_is_kswapd() ||
3053             (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
3054            sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
3055                set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
3056
3057        /*
3058         * Stall direct reclaim for IO completions if underlying BDIs
3059         * and node is congested. Allow kswapd to continue until it
3060         * starts encountering unqueued dirty pages or cycling through
3061         * the LRU too quickly.
3062         */
3063        if (!current_is_kswapd() && current_may_throttle() &&
3064            !sc->hibernation_mode &&
3065            test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
3066                wait_iff_congested(BLK_RW_ASYNC, HZ/10);
3067
3068        if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
3069                                    sc))
3070                goto again;
3071
3072        /*
3073         * Kswapd gives up on balancing particular nodes after too
3074         * many failures to reclaim anything from them and goes to
3075         * sleep. On reclaim progress, reset the failure counter. A
3076         * successful direct reclaim run will revive a dormant kswapd.
3077         */
3078        if (reclaimable)
3079                pgdat->kswapd_failures = 0;
3080}
3081
3082/*
3083 * Returns true if compaction should go ahead for a costly-order request, or
3084 * the allocation would already succeed without compaction. Return false if we
3085 * should reclaim first.
3086 */
3087static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
3088{
3089        unsigned long watermark;
3090        enum compact_result suitable;
3091
3092        suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx);
3093        if (suitable == COMPACT_SUCCESS)
3094                /* Allocation should succeed already. Don't reclaim. */
3095                return true;
3096        if (suitable == COMPACT_SKIPPED)
3097                /* Compaction cannot yet proceed. Do reclaim. */
3098                return false;
3099
3100        /*
3101         * Compaction is already possible, but it takes time to run and there
3102         * are potentially other callers using the pages just freed. So proceed
3103         * with reclaim to make a buffer of free pages available to give
3104         * compaction a reasonable chance of completing and allocating the page.
3105         * Note that we won't actually reclaim the whole buffer in one attempt
3106         * as the target watermark in should_continue_reclaim() is lower. But if
3107         * we are already above the high+gap watermark, don't reclaim at all.
3108         */
3109        watermark = high_wmark_pages(zone) + compact_gap(sc->order);
3110
3111        return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
3112}
3113
3114/*
3115 * This is the direct reclaim path, for page-allocating processes.  We only
3116 * try to reclaim pages from zones which will satisfy the caller's allocation
3117 * request.
3118 *
3119 * If a zone is deemed to be full of pinned pages then just give it a light
3120 * scan then give up on it.
3121 */
3122static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
3123{
3124        struct zoneref *z;
3125        struct zone *zone;
3126        unsigned long nr_soft_reclaimed;
3127        unsigned long nr_soft_scanned;
3128        gfp_t orig_mask;
3129        pg_data_t *last_pgdat = NULL;
3130
3131        /*
3132         * If the number of buffer_heads in the machine exceeds the maximum
3133         * allowed level, force direct reclaim to scan the highmem zone as
3134         * highmem pages could be pinning lowmem pages storing buffer_heads
3135         */
3136        orig_mask = sc->gfp_mask;
3137        if (buffer_heads_over_limit) {
3138                sc->gfp_mask |= __GFP_HIGHMEM;
3139                sc->reclaim_idx = gfp_zone(sc->gfp_mask);
3140        }
3141
3142        for_each_zone_zonelist_nodemask(zone, z, zonelist,
3143                                        sc->reclaim_idx, sc->nodemask) {
3144                /*
3145                 * Take care memory controller reclaiming has small influence
3146                 * to global LRU.
3147                 */
3148                if (!cgroup_reclaim(sc)) {
3149                        if (!cpuset_zone_allowed(zone,
3150                                                 GFP_KERNEL | __GFP_HARDWALL))
3151                                continue;
3152
3153                        /*
3154                         * If we already have plenty of memory free for
3155                         * compaction in this zone, don't free any more.
3156                         * Even though compaction is invoked for any
3157                         * non-zero order, only frequent costly order
3158                         * reclamation is disruptive enough to become a
3159                         * noticeable problem, like transparent huge
3160                         * page allocations.
3161                         */
3162                        if (IS_ENABLED(CONFIG_COMPACTION) &&
3163                            sc->order > PAGE_ALLOC_COSTLY_ORDER &&
3164                            compaction_ready(zone, sc)) {
3165                                sc->compaction_ready = true;
3166                                continue;
3167                        }
3168
3169                        /*
3170                         * Shrink each node in the zonelist once. If the
3171                         * zonelist is ordered by zone (not the default) then a
3172                         * node may be shrunk multiple times but in that case
3173                         * the user prefers lower zones being preserved.
3174                         */
3175                        if (zone->zone_pgdat == last_pgdat)
3176                                continue;
3177
3178                        /*
3179                         * This steals pages from memory cgroups over softlimit
3180                         * and returns the number of reclaimed pages and
3181                         * scanned pages. This works for global memory pressure
3182                         * and balancing, not for a memcg's limit.
3183                         */
3184                        nr_soft_scanned = 0;
3185                        nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
3186                                                sc->order, sc->gfp_mask,
3187                                                &nr_soft_scanned);
3188                        sc->nr_reclaimed += nr_soft_reclaimed;
3189                        sc->nr_scanned += nr_soft_scanned;
3190                        /* need some check for avoid more shrink_zone() */
3191                }
3192
3193                /* See comment about same check for global reclaim above */
3194                if (zone->zone_pgdat == last_pgdat)
3195                        continue;
3196                last_pgdat = zone->zone_pgdat;
3197                shrink_node(zone->zone_pgdat, sc);
3198        }
3199
3200        /*
3201         * Restore to original mask to avoid the impact on the caller if we
3202         * promoted it to __GFP_HIGHMEM.
3203         */
3204        sc->gfp_mask = orig_mask;
3205}
3206
3207static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)
3208{
3209        struct lruvec *target_lruvec;
3210        unsigned long refaults;
3211
3212        target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
3213        refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
3214        target_lruvec->refaults[0] = refaults;
3215        refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
3216        target_lruvec->refaults[1] = refaults;
3217}
3218
3219/*
3220 * This is the main entry point to direct page reclaim.
3221 *
3222 * If a full scan of the inactive list fails to free enough memory then we
3223 * are "out of memory" and something needs to be killed.
3224 *
3225 * If the caller is !__GFP_FS then the probability of a failure is reasonably
3226 * high - the zone may be full of dirty or under-writeback pages, which this
3227 * caller can't do much about.  We kick the writeback threads and take explicit
3228 * naps in the hope that some of these pages can be written.  But if the
3229 * allocating task holds filesystem locks which prevent writeout this might not
3230 * work, and the allocation attempt will fail.
3231 *
3232 * returns:     0, if no pages reclaimed
3233 *              else, the number of pages reclaimed
3234 */
3235static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
3236                                          struct scan_control *sc)
3237{
3238        int initial_priority = sc->priority;
3239        pg_data_t *last_pgdat;
3240        struct zoneref *z;
3241        struct zone *zone;
3242retry:
3243        delayacct_freepages_start();
3244
3245        if (!cgroup_reclaim(sc))
3246                __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
3247
3248        do {
3249                vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
3250                                sc->priority);
3251                sc->nr_scanned = 0;
3252                shrink_zones(zonelist, sc);
3253
3254                if (sc->nr_reclaimed >= sc->nr_to_reclaim)
3255                        break;
3256
3257                if (sc->compaction_ready)
3258                        break;
3259
3260                /*
3261                 * If we're getting trouble reclaiming, start doing
3262                 * writepage even in laptop mode.
3263                 */
3264                if (sc->priority < DEF_PRIORITY - 2)
3265                        sc->may_writepage = 1;
3266        } while (--sc->priority >= 0);
3267
3268        last_pgdat = NULL;
3269        for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx,
3270                                        sc->nodemask) {
3271                if (zone->zone_pgdat == last_pgdat)
3272                        continue;
3273                last_pgdat = zone->zone_pgdat;
3274
3275                snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3276
3277                if (cgroup_reclaim(sc)) {
3278                        struct lruvec *lruvec;
3279
3280                        lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
3281                                                   zone->zone_pgdat);
3282                        clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3283                }
3284        }
3285
3286        delayacct_freepages_end();
3287
3288        if (sc->nr_reclaimed)
3289                return sc->nr_reclaimed;
3290
3291        /* Aborted reclaim to try compaction? don't OOM, then */
3292        if (sc->compaction_ready)
3293                return 1;
3294
3295        /*
3296         * We make inactive:active ratio decisions based on the node's
3297         * composition of memory, but a restrictive reclaim_idx or a
3298         * memory.low cgroup setting can exempt large amounts of
3299         * memory from reclaim. Neither of which are very common, so
3300         * instead of doing costly eligibility calculations of the
3301         * entire cgroup subtree up front, we assume the estimates are
3302         * good, and retry with forcible deactivation if that fails.
3303         */
3304        if (sc->skipped_deactivate) {
3305                sc->priority = initial_priority;
3306                sc->force_deactivate = 1;
3307                sc->skipped_deactivate = 0;
3308                goto retry;
3309        }
3310
3311        /* Untapped cgroup reserves?  Don't OOM, retry. */
3312        if (sc->memcg_low_skipped) {
3313                sc->priority = initial_priority;
3314                sc->force_deactivate = 0;
3315                sc->memcg_low_reclaim = 1;
3316                sc->memcg_low_skipped = 0;
3317                goto retry;
3318        }
3319
3320        return 0;
3321}
3322
3323static bool allow_direct_reclaim(pg_data_t *pgdat)
3324{
3325        struct zone *zone;
3326        unsigned long pfmemalloc_reserve = 0;
3327        unsigned long free_pages = 0;
3328        int i;
3329        bool wmark_ok;
3330
3331        if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3332                return true;
3333
3334        for (i = 0; i <= ZONE_NORMAL; i++) {
3335                zone = &pgdat->node_zones[i];
3336                if (!managed_zone(zone))
3337                        continue;
3338
3339                if (!zone_reclaimable_pages(zone))
3340                        continue;
3341
3342                pfmemalloc_reserve += min_wmark_pages(zone);
3343                free_pages += zone_page_state(zone, NR_FREE_PAGES);
3344        }
3345
3346        /* If there are no reserves (unexpected config) then do not throttle */
3347        if (!pfmemalloc_reserve)
3348                return true;
3349
3350        wmark_ok = free_pages > pfmemalloc_reserve / 2;
3351
3352        /* kswapd must be awake if processes are being throttled */
3353        if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
3354                if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
3355                        WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);
3356
3357                wake_up_interruptible(&pgdat->kswapd_wait);
3358        }
3359
3360        return wmark_ok;
3361}
3362
3363/*
3364 * Throttle direct reclaimers if backing storage is backed by the network
3365 * and the PFMEMALLOC reserve for the preferred node is getting dangerously
3366 * depleted. kswapd will continue to make progress and wake the processes
3367 * when the low watermark is reached.
3368 *
3369 * Returns true if a fatal signal was delivered during throttling. If this
3370 * happens, the page allocator should not consider triggering the OOM killer.
3371 */
3372static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
3373                                        nodemask_t *nodemask)
3374{
3375        struct zoneref *z;
3376        struct zone *zone;
3377        pg_data_t *pgdat = NULL;
3378
3379        /*
3380         * Kernel threads should not be throttled as they may be indirectly
3381         * responsible for cleaning pages necessary for reclaim to make forward
3382         * progress. kjournald for example may enter direct reclaim while
3383         * committing a transaction where throttling it could forcing other
3384         * processes to block on log_wait_commit().
3385         */
3386        if (current->flags & PF_KTHREAD)
3387                goto out;
3388
3389        /*
3390         * If a fatal signal is pending, this process should not throttle.
3391         * It should return quickly so it can exit and free its memory
3392         */
3393        if (fatal_signal_pending(current))
3394                goto out;
3395
3396        /*
3397         * Check if the pfmemalloc reserves are ok by finding the first node
3398         * with a usable ZONE_NORMAL or lower zone. The expectation is that
3399         * GFP_KERNEL will be required for allocating network buffers when
3400         * swapping over the network so ZONE_HIGHMEM is unusable.
3401         *
3402         * Throttling is based on the first usable node and throttled processes
3403         * wait on a queue until kswapd makes progress and wakes them. There
3404         * is an affinity then between processes waking up and where reclaim
3405         * progress has been made assuming the process wakes on the same node.
3406         * More importantly, processes running on remote nodes will not compete
3407         * for remote pfmemalloc reserves and processes on different nodes
3408         * should make reasonable progress.
3409         */
3410        for_each_zone_zonelist_nodemask(zone, z, zonelist,
3411                                        gfp_zone(gfp_mask), nodemask) {
3412                if (zone_idx(zone) > ZONE_NORMAL)
3413                        continue;
3414
3415                /* Throttle based on the first usable node */
3416                pgdat = zone->zone_pgdat;
3417                if (allow_direct_reclaim(pgdat))
3418                        goto out;
3419                break;
3420        }
3421
3422        /* If no zone was usable by the allocation flags then do not throttle */
3423        if (!pgdat)
3424                goto out;
3425
3426        /* Account for the throttling */
3427        count_vm_event(PGSCAN_DIRECT_THROTTLE);
3428
3429        /*
3430         * If the caller cannot enter the filesystem, it's possible that it
3431         * is due to the caller holding an FS lock or performing a journal
3432         * transaction in the case of a filesystem like ext[3|4]. In this case,
3433         * it is not safe to block on pfmemalloc_wait as kswapd could be
3434         * blocked waiting on the same lock. Instead, throttle for up to a
3435         * second before continuing.
3436         */
3437        if (!(gfp_mask & __GFP_FS)) {
3438                wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
3439                        allow_direct_reclaim(pgdat), HZ);
3440
3441                goto check_pending;
3442        }
3443
3444        /* Throttle until kswapd wakes the process */
3445        wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
3446                allow_direct_reclaim(pgdat));
3447
3448check_pending:
3449        if (fatal_signal_pending(current))
3450                return true;
3451
3452out:
3453        return false;
3454}
3455
3456unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
3457                                gfp_t gfp_mask, nodemask_t *nodemask)
3458{
3459        unsigned long nr_reclaimed;
3460        struct scan_control sc = {
3461                .nr_to_reclaim = SWAP_CLUSTER_MAX,
3462                .gfp_mask = current_gfp_context(gfp_mask),
3463                .reclaim_idx = gfp_zone(gfp_mask),
3464                .order = order,
3465                .nodemask = nodemask,
3466                .priority = DEF_PRIORITY,
3467                .may_writepage = !laptop_mode,
3468                .may_unmap = 1,
3469                .may_swap = 1,
3470        };
3471
3472        /*
3473         * scan_control uses s8 fields for order, priority, and reclaim_idx.
3474         * Confirm they are large enough for max values.
3475         */
3476        BUILD_BUG_ON(MAX_ORDER > S8_MAX);
3477        BUILD_BUG_ON(DEF_PRIORITY > S8_MAX);
3478        BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX);
3479
3480        /*
3481         * Do not enter reclaim if fatal signal was delivered while throttled.
3482         * 1 is returned so that the page allocator does not OOM kill at this
3483         * point.
3484         */
3485        if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3486                return 1;
3487
3488        set_task_reclaim_state(current, &sc.reclaim_state);
3489        trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask);
3490
3491        nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3492
3493        trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
3494        set_task_reclaim_state(current, NULL);
3495
3496        return nr_reclaimed;
3497}
3498
3499#ifdef CONFIG_MEMCG
3500
3501/* Only used by soft limit reclaim. Do not reuse for anything else. */
3502unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3503                                                gfp_t gfp_mask, bool noswap,
3504                                                pg_data_t *pgdat,
3505                                                unsigned long *nr_scanned)
3506{
3507        struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
3508        struct scan_control sc = {
3509                .nr_to_reclaim = SWAP_CLUSTER_MAX,
3510                .target_mem_cgroup = memcg,
3511                .may_writepage = !laptop_mode,
3512                .may_unmap = 1,
3513                .reclaim_idx = MAX_NR_ZONES - 1,
3514                .may_swap = !noswap,
3515        };
3516
3517        WARN_ON_ONCE(!current->reclaim_state);
3518
3519        sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
3520                        (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3521
3522        trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
3523                                                      sc.gfp_mask);
3524
3525        /*
3526         * NOTE: Although we can get the priority field, using it
3527         * here is not a good idea, since it limits the pages we can scan.
3528         * if we don't reclaim here, the shrink_node from balance_pgdat
3529         * will pick up pages from other mem cgroup's as well. We hack
3530         * the priority and make it zero.
3531         */
3532        shrink_lruvec(lruvec, &sc);
3533
3534        trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3535
3536        *nr_scanned = sc.nr_scanned;
3537
3538        return sc.nr_reclaimed;
3539}
3540
3541unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3542                                           unsigned long nr_pages,
3543                                           gfp_t gfp_mask,
3544                                           bool may_swap)
3545{
3546        unsigned long nr_reclaimed;
3547        unsigned int noreclaim_flag;
3548        struct scan_control sc = {
3549                .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3550                .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
3551                                (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
3552                .reclaim_idx = MAX_NR_ZONES - 1,
3553                .target_mem_cgroup = memcg,
3554                .priority = DEF_PRIORITY,
3555                .may_writepage = !laptop_mode,
3556                .may_unmap = 1,
3557                .may_swap = may_swap,
3558        };
3559        /*
3560         * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
3561         * equal pressure on all the nodes. This is based on the assumption that
3562         * the reclaim does not bail out early.
3563         */
3564        struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3565
3566        set_task_reclaim_state(current, &sc.reclaim_state);
3567        trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
3568        noreclaim_flag = memalloc_noreclaim_save();
3569
3570        nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3571
3572        memalloc_noreclaim_restore(noreclaim_flag);
3573        trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
3574        set_task_reclaim_state(current, NULL);
3575
3576        return nr_reclaimed;
3577}
3578#endif
3579
3580static void age_active_anon(struct pglist_data *pgdat,
3581                                struct scan_control *sc)
3582{
3583        struct mem_cgroup *memcg;
3584        struct lruvec *lruvec;
3585
3586        if (!total_swap_pages)
3587                return;
3588
3589        lruvec = mem_cgroup_lruvec(NULL, pgdat);
3590        if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
3591                return;
3592
3593        memcg = mem_cgroup_iter(NULL, NULL, NULL);
3594        do {
3595                lruvec = mem_cgroup_lruvec(memcg, pgdat);
3596                shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3597                                   sc, LRU_ACTIVE_ANON);
3598                memcg = mem_cgroup_iter(NULL, memcg, NULL);
3599        } while (memcg);
3600}
3601
3602static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
3603{
3604        int i;
3605        struct zone *zone;
3606
3607        /*
3608         * Check for watermark boosts top-down as the higher zones
3609         * are more likely to be boosted. Both watermarks and boosts
3610         * should not be checked at the same time as reclaim would
3611         * start prematurely when there is no boosting and a lower
3612         * zone is balanced.
3613         */
3614        for (i = highest_zoneidx; i >= 0; i--) {
3615                zone = pgdat->node_zones + i;
3616                if (!managed_zone(zone))
3617                        continue;
3618
3619                if (zone->watermark_boost)
3620                        return true;
3621        }
3622
3623        return false;
3624}
3625
3626/*
3627 * Returns true if there is an eligible zone balanced for the request order
3628 * and highest_zoneidx
3629 */
3630static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
3631{
3632        int i;
3633        unsigned long mark = -1;
3634        struct zone *zone;
3635
3636        /*
3637         * Check watermarks bottom-up as lower zones are more likely to
3638         * meet watermarks.
3639         */
3640        for (i = 0; i <= highest_zoneidx; i++) {
3641                zone = pgdat->node_zones + i;
3642
3643                if (!managed_zone(zone))
3644                        continue;
3645
3646                mark = high_wmark_pages(zone);
3647                if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx))
3648                        return true;
3649        }
3650
3651        /*
3652         * If a node has no populated zone within highest_zoneidx, it does not
3653         * need balancing by definition. This can happen if a zone-restricted
3654         * allocation tries to wake a remote kswapd.
3655         */
3656        if (mark == -1)
3657                return true;
3658
3659        return false;
3660}
3661
3662/* Clear pgdat state for congested, dirty or under writeback. */
3663static void clear_pgdat_congested(pg_data_t *pgdat)
3664{
3665        struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
3666
3667        clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3668        clear_bit(PGDAT_DIRTY, &pgdat->flags);
3669        clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
3670}
3671
3672/*
3673 * Prepare kswapd for sleeping. This verifies that there are no processes
3674 * waiting in throttle_direct_reclaim() and that watermarks have been met.
3675 *
3676 * Returns true if kswapd is ready to sleep
3677 */
3678static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
3679                                int highest_zoneidx)
3680{
3681        /*
3682         * The throttled processes are normally woken up in balance_pgdat() as
3683         * soon as allow_direct_reclaim() is true. But there is a potential
3684         * race between when kswapd checks the watermarks and a process gets
3685         * throttled. There is also a potential race if processes get
3686         * throttled, kswapd wakes, a large process exits thereby balancing the
3687         * zones, which causes kswapd to exit balance_pgdat() before reaching
3688         * the wake up checks. If kswapd is going to sleep, no process should
3689         * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
3690         * the wake up is premature, processes will wake kswapd and get
3691         * throttled again. The difference from wake ups in balance_pgdat() is
3692         * that here we are under prepare_to_wait().
3693         */
3694        if (waitqueue_active(&pgdat->pfmemalloc_wait))
3695                wake_up_all(&pgdat->pfmemalloc_wait);
3696
3697        /* Hopeless node, leave it to direct reclaim */
3698        if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3699                return true;
3700
3701        if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
3702                clear_pgdat_congested(pgdat);
3703                return true;
3704        }
3705
3706        return false;
3707}
3708
3709/*
3710 * kswapd shrinks a node of pages that are at or below the highest usable
3711 * zone that is currently unbalanced.
3712 *
3713 * Returns true if kswapd scanned at least the requested number of pages to
3714 * reclaim or if the lack of progress was due to pages under writeback.
3715 * This is used to determine if the scanning priority needs to be raised.
3716 */
3717static bool kswapd_shrink_node(pg_data_t *pgdat,
3718                               struct scan_control *sc)
3719{
3720        struct zone *zone;
3721        int z;
3722
3723        /* Reclaim a number of pages proportional to the number of zones */
3724        sc->nr_to_reclaim = 0;
3725        for (z = 0; z <= sc->reclaim_idx; z++) {
3726                zone = pgdat->node_zones + z;
3727                if (!managed_zone(zone))
3728                        continue;
3729
3730                sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
3731        }
3732
3733        /*
3734         * Historically care was taken to put equal pressure on all zones but
3735         * now pressure is applied based on node LRU order.
3736         */
3737        shrink_node(pgdat, sc);
3738
3739        /*
3740         * Fragmentation may mean that the system cannot be rebalanced for
3741         * high-order allocations. If twice the allocation size has been
3742         * reclaimed then recheck watermarks only at order-0 to prevent
3743         * excessive reclaim. Assume that a process requested a high-order
3744         * can direct reclaim/compact.
3745         */
3746        if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
3747                sc->order = 0;
3748
3749        return sc->nr_scanned >= sc->nr_to_reclaim;
3750}
3751
3752/* Page allocator PCP high watermark is lowered if reclaim is active. */
3753static inline void
3754update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active)
3755{
3756        int i;
3757        struct zone *zone;
3758
3759        for (i = 0; i <= highest_zoneidx; i++) {
3760                zone = pgdat->node_zones + i;
3761
3762                if (!managed_zone(zone))
3763                        continue;
3764
3765                if (active)
3766                        set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
3767                else
3768                        clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
3769        }
3770}
3771
3772static inline void
3773set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
3774{
3775        update_reclaim_active(pgdat, highest_zoneidx, true);
3776}
3777
3778static inline void
3779clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
3780{
3781        update_reclaim_active(pgdat, highest_zoneidx, false);
3782}
3783
3784/*
3785 * For kswapd, balance_pgdat() will reclaim pages across a node from zones
3786 * that are eligible for use by the caller until at least one zone is
3787 * balanced.
3788 *
3789 * Returns the order kswapd finished reclaiming at.
3790 *
3791 * kswapd scans the zones in the highmem->normal->dma direction.  It skips
3792 * zones which have free_pages > high_wmark_pages(zone), but once a zone is
3793 * found to have free_pages <= high_wmark_pages(zone), any page in that zone
3794 * or lower is eligible for reclaim until at least one usable zone is
3795 * balanced.
3796 */
3797static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
3798{
3799        int i;
3800        unsigned long nr_soft_reclaimed;
3801        unsigned long nr_soft_scanned;
3802        unsigned long pflags;
3803        unsigned long nr_boost_reclaim;
3804        unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
3805        bool boosted;
3806        struct zone *zone;
3807        struct scan_control sc = {
3808                .gfp_mask = GFP_KERNEL,
3809                .order = order,
3810                .may_unmap = 1,
3811        };
3812
3813        set_task_reclaim_state(current, &sc.reclaim_state);
3814        psi_memstall_enter(&pflags);
3815        __fs_reclaim_acquire();
3816
3817        count_vm_event(PAGEOUTRUN);
3818
3819        /*
3820         * Account for the reclaim boost. Note that the zone boost is left in
3821         * place so that parallel allocations that are near the watermark will
3822         * stall or direct reclaim until kswapd is finished.
3823         */
3824        nr_boost_reclaim = 0;
3825        for (i = 0; i <= highest_zoneidx; i++) {
3826                zone = pgdat->node_zones + i;
3827                if (!managed_zone(zone))
3828                        continue;
3829
3830                nr_boost_reclaim += zone->watermark_boost;
3831                zone_boosts[i] = zone->watermark_boost;
3832        }
3833        boosted = nr_boost_reclaim;
3834
3835restart:
3836        set_reclaim_active(pgdat, highest_zoneidx);
3837        sc.priority = DEF_PRIORITY;
3838        do {
3839                unsigned long nr_reclaimed = sc.nr_reclaimed;
3840                bool raise_priority = true;
3841                bool balanced;
3842                bool ret;
3843
3844                sc.reclaim_idx = highest_zoneidx;
3845
3846                /*
3847                 * If the number of buffer_heads exceeds the maximum allowed
3848                 * then consider reclaiming from all zones. This has a dual
3849                 * purpose -- on 64-bit systems it is expected that
3850                 * buffer_heads are stripped during active rotation. On 32-bit
3851                 * systems, highmem pages can pin lowmem memory and shrinking
3852                 * buffers can relieve lowmem pressure. Reclaim may still not
3853                 * go ahead if all eligible zones for the original allocation
3854                 * request are balanced to avoid excessive reclaim from kswapd.
3855                 */
3856                if (buffer_heads_over_limit) {
3857                        for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
3858                                zone = pgdat->node_zones + i;
3859                                if (!managed_zone(zone))
3860                                        continue;
3861
3862                                sc.reclaim_idx = i;
3863                                break;
3864                        }
3865                }
3866
3867                /*
3868                 * If the pgdat is imbalanced then ignore boosting and preserve
3869                 * the watermarks for a later time and restart. Note that the
3870                 * zone watermarks will be still reset at the end of balancing
3871                 * on the grounds that the normal reclaim should be enough to
3872                 * re-evaluate if boosting is required when kswapd next wakes.
3873                 */
3874                balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);
3875                if (!balanced && nr_boost_reclaim) {
3876                        nr_boost_reclaim = 0;
3877                        goto restart;
3878                }
3879
3880                /*
3881                 * If boosting is not active then only reclaim if there are no
3882                 * eligible zones. Note that sc.reclaim_idx is not used as
3883                 * buffer_heads_over_limit may have adjusted it.
3884                 */
3885                if (!nr_boost_reclaim && balanced)
3886                        goto out;
3887
3888                /* Limit the priority of boosting to avoid reclaim writeback */
3889                if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
3890                        raise_priority = false;
3891
3892                /*
3893                 * Do not writeback or swap pages for boosted reclaim. The
3894                 * intent is to relieve pressure not issue sub-optimal IO
3895                 * from reclaim context. If no pages are reclaimed, the
3896                 * reclaim will be aborted.
3897                 */
3898                sc.may_writepage = !laptop_mode && !nr_boost_reclaim;
3899                sc.may_swap = !nr_boost_reclaim;
3900
3901                /*
3902                 * Do some background aging of the anon list, to give
3903                 * pages a chance to be referenced before reclaiming. All
3904                 * pages are rotated regardless of classzone as this is
3905                 * about consistent aging.
3906                 */
3907                age_active_anon(pgdat, &sc);
3908
3909                /*
3910                 * If we're getting trouble reclaiming, start doing writepage
3911                 * even in laptop mode.
3912                 */
3913                if (sc.priority < DEF_PRIORITY - 2)
3914                        sc.may_writepage = 1;
3915
3916                /* Call soft limit reclaim before calling shrink_node. */
3917                sc.nr_scanned = 0;
3918                nr_soft_scanned = 0;
3919                nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
3920                                                sc.gfp_mask, &nr_soft_scanned);
3921                sc.nr_reclaimed += nr_soft_reclaimed;
3922
3923                /*
3924                 * There should be no need to raise the scanning priority if
3925                 * enough pages are already being scanned that that high
3926                 * watermark would be met at 100% efficiency.
3927                 */
3928                if (kswapd_shrink_node(pgdat, &sc))
3929                        raise_priority = false;
3930
3931                /*
3932                 * If the low watermark is met there is no need for processes
3933                 * to be throttled on pfmemalloc_wait as they should not be
3934                 * able to safely make forward progress. Wake them
3935                 */
3936                if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
3937                                allow_direct_reclaim(pgdat))
3938                        wake_up_all(&pgdat->pfmemalloc_wait);
3939
3940                /* Check if kswapd should be suspending */
3941                __fs_reclaim_release();
3942                ret = try_to_freeze();
3943                __fs_reclaim_acquire();
3944                if (ret || kthread_should_stop())
3945                        break;
3946
3947                /*
3948                 * Raise priority if scanning rate is too low or there was no
3949                 * progress in reclaiming pages
3950                 */
3951                nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
3952                nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
3953
3954                /*
3955                 * If reclaim made no progress for a boost, stop reclaim as
3956                 * IO cannot be queued and it could be an infinite loop in
3957                 * extreme circumstances.
3958                 */
3959                if (nr_boost_reclaim && !nr_reclaimed)
3960                        break;
3961
3962                if (raise_priority || !nr_reclaimed)
3963                        sc.priority--;
3964        } while (sc.priority >= 1);
3965
3966        if (!sc.nr_reclaimed)
3967                pgdat->kswapd_failures++;
3968
3969out:
3970        clear_reclaim_active(pgdat, highest_zoneidx);
3971
3972        /* If reclaim was boosted, account for the reclaim done in this pass */
3973        if (boosted) {
3974                unsigned long flags;
3975
3976                for (i = 0; i <= highest_zoneidx; i++) {
3977                        if (!zone_boosts[i])
3978                                continue;
3979
3980                        /* Increments are under the zone lock */
3981                        zone = pgdat->node_zones + i;
3982                        spin_lock_irqsave(&zone->lock, flags);
3983                        zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
3984                        spin_unlock_irqrestore(&zone->lock, flags);
3985                }
3986
3987                /*
3988                 * As there is now likely space, wakeup kcompact to defragment
3989                 * pageblocks.
3990                 */
3991                wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);
3992        }
3993
3994        snapshot_refaults(NULL, pgdat);
3995        __fs_reclaim_release();
3996        psi_memstall_leave(&pflags);
3997        set_task_reclaim_state(current, NULL);
3998
3999        /*
4000         * Return the order kswapd stopped reclaiming at as
4001         * prepare_kswapd_sleep() takes it into account. If another caller
4002         * entered the allocator slow path while kswapd was awake, order will
4003         * remain at the higher level.
4004         */
4005        return sc.order;
4006}
4007
4008/*
4009 * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
4010 * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
4011 * not a valid index then either kswapd runs for first time or kswapd couldn't
4012 * sleep after previous reclaim attempt (node is still unbalanced). In that
4013 * case return the zone index of the previous kswapd reclaim cycle.
4014 */
4015static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,
4016                                           enum zone_type prev_highest_zoneidx)
4017{
4018        enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
4019
4020        return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx;
4021}
4022
4023static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
4024                                unsigned int highest_zoneidx)
4025{
4026        long remaining = 0;
4027        DEFINE_WAIT(wait);
4028
4029        if (freezing(current) || kthread_should_stop())
4030                return;
4031
4032        prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
4033
4034        /*
4035         * Try to sleep for a short interval. Note that kcompactd will only be
4036         * woken if it is possible to sleep for a short interval. This is
4037         * deliberate on the assumption that if reclaim cannot keep an
4038         * eligible zone balanced that it's also unlikely that compaction will
4039         * succeed.
4040         */
4041        if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
4042                /*
4043                 * Compaction records what page blocks it recently failed to
4044                 * isolate pages from and skips them in the future scanning.
4045                 * When kswapd is going to sleep, it is reasonable to assume
4046                 * that pages and compaction may succeed so reset the cache.
4047                 */
4048                reset_isolation_suitable(pgdat);
4049
4050                /*
4051                 * We have freed the memory, now we should compact it to make
4052                 * allocation of the requested order possible.
4053                 */
4054                wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);
4055
4056                remaining = schedule_timeout(HZ/10);
4057
4058                /*
4059                 * If woken prematurely then reset kswapd_highest_zoneidx and
4060                 * order. The values will either be from a wakeup request or
4061                 * the previous request that slept prematurely.
4062                 */
4063                if (remaining) {
4064                        WRITE_ONCE(pgdat->kswapd_highest_zoneidx,
4065                                        kswapd_highest_zoneidx(pgdat,
4066                                                        highest_zoneidx));
4067
4068                        if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)
4069                                WRITE_ONCE(pgdat->kswapd_order, reclaim_order);
4070                }
4071
4072                finish_wait(&pgdat->kswapd_wait, &wait);
4073                prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
4074        }
4075
4076        /*
4077         * After a short sleep, check if it was a premature sleep. If not, then
4078         * go fully to sleep until explicitly woken up.
4079         */
4080        if (!remaining &&
4081            prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
4082                trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
4083
4084                /*
4085                 * vmstat counters are not perfectly accurate and the estimated
4086                 * value for counters such as NR_FREE_PAGES can deviate from the
4087                 * true value by nr_online_cpus * threshold. To avoid the zone
4088                 * watermarks being breached while under pressure, we reduce the
4089                 * per-cpu vmstat threshold while kswapd is awake and restore
4090                 * them before going back to sleep.
4091                 */
4092                set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
4093
4094                if (!kthread_should_stop())
4095                        schedule();
4096
4097                set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
4098        } else {
4099                if (remaining)
4100                        count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
4101                else
4102                        count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
4103        }
4104        finish_wait(&pgdat->kswapd_wait, &wait);
4105}
4106
4107/*
4108 * The background pageout daemon, started as a kernel thread
4109 * from the init process.
4110 *
4111 * This basically trickles out pages so that we have _some_
4112 * free memory available even if there is no other activity
4113 * that frees anything up. This is needed for things like routing
4114 * etc, where we otherwise might have all activity going on in
4115 * asynchronous contexts that cannot page things out.
4116 *
4117 * If there are applications that are active memory-allocators
4118 * (most normal use), this basically shouldn't matter.
4119 */
4120static int kswapd(void *p)
4121{
4122        unsigned int alloc_order, reclaim_order;
4123        unsigned int highest_zoneidx = MAX_NR_ZONES - 1;
4124        pg_data_t *pgdat = (pg_data_t *)p;
4125        struct task_struct *tsk = current;
4126        const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
4127
4128        if (!cpumask_empty(cpumask))
4129                set_cpus_allowed_ptr(tsk, cpumask);
4130
4131        /*
4132         * Tell the memory management that we're a "memory allocator",
4133         * and that if we need more memory we should get access to it
4134         * regardless (see "__alloc_pages()"). "kswapd" should
4135         * never get caught in the normal page freeing logic.
4136         *
4137         * (Kswapd normally doesn't need memory anyway, but sometimes
4138         * you need a small amount of memory in order to be able to
4139         * page out something else, and this flag essentially protects
4140         * us from recursively trying to free more memory as we're
4141         * trying to free the first piece of memory in the first place).
4142         */
4143        tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
4144        set_freezable();
4145
4146        WRITE_ONCE(pgdat->kswapd_order, 0);
4147        WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
4148        for ( ; ; ) {
4149                bool ret;
4150
4151                alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
4152                highest_zoneidx = kswapd_highest_zoneidx(pgdat,
4153                                                        highest_zoneidx);
4154
4155kswapd_try_sleep:
4156                kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
4157                                        highest_zoneidx);
4158
4159                /* Read the new order and highest_zoneidx */
4160                alloc_order = READ_ONCE(pgdat->kswapd_order);
4161                highest_zoneidx = kswapd_highest_zoneidx(pgdat,
4162                                                        highest_zoneidx);
4163                WRITE_ONCE(pgdat->kswapd_order, 0);
4164                WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
4165
4166                ret = try_to_freeze();
4167                if (kthread_should_stop())
4168                        break;
4169
4170                /*
4171                 * We can speed up thawing tasks if we don't call balance_pgdat
4172                 * after returning from the refrigerator
4173                 */
4174                if (ret)
4175                        continue;
4176
4177                /*
4178                 * Reclaim begins at the requested order but if a high-order
4179                 * reclaim fails then kswapd falls back to reclaiming for
4180                 * order-0. If that happens, kswapd will consider sleeping
4181                 * for the order it finished reclaiming at (reclaim_order)
4182                 * but kcompactd is woken to compact for the original
4183                 * request (alloc_order).
4184                 */
4185                trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,
4186                                                alloc_order);
4187                reclaim_order = balance_pgdat(pgdat, alloc_order,
4188                                                highest_zoneidx);
4189                if (reclaim_order < alloc_order)
4190                        goto kswapd_try_sleep;
4191        }
4192
4193        tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD);
4194
4195        return 0;
4196}
4197
4198/*
4199 * A zone is low on free memory or too fragmented for high-order memory.  If
4200 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
4201 * pgdat.  It will wake up kcompactd after reclaiming memory.  If kswapd reclaim
4202 * has failed or is not needed, still wake up kcompactd if only compaction is
4203 * needed.
4204 */
4205void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
4206                   enum zone_type highest_zoneidx)
4207{
4208        pg_data_t *pgdat;
4209        enum zone_type curr_idx;
4210
4211        if (!managed_zone(zone))
4212                return;
4213
4214        if (!cpuset_zone_allowed(zone, gfp_flags))
4215                return;
4216
4217        pgdat = zone->zone_pgdat;
4218        curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
4219
4220        if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx)
4221                WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);
4222
4223        if (READ_ONCE(pgdat->kswapd_order) < order)
4224                WRITE_ONCE(pgdat->kswapd_order, order);
4225
4226        if (!waitqueue_active(&pgdat->kswapd_wait))
4227                return;
4228
4229        /* Hopeless node, leave it to direct reclaim if possible */
4230        if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
4231            (pgdat_balanced(pgdat, order, highest_zoneidx) &&
4232             !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
4233                /*
4234                 * There may be plenty of free memory available, but it's too
4235                 * fragmented for high-order allocations.  Wake up kcompactd
4236                 * and rely on compaction_suitable() to determine if it's
4237                 * needed.  If it fails, it will defer subsequent attempts to
4238                 * ratelimit its work.
4239                 */
4240                if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
4241                        wakeup_kcompactd(pgdat, order, highest_zoneidx);
4242                return;
4243        }
4244
4245        trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,
4246                                      gfp_flags);
4247        wake_up_interruptible(&pgdat->kswapd_wait);
4248}
4249
4250#ifdef CONFIG_HIBERNATION
4251/*
4252 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
4253 * freed pages.
4254 *
4255 * Rather than trying to age LRUs the aim is to preserve the overall
4256 * LRU order by reclaiming preferentially
4257 * inactive > active > active referenced > active mapped
4258 */
4259unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
4260{
4261        struct scan_control sc = {
4262                .nr_to_reclaim = nr_to_reclaim,
4263                .gfp_mask = GFP_HIGHUSER_MOVABLE,
4264                .reclaim_idx = MAX_NR_ZONES - 1,
4265                .priority = DEF_PRIORITY,
4266                .may_writepage = 1,
4267                .may_unmap = 1,
4268                .may_swap = 1,
4269                .hibernation_mode = 1,
4270        };
4271        struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
4272        unsigned long nr_reclaimed;
4273        unsigned int noreclaim_flag;
4274
4275        fs_reclaim_acquire(sc.gfp_mask);
4276        noreclaim_flag = memalloc_noreclaim_save();
4277        set_task_reclaim_state(current, &sc.reclaim_state);
4278
4279        nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
4280
4281        set_task_reclaim_state(current, NULL);
4282        memalloc_noreclaim_restore(noreclaim_flag);
4283        fs_reclaim_release(sc.gfp_mask);
4284
4285        return nr_reclaimed;
4286}
4287#endif /* CONFIG_HIBERNATION */
4288
4289/*
4290 * This kswapd start function will be called by init and node-hot-add.
4291 * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
4292 */
4293int kswapd_run(int nid)
4294{
4295        pg_data_t *pgdat = NODE_DATA(nid);
4296        int ret = 0;
4297
4298        if (pgdat->kswapd)
4299                return 0;
4300
4301        pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
4302        if (IS_ERR(pgdat->kswapd)) {
4303                /* failure at boot is fatal */
4304                BUG_ON(system_state < SYSTEM_RUNNING);
4305                pr_err("Failed to start kswapd on node %d\n", nid);
4306                ret = PTR_ERR(pgdat->kswapd);
4307                pgdat->kswapd = NULL;
4308        }
4309        return ret;
4310}
4311
4312/*
4313 * Called by memory hotplug when all memory in a node is offlined.  Caller must
4314 * hold mem_hotplug_begin/end().
4315 */
4316void kswapd_stop(int nid)
4317{
4318        struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
4319
4320        if (kswapd) {
4321                kthread_stop(kswapd);
4322                NODE_DATA(nid)->kswapd = NULL;
4323        }
4324}
4325
4326static int __init kswapd_init(void)
4327{
4328        int nid;
4329
4330        swap_setup();
4331        for_each_node_state(nid, N_MEMORY)
4332                kswapd_run(nid);
4333        return 0;
4334}
4335
4336module_init(kswapd_init)
4337
4338#ifdef CONFIG_NUMA
4339/*
4340 * Node reclaim mode
4341 *
4342 * If non-zero call node_reclaim when the number of free pages falls below
4343 * the watermarks.
4344 */
4345int node_reclaim_mode __read_mostly;
4346
4347/*
4348 * Priority for NODE_RECLAIM. This determines the fraction of pages
4349 * of a node considered for each zone_reclaim. 4 scans 1/16th of
4350 * a zone.
4351 */
4352#define NODE_RECLAIM_PRIORITY 4
4353
4354/*
4355 * Percentage of pages in a zone that must be unmapped for node_reclaim to
4356 * occur.
4357 */
4358int sysctl_min_unmapped_ratio = 1;
4359
4360/*
4361 * If the number of slab pages in a zone grows beyond this percentage then
4362 * slab reclaim needs to occur.
4363 */
4364int sysctl_min_slab_ratio = 5;
4365
4366static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
4367{
4368        unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
4369        unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
4370                node_page_state(pgdat, NR_ACTIVE_FILE);
4371
4372        /*
4373         * It's possible for there to be more file mapped pages than
4374         * accounted for by the pages on the file LRU lists because
4375         * tmpfs pages accounted for as ANON can also be FILE_MAPPED
4376         */
4377        return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
4378}
4379
4380/* Work out how many page cache pages we can reclaim in this reclaim_mode */
4381static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
4382{
4383        unsigned long nr_pagecache_reclaimable;
4384        unsigned long delta = 0;
4385
4386        /*
4387         * If RECLAIM_UNMAP is set, then all file pages are considered
4388         * potentially reclaimable. Otherwise, we have to worry about
4389         * pages like swapcache and node_unmapped_file_pages() provides
4390         * a better estimate
4391         */
4392        if (node_reclaim_mode & RECLAIM_UNMAP)
4393                nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
4394        else
4395                nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
4396
4397        /* If we can't clean pages, remove dirty pages from consideration */
4398        if (!(node_reclaim_mode & RECLAIM_WRITE))
4399                delta += node_page_state(pgdat, NR_FILE_DIRTY);
4400
4401        /* Watch for any possible underflows due to delta */
4402        if (unlikely(delta > nr_pagecache_reclaimable))
4403                delta = nr_pagecache_reclaimable;
4404
4405        return nr_pagecache_reclaimable - delta;
4406}
4407
4408/*
4409 * Try to free up some pages from this node through reclaim.
4410 */
4411static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4412{
4413        /* Minimum pages needed in order to stay on node */
4414        const unsigned long nr_pages = 1 << order;
4415        struct task_struct *p = current;
4416        unsigned int noreclaim_flag;
4417        struct scan_control sc = {
4418                .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
4419                .gfp_mask = current_gfp_context(gfp_mask),
4420                .order = order,
4421                .priority = NODE_RECLAIM_PRIORITY,
4422                .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
4423                .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
4424                .may_swap = 1,
4425                .reclaim_idx = gfp_zone(gfp_mask),
4426        };
4427        unsigned long pflags;
4428
4429        trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,
4430                                           sc.gfp_mask);
4431
4432        cond_resched();
4433        psi_memstall_enter(&pflags);
4434        fs_reclaim_acquire(sc.gfp_mask);
4435        /*
4436         * We need to be able to allocate from the reserves for RECLAIM_UNMAP
4437         * and we also need to be able to write out pages for RECLAIM_WRITE
4438         * and RECLAIM_UNMAP.
4439         */
4440        noreclaim_flag = memalloc_noreclaim_save();
4441        p->flags |= PF_SWAPWRITE;
4442        set_task_reclaim_state(p, &sc.reclaim_state);
4443
4444        if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
4445                /*
4446                 * Free memory by calling shrink node with increasing
4447                 * priorities until we have enough memory freed.
4448                 */
4449                do {
4450                        shrink_node(pgdat, &sc);
4451                } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
4452        }
4453
4454        set_task_reclaim_state(p, NULL);
4455        current->flags &= ~PF_SWAPWRITE;
4456        memalloc_noreclaim_restore(noreclaim_flag);
4457        fs_reclaim_release(sc.gfp_mask);
4458        psi_memstall_leave(&pflags);
4459
4460        trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed);
4461
4462        return sc.nr_reclaimed >= nr_pages;
4463}
4464
4465int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4466{
4467        int ret;
4468
4469        /*
4470         * Node reclaim reclaims unmapped file backed pages and
4471         * slab pages if we are over the defined limits.
4472         *
4473         * A small portion of unmapped file backed pages is needed for
4474         * file I/O otherwise pages read by file I/O will be immediately
4475         * thrown out if the node is overallocated. So we do not reclaim
4476         * if less than a specified percentage of the node is used by
4477         * unmapped file backed pages.
4478         */
4479        if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
4480            node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
4481            pgdat->min_slab_pages)
4482                return NODE_RECLAIM_FULL;
4483
4484        /*
4485         * Do not scan if the allocation should not be delayed.
4486         */
4487        if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
4488                return NODE_RECLAIM_NOSCAN;
4489
4490        /*
4491         * Only run node reclaim on the local node or on nodes that do not
4492         * have associated processors. This will favor the local processor
4493         * over remote processors and spread off node memory allocations
4494         * as wide as possible.
4495         */
4496        if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
4497                return NODE_RECLAIM_NOSCAN;
4498
4499        if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
4500                return NODE_RECLAIM_NOSCAN;
4501
4502        ret = __node_reclaim(pgdat, gfp_mask, order);
4503        clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
4504
4505        if (!ret)
4506                count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
4507
4508        return ret;
4509}
4510#endif
4511
4512/**
4513 * check_move_unevictable_pages - check pages for evictability and move to
4514 * appropriate zone lru list
4515 * @pvec: pagevec with lru pages to check
4516 *
4517 * Checks pages for evictability, if an evictable page is in the unevictable
4518 * lru list, moves it to the appropriate evictable lru list. This function
4519 * should be only used for lru pages.
4520 */
4521void check_move_unevictable_pages(struct pagevec *pvec)
4522{
4523        struct lruvec *lruvec = NULL;
4524        int pgscanned = 0;
4525        int pgrescued = 0;
4526        int i;
4527
4528        for (i = 0; i < pvec->nr; i++) {
4529                struct page *page = pvec->pages[i];
4530                int nr_pages;
4531
4532                if (PageTransTail(page))
4533                        continue;
4534
4535                nr_pages = thp_nr_pages(page);
4536                pgscanned += nr_pages;
4537
4538                /* block memcg migration during page moving between lru */
4539                if (!TestClearPageLRU(page))
4540                        continue;
4541
4542                lruvec = relock_page_lruvec_irq(page, lruvec);
4543                if (page_evictable(page) && PageUnevictable(page)) {
4544                        del_page_from_lru_list(page, lruvec);
4545                        ClearPageUnevictable(page);
4546                        add_page_to_lru_list(page, lruvec);
4547                        pgrescued += nr_pages;
4548                }
4549                SetPageLRU(page);
4550        }
4551
4552        if (lruvec) {
4553                __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
4554                __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4555                unlock_page_lruvec_irq(lruvec);
4556        } else if (pgscanned) {
4557                count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4558        }
4559}
4560EXPORT_SYMBOL_GPL(check_move_unevictable_pages);
4561
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.