linux/mm/hugetlb.c
<<
>>
Prefs
   1/*
   2 * Generic hugetlb support.
   3 * (C) William Irwin, April 2004
   4 */
   5#include <linux/list.h>
   6#include <linux/init.h>
   7#include <linux/module.h>
   8#include <linux/mm.h>
   9#include <linux/seq_file.h>
  10#include <linux/sysctl.h>
  11#include <linux/highmem.h>
  12#include <linux/mmu_notifier.h>
  13#include <linux/nodemask.h>
  14#include <linux/pagemap.h>
  15#include <linux/mempolicy.h>
  16#include <linux/cpuset.h>
  17#include <linux/mutex.h>
  18#include <linux/bootmem.h>
  19#include <linux/sysfs.h>
  20#include <linux/slab.h>
  21#include <linux/rmap.h>
  22#include <linux/swap.h>
  23#include <linux/swapops.h>
  24
  25#include <asm/page.h>
  26#include <asm/pgtable.h>
  27#include <linux/io.h>
  28
  29#include <linux/hugetlb.h>
  30#include <linux/node.h>
  31#include "internal.h"
  32
  33const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
  34static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
  35unsigned long hugepages_treat_as_movable;
  36
  37static int max_hstate;
  38unsigned int default_hstate_idx;
  39struct hstate hstates[HUGE_MAX_HSTATE];
  40
  41__initdata LIST_HEAD(huge_boot_pages);
  42
  43/* for command line parsing */
  44static struct hstate * __initdata parsed_hstate;
  45static unsigned long __initdata default_hstate_max_huge_pages;
  46static unsigned long __initdata default_hstate_size;
  47
  48#define for_each_hstate(h) \
  49        for ((h) = hstates; (h) < &hstates[max_hstate]; (h)++)
  50
  51/*
  52 * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
  53 */
  54static DEFINE_SPINLOCK(hugetlb_lock);
  55
  56static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
  57{
  58        bool free = (spool->count == 0) && (spool->used_hpages == 0);
  59
  60        spin_unlock(&spool->lock);
  61
  62        /* If no pages are used, and no other handles to the subpool
  63         * remain, free the subpool the subpool remain */
  64        if (free)
  65                kfree(spool);
  66}
  67
  68struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
  69{
  70        struct hugepage_subpool *spool;
  71
  72        spool = kmalloc(sizeof(*spool), GFP_KERNEL);
  73        if (!spool)
  74                return NULL;
  75
  76        spin_lock_init(&spool->lock);
  77        spool->count = 1;
  78        spool->max_hpages = nr_blocks;
  79        spool->used_hpages = 0;
  80
  81        return spool;
  82}
  83
  84void hugepage_put_subpool(struct hugepage_subpool *spool)
  85{
  86        spin_lock(&spool->lock);
  87        BUG_ON(!spool->count);
  88        spool->count--;
  89        unlock_or_release_subpool(spool);
  90}
  91
  92static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
  93                                      long delta)
  94{
  95        int ret = 0;
  96
  97        if (!spool)
  98                return 0;
  99
 100        spin_lock(&spool->lock);
 101        if ((spool->used_hpages + delta) <= spool->max_hpages) {
 102                spool->used_hpages += delta;
 103        } else {
 104                ret = -ENOMEM;
 105        }
 106        spin_unlock(&spool->lock);
 107
 108        return ret;
 109}
 110
 111static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
 112                                       long delta)
 113{
 114        if (!spool)
 115                return;
 116
 117        spin_lock(&spool->lock);
 118        spool->used_hpages -= delta;
 119        /* If hugetlbfs_put_super couldn't free spool due to
 120        * an outstanding quota reference, free it now. */
 121        unlock_or_release_subpool(spool);
 122}
 123
 124static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
 125{
 126        return HUGETLBFS_SB(inode->i_sb)->spool;
 127}
 128
 129static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
 130{
 131        return subpool_inode(vma->vm_file->f_dentry->d_inode);
 132}
 133
 134/*
 135 * Region tracking -- allows tracking of reservations and instantiated pages
 136 *                    across the pages in a mapping.
 137 *
 138 * The region data structures are protected by a combination of the mmap_sem
 139 * and the hugetlb_instantion_mutex.  To access or modify a region the caller
 140 * must either hold the mmap_sem for write, or the mmap_sem for read and
 141 * the hugetlb_instantiation mutex:
 142 *
 143 *      down_write(&mm->mmap_sem);
 144 * or
 145 *      down_read(&mm->mmap_sem);
 146 *      mutex_lock(&hugetlb_instantiation_mutex);
 147 */
 148struct file_region {
 149        struct list_head link;
 150        long from;
 151        long to;
 152};
 153
 154static long region_add(struct list_head *head, long f, long t)
 155{
 156        struct file_region *rg, *nrg, *trg;
 157
 158        /* Locate the region we are either in or before. */
 159        list_for_each_entry(rg, head, link)
 160                if (f <= rg->to)
 161                        break;
 162
 163        /* Round our left edge to the current segment if it encloses us. */
 164        if (f > rg->from)
 165                f = rg->from;
 166
 167        /* Check for and consume any regions we now overlap with. */
 168        nrg = rg;
 169        list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
 170                if (&rg->link == head)
 171                        break;
 172                if (rg->from > t)
 173                        break;
 174
 175                /* If this area reaches higher then extend our area to
 176                 * include it completely.  If this is not the first area
 177                 * which we intend to reuse, free it. */
 178                if (rg->to > t)
 179                        t = rg->to;
 180                if (rg != nrg) {
 181                        list_del(&rg->link);
 182                        kfree(rg);
 183                }
 184        }
 185        nrg->from = f;
 186        nrg->to = t;
 187        return 0;
 188}
 189
 190static long region_chg(struct list_head *head, long f, long t)
 191{
 192        struct file_region *rg, *nrg;
 193        long chg = 0;
 194
 195        /* Locate the region we are before or in. */
 196        list_for_each_entry(rg, head, link)
 197                if (f <= rg->to)
 198                        break;
 199
 200        /* If we are below the current region then a new region is required.
 201         * Subtle, allocate a new region at the position but make it zero
 202         * size such that we can guarantee to record the reservation. */
 203        if (&rg->link == head || t < rg->from) {
 204                nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
 205                if (!nrg)
 206                        return -ENOMEM;
 207                nrg->from = f;
 208                nrg->to   = f;
 209                INIT_LIST_HEAD(&nrg->link);
 210                list_add(&nrg->link, rg->link.prev);
 211
 212                return t - f;
 213        }
 214
 215        /* Round our left edge to the current segment if it encloses us. */
 216        if (f > rg->from)
 217                f = rg->from;
 218        chg = t - f;
 219
 220        /* Check for and consume any regions we now overlap with. */
 221        list_for_each_entry(rg, rg->link.prev, link) {
 222                if (&rg->link == head)
 223                        break;
 224                if (rg->from > t)
 225                        return chg;
 226
 227                /* We overlap with this area, if it extends further than
 228                 * us then we must extend ourselves.  Account for its
 229                 * existing reservation. */
 230                if (rg->to > t) {
 231                        chg += rg->to - t;
 232                        t = rg->to;
 233                }
 234                chg -= rg->to - rg->from;
 235        }
 236        return chg;
 237}
 238
 239static long region_truncate(struct list_head *head, long end)
 240{
 241        struct file_region *rg, *trg;
 242        long chg = 0;
 243
 244        /* Locate the region we are either in or before. */
 245        list_for_each_entry(rg, head, link)
 246                if (end <= rg->to)
 247                        break;
 248        if (&rg->link == head)
 249                return 0;
 250
 251        /* If we are in the middle of a region then adjust it. */
 252        if (end > rg->from) {
 253                chg = rg->to - end;
 254                rg->to = end;
 255                rg = list_entry(rg->link.next, typeof(*rg), link);
 256        }
 257
 258        /* Drop any remaining regions. */
 259        list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
 260                if (&rg->link == head)
 261                        break;
 262                chg += rg->to - rg->from;
 263                list_del(&rg->link);
 264                kfree(rg);
 265        }
 266        return chg;
 267}
 268
 269static long region_count(struct list_head *head, long f, long t)
 270{
 271        struct file_region *rg;
 272        long chg = 0;
 273
 274        /* Locate each segment we overlap with, and count that overlap. */
 275        list_for_each_entry(rg, head, link) {
 276                long seg_from;
 277                long seg_to;
 278
 279                if (rg->to <= f)
 280                        continue;
 281                if (rg->from >= t)
 282                        break;
 283
 284                seg_from = max(rg->from, f);
 285                seg_to = min(rg->to, t);
 286
 287                chg += seg_to - seg_from;
 288        }
 289
 290        return chg;
 291}
 292
 293/*
 294 * Convert the address within this vma to the page offset within
 295 * the mapping, in pagecache page units; huge pages here.
 296 */
 297static pgoff_t vma_hugecache_offset(struct hstate *h,
 298                        struct vm_area_struct *vma, unsigned long address)
 299{
 300        return ((address - vma->vm_start) >> huge_page_shift(h)) +
 301                        (vma->vm_pgoff >> huge_page_order(h));
 302}
 303
 304pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
 305                                     unsigned long address)
 306{
 307        return vma_hugecache_offset(hstate_vma(vma), vma, address);
 308}
 309
 310/*
 311 * Return the size of the pages allocated when backing a VMA. In the majority
 312 * cases this will be same size as used by the page table entries.
 313 */
 314unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
 315{
 316        struct hstate *hstate;
 317
 318        if (!is_vm_hugetlb_page(vma))
 319                return PAGE_SIZE;
 320
 321        hstate = hstate_vma(vma);
 322
 323        return 1UL << (hstate->order + PAGE_SHIFT);
 324}
 325EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
 326
 327/*
 328 * Return the page size being used by the MMU to back a VMA. In the majority
 329 * of cases, the page size used by the kernel matches the MMU size. On
 330 * architectures where it differs, an architecture-specific version of this
 331 * function is required.
 332 */
 333#ifndef vma_mmu_pagesize
 334unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
 335{
 336        return vma_kernel_pagesize(vma);
 337}
 338#endif
 339
 340/*
 341 * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
 342 * bits of the reservation map pointer, which are always clear due to
 343 * alignment.
 344 */
 345#define HPAGE_RESV_OWNER    (1UL << 0)
 346#define HPAGE_RESV_UNMAPPED (1UL << 1)
 347#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
 348
 349/*
 350 * These helpers are used to track how many pages are reserved for
 351 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
 352 * is guaranteed to have their future faults succeed.
 353 *
 354 * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
 355 * the reserve counters are updated with the hugetlb_lock held. It is safe
 356 * to reset the VMA at fork() time as it is not in use yet and there is no
 357 * chance of the global counters getting corrupted as a result of the values.
 358 *
 359 * The private mapping reservation is represented in a subtly different
 360 * manner to a shared mapping.  A shared mapping has a region map associated
 361 * with the underlying file, this region map represents the backing file
 362 * pages which have ever had a reservation assigned which this persists even
 363 * after the page is instantiated.  A private mapping has a region map
 364 * associated with the original mmap which is attached to all VMAs which
 365 * reference it, this region map represents those offsets which have consumed
 366 * reservation ie. where pages have been instantiated.
 367 */
 368static unsigned long get_vma_private_data(struct vm_area_struct *vma)
 369{
 370        return (unsigned long)vma->vm_private_data;
 371}
 372
 373static void set_vma_private_data(struct vm_area_struct *vma,
 374                                                        unsigned long value)
 375{
 376        vma->vm_private_data = (void *)value;
 377}
 378
 379struct resv_map {
 380        struct kref refs;
 381        struct list_head regions;
 382};
 383
 384static struct resv_map *resv_map_alloc(void)
 385{
 386        struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
 387        if (!resv_map)
 388                return NULL;
 389
 390        kref_init(&resv_map->refs);
 391        INIT_LIST_HEAD(&resv_map->regions);
 392
 393        return resv_map;
 394}
 395
 396static void resv_map_release(struct kref *ref)
 397{
 398        struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
 399
 400        /* Clear out any active regions before we release the map. */
 401        region_truncate(&resv_map->regions, 0);
 402        kfree(resv_map);
 403}
 404
 405static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 406{
 407        VM_BUG_ON(!is_vm_hugetlb_page(vma));
 408        if (!(vma->vm_flags & VM_MAYSHARE))
 409                return (struct resv_map *)(get_vma_private_data(vma) &
 410                                                        ~HPAGE_RESV_MASK);
 411        return NULL;
 412}
 413
 414static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 415{
 416        VM_BUG_ON(!is_vm_hugetlb_page(vma));
 417        VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
 418
 419        set_vma_private_data(vma, (get_vma_private_data(vma) &
 420                                HPAGE_RESV_MASK) | (unsigned long)map);
 421}
 422
 423static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
 424{
 425        VM_BUG_ON(!is_vm_hugetlb_page(vma));
 426        VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
 427
 428        set_vma_private_data(vma, get_vma_private_data(vma) | flags);
 429}
 430
 431static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
 432{
 433        VM_BUG_ON(!is_vm_hugetlb_page(vma));
 434
 435        return (get_vma_private_data(vma) & flag) != 0;
 436}
 437
 438/* Decrement the reserved pages in the hugepage pool by one */
 439static void decrement_hugepage_resv_vma(struct hstate *h,
 440                        struct vm_area_struct *vma)
 441{
 442        if (vma->vm_flags & VM_NORESERVE)
 443                return;
 444
 445        if (vma->vm_flags & VM_MAYSHARE) {
 446                /* Shared mappings always use reserves */
 447                h->resv_huge_pages--;
 448        } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
 449                /*
 450                 * Only the process that called mmap() has reserves for
 451                 * private mappings.
 452                 */
 453                h->resv_huge_pages--;
 454        }
 455}
 456
 457/* Reset counters to 0 and clear all HPAGE_RESV_* flags */
 458void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
 459{
 460        VM_BUG_ON(!is_vm_hugetlb_page(vma));
 461        if (!(vma->vm_flags & VM_MAYSHARE))
 462                vma->vm_private_data = (void *)0;
 463}
 464
 465/* Returns true if the VMA has associated reserve pages */
 466static int vma_has_reserves(struct vm_area_struct *vma)
 467{
 468        if (vma->vm_flags & VM_MAYSHARE)
 469                return 1;
 470        if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
 471                return 1;
 472        return 0;
 473}
 474
 475static void copy_gigantic_page(struct page *dst, struct page *src)
 476{
 477        int i;
 478        struct hstate *h = page_hstate(src);
 479        struct page *dst_base = dst;
 480        struct page *src_base = src;
 481
 482        for (i = 0; i < pages_per_huge_page(h); ) {
 483                cond_resched();
 484                copy_highpage(dst, src);
 485
 486                i++;
 487                dst = mem_map_next(dst, dst_base, i);
 488                src = mem_map_next(src, src_base, i);
 489        }
 490}
 491
 492void copy_huge_page(struct page *dst, struct page *src)
 493{
 494        int i;
 495        struct hstate *h = page_hstate(src);
 496
 497        if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
 498                copy_gigantic_page(dst, src);
 499                return;
 500        }
 501
 502        might_sleep();
 503        for (i = 0; i < pages_per_huge_page(h); i++) {
 504                cond_resched();
 505                copy_highpage(dst + i, src + i);
 506        }
 507}
 508
 509static void enqueue_huge_page(struct hstate *h, struct page *page)
 510{
 511        int nid = page_to_nid(page);
 512        list_add(&page->lru, &h->hugepage_freelists[nid]);
 513        h->free_huge_pages++;
 514        h->free_huge_pages_node[nid]++;
 515}
 516
 517static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
 518{
 519        struct page *page;
 520
 521        if (list_empty(&h->hugepage_freelists[nid]))
 522                return NULL;
 523        page = list_entry(h->hugepage_freelists[nid].next, struct page, lru);
 524        list_del(&page->lru);
 525        set_page_refcounted(page);
 526        h->free_huge_pages--;
 527        h->free_huge_pages_node[nid]--;
 528        return page;
 529}
 530
 531static struct page *dequeue_huge_page_vma(struct hstate *h,
 532                                struct vm_area_struct *vma,
 533                                unsigned long address, int avoid_reserve)
 534{
 535        struct page *page = NULL;
 536        struct mempolicy *mpol;
 537        nodemask_t *nodemask;
 538        struct zonelist *zonelist;
 539        struct zone *zone;
 540        struct zoneref *z;
 541        unsigned int cpuset_mems_cookie;
 542
 543retry_cpuset:
 544        cpuset_mems_cookie = get_mems_allowed();
 545        zonelist = huge_zonelist(vma, address,
 546                                        htlb_alloc_mask, &mpol, &nodemask);
 547        /*
 548         * A child process with MAP_PRIVATE mappings created by their parent
 549         * have no page reserves. This check ensures that reservations are
 550         * not "stolen". The child may still get SIGKILLed
 551         */
 552        if (!vma_has_reserves(vma) &&
 553                        h->free_huge_pages - h->resv_huge_pages == 0)
 554                goto err;
 555
 556        /* If reserves cannot be used, ensure enough pages are in the pool */
 557        if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
 558                goto err;
 559
 560        for_each_zone_zonelist_nodemask(zone, z, zonelist,
 561                                                MAX_NR_ZONES - 1, nodemask) {
 562                if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask)) {
 563                        page = dequeue_huge_page_node(h, zone_to_nid(zone));
 564                        if (page) {
 565                                if (!avoid_reserve)
 566                                        decrement_hugepage_resv_vma(h, vma);
 567                                break;
 568                        }
 569                }
 570        }
 571
 572        mpol_cond_put(mpol);
 573        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
 574                goto retry_cpuset;
 575        return page;
 576
 577err:
 578        mpol_cond_put(mpol);
 579        return NULL;
 580}
 581
 582static void update_and_free_page(struct hstate *h, struct page *page)
 583{
 584        int i;
 585
 586        VM_BUG_ON(h->order >= MAX_ORDER);
 587
 588        h->nr_huge_pages--;
 589        h->nr_huge_pages_node[page_to_nid(page)]--;
 590        for (i = 0; i < pages_per_huge_page(h); i++) {
 591                page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
 592                                1 << PG_referenced | 1 << PG_dirty |
 593                                1 << PG_active | 1 << PG_reserved |
 594                                1 << PG_private | 1 << PG_writeback);
 595        }
 596        set_compound_page_dtor(page, NULL);
 597        set_page_refcounted(page);
 598        arch_release_hugepage(page);
 599        __free_pages(page, huge_page_order(h));
 600}
 601
 602struct hstate *size_to_hstate(unsigned long size)
 603{
 604        struct hstate *h;
 605
 606        for_each_hstate(h) {
 607                if (huge_page_size(h) == size)
 608                        return h;
 609        }
 610        return NULL;
 611}
 612
 613static void free_huge_page(struct page *page)
 614{
 615        /*
 616         * Can't pass hstate in here because it is called from the
 617         * compound page destructor.
 618         */
 619        struct hstate *h = page_hstate(page);
 620        int nid = page_to_nid(page);
 621        struct hugepage_subpool *spool =
 622                (struct hugepage_subpool *)page_private(page);
 623
 624        set_page_private(page, 0);
 625        page->mapping = NULL;
 626        BUG_ON(page_count(page));
 627        BUG_ON(page_mapcount(page));
 628        INIT_LIST_HEAD(&page->lru);
 629
 630        spin_lock(&hugetlb_lock);
 631        if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
 632                update_and_free_page(h, page);
 633                h->surplus_huge_pages--;
 634                h->surplus_huge_pages_node[nid]--;
 635        } else {
 636                enqueue_huge_page(h, page);
 637        }
 638        spin_unlock(&hugetlb_lock);
 639        hugepage_subpool_put_pages(spool, 1);
 640}
 641
 642static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
 643{
 644        set_compound_page_dtor(page, free_huge_page);
 645        spin_lock(&hugetlb_lock);
 646        h->nr_huge_pages++;
 647        h->nr_huge_pages_node[nid]++;
 648        spin_unlock(&hugetlb_lock);
 649        put_page(page); /* free it into the hugepage allocator */
 650}
 651
 652static void prep_compound_gigantic_page(struct page *page, unsigned long order)
 653{
 654        int i;
 655        int nr_pages = 1 << order;
 656        struct page *p = page + 1;
 657
 658        /* we rely on prep_new_huge_page to set the destructor */
 659        set_compound_order(page, order);
 660        __SetPageHead(page);
 661        for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
 662                __SetPageTail(p);
 663                set_page_count(p, 0);
 664                p->first_page = page;
 665        }
 666}
 667
 668int PageHuge(struct page *page)
 669{
 670        compound_page_dtor *dtor;
 671
 672        if (!PageCompound(page))
 673                return 0;
 674
 675        page = compound_head(page);
 676        dtor = get_compound_page_dtor(page);
 677
 678        return dtor == free_huge_page;
 679}
 680EXPORT_SYMBOL_GPL(PageHuge);
 681
 682static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
 683{
 684        struct page *page;
 685
 686        if (h->order >= MAX_ORDER)
 687                return NULL;
 688
 689        page = alloc_pages_exact_node(nid,
 690                htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
 691                                                __GFP_REPEAT|__GFP_NOWARN,
 692                huge_page_order(h));
 693        if (page) {
 694                if (arch_prepare_hugepage(page)) {
 695                        __free_pages(page, huge_page_order(h));
 696                        return NULL;
 697                }
 698                prep_new_huge_page(h, page, nid);
 699        }
 700
 701        return page;
 702}
 703
 704/*
 705 * common helper functions for hstate_next_node_to_{alloc|free}.
 706 * We may have allocated or freed a huge page based on a different
 707 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
 708 * be outside of *nodes_allowed.  Ensure that we use an allowed
 709 * node for alloc or free.
 710 */
 711static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
 712{
 713        nid = next_node(nid, *nodes_allowed);
 714        if (nid == MAX_NUMNODES)
 715                nid = first_node(*nodes_allowed);
 716        VM_BUG_ON(nid >= MAX_NUMNODES);
 717
 718        return nid;
 719}
 720
 721static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
 722{
 723        if (!node_isset(nid, *nodes_allowed))
 724                nid = next_node_allowed(nid, nodes_allowed);
 725        return nid;
 726}
 727
 728/*
 729 * returns the previously saved node ["this node"] from which to
 730 * allocate a persistent huge page for the pool and advance the
 731 * next node from which to allocate, handling wrap at end of node
 732 * mask.
 733 */
 734static int hstate_next_node_to_alloc(struct hstate *h,
 735                                        nodemask_t *nodes_allowed)
 736{
 737        int nid;
 738
 739        VM_BUG_ON(!nodes_allowed);
 740
 741        nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
 742        h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
 743
 744        return nid;
 745}
 746
 747static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
 748{
 749        struct page *page;
 750        int start_nid;
 751        int next_nid;
 752        int ret = 0;
 753
 754        start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
 755        next_nid = start_nid;
 756
 757        do {
 758                page = alloc_fresh_huge_page_node(h, next_nid);
 759                if (page) {
 760                        ret = 1;
 761                        break;
 762                }
 763                next_nid = hstate_next_node_to_alloc(h, nodes_allowed);
 764        } while (next_nid != start_nid);
 765
 766        if (ret)
 767                count_vm_event(HTLB_BUDDY_PGALLOC);
 768        else
 769                count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
 770
 771        return ret;
 772}
 773
 774/*
 775 * helper for free_pool_huge_page() - return the previously saved
 776 * node ["this node"] from which to free a huge page.  Advance the
 777 * next node id whether or not we find a free huge page to free so
 778 * that the next attempt to free addresses the next node.
 779 */
 780static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
 781{
 782        int nid;
 783
 784        VM_BUG_ON(!nodes_allowed);
 785
 786        nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
 787        h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
 788
 789        return nid;
 790}
 791
 792/*
 793 * Free huge page from pool from next node to free.
 794 * Attempt to keep persistent huge pages more or less
 795 * balanced over allowed nodes.
 796 * Called with hugetlb_lock locked.
 797 */
 798static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
 799                                                         bool acct_surplus)
 800{
 801        int start_nid;
 802        int next_nid;
 803        int ret = 0;
 804
 805        start_nid = hstate_next_node_to_free(h, nodes_allowed);
 806        next_nid = start_nid;
 807
 808        do {
 809                /*
 810                 * If we're returning unused surplus pages, only examine
 811                 * nodes with surplus pages.
 812                 */
 813                if ((!acct_surplus || h->surplus_huge_pages_node[next_nid]) &&
 814                    !list_empty(&h->hugepage_freelists[next_nid])) {
 815                        struct page *page =
 816                                list_entry(h->hugepage_freelists[next_nid].next,
 817                                          struct page, lru);
 818                        list_del(&page->lru);
 819                        h->free_huge_pages--;
 820                        h->free_huge_pages_node[next_nid]--;
 821                        if (acct_surplus) {
 822                                h->surplus_huge_pages--;
 823                                h->surplus_huge_pages_node[next_nid]--;
 824                        }
 825                        update_and_free_page(h, page);
 826                        ret = 1;
 827                        break;
 828                }
 829                next_nid = hstate_next_node_to_free(h, nodes_allowed);
 830        } while (next_nid != start_nid);
 831
 832        return ret;
 833}
 834
 835static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
 836{
 837        struct page *page;
 838        unsigned int r_nid;
 839
 840        if (h->order >= MAX_ORDER)
 841                return NULL;
 842
 843        /*
 844         * Assume we will successfully allocate the surplus page to
 845         * prevent racing processes from causing the surplus to exceed
 846         * overcommit
 847         *
 848         * This however introduces a different race, where a process B
 849         * tries to grow the static hugepage pool while alloc_pages() is
 850         * called by process A. B will only examine the per-node
 851         * counters in determining if surplus huge pages can be
 852         * converted to normal huge pages in adjust_pool_surplus(). A
 853         * won't be able to increment the per-node counter, until the
 854         * lock is dropped by B, but B doesn't drop hugetlb_lock until
 855         * no more huge pages can be converted from surplus to normal
 856         * state (and doesn't try to convert again). Thus, we have a
 857         * case where a surplus huge page exists, the pool is grown, and
 858         * the surplus huge page still exists after, even though it
 859         * should just have been converted to a normal huge page. This
 860         * does not leak memory, though, as the hugepage will be freed
 861         * once it is out of use. It also does not allow the counters to
 862         * go out of whack in adjust_pool_surplus() as we don't modify
 863         * the node values until we've gotten the hugepage and only the
 864         * per-node value is checked there.
 865         */
 866        spin_lock(&hugetlb_lock);
 867        if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
 868                spin_unlock(&hugetlb_lock);
 869                return NULL;
 870        } else {
 871                h->nr_huge_pages++;
 872                h->surplus_huge_pages++;
 873        }
 874        spin_unlock(&hugetlb_lock);
 875
 876        if (nid == NUMA_NO_NODE)
 877                page = alloc_pages(htlb_alloc_mask|__GFP_COMP|
 878                                   __GFP_REPEAT|__GFP_NOWARN,
 879                                   huge_page_order(h));
 880        else
 881                page = alloc_pages_exact_node(nid,
 882                        htlb_alloc_mask|__GFP_COMP|__GFP_THISNODE|
 883                        __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
 884
 885        if (page && arch_prepare_hugepage(page)) {
 886                __free_pages(page, huge_page_order(h));
 887                page = NULL;
 888        }
 889
 890        spin_lock(&hugetlb_lock);
 891        if (page) {
 892                r_nid = page_to_nid(page);
 893                set_compound_page_dtor(page, free_huge_page);
 894                /*
 895                 * We incremented the global counters already
 896                 */
 897                h->nr_huge_pages_node[r_nid]++;
 898                h->surplus_huge_pages_node[r_nid]++;
 899                __count_vm_event(HTLB_BUDDY_PGALLOC);
 900        } else {
 901                h->nr_huge_pages--;
 902                h->surplus_huge_pages--;
 903                __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
 904        }
 905        spin_unlock(&hugetlb_lock);
 906
 907        return page;
 908}
 909
 910/*
 911 * This allocation function is useful in the context where vma is irrelevant.
 912 * E.g. soft-offlining uses this function because it only cares physical
 913 * address of error page.
 914 */
 915struct page *alloc_huge_page_node(struct hstate *h, int nid)
 916{
 917        struct page *page;
 918
 919        spin_lock(&hugetlb_lock);
 920        page = dequeue_huge_page_node(h, nid);
 921        spin_unlock(&hugetlb_lock);
 922
 923        if (!page)
 924                page = alloc_buddy_huge_page(h, nid);
 925
 926        return page;
 927}
 928
 929/*
 930 * Increase the hugetlb pool such that it can accommodate a reservation
 931 * of size 'delta'.
 932 */
 933static int gather_surplus_pages(struct hstate *h, int delta)
 934{
 935        struct list_head surplus_list;
 936        struct page *page, *tmp;
 937        int ret, i;
 938        int needed, allocated;
 939        bool alloc_ok = true;
 940
 941        needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
 942        if (needed <= 0) {
 943                h->resv_huge_pages += delta;
 944                return 0;
 945        }
 946
 947        allocated = 0;
 948        INIT_LIST_HEAD(&surplus_list);
 949
 950        ret = -ENOMEM;
 951retry:
 952        spin_unlock(&hugetlb_lock);
 953        for (i = 0; i < needed; i++) {
 954                page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
 955                if (!page) {
 956                        alloc_ok = false;
 957                        break;
 958                }
 959                list_add(&page->lru, &surplus_list);
 960        }
 961        allocated += i;
 962
 963        /*
 964         * After retaking hugetlb_lock, we need to recalculate 'needed'
 965         * because either resv_huge_pages or free_huge_pages may have changed.
 966         */
 967        spin_lock(&hugetlb_lock);
 968        needed = (h->resv_huge_pages + delta) -
 969                        (h->free_huge_pages + allocated);
 970        if (needed > 0) {
 971                if (alloc_ok)
 972                        goto retry;
 973                /*
 974                 * We were not able to allocate enough pages to
 975                 * satisfy the entire reservation so we free what
 976                 * we've allocated so far.
 977                 */
 978                goto free;
 979        }
 980        /*
 981         * The surplus_list now contains _at_least_ the number of extra pages
 982         * needed to accommodate the reservation.  Add the appropriate number
 983         * of pages to the hugetlb pool and free the extras back to the buddy
 984         * allocator.  Commit the entire reservation here to prevent another
 985         * process from stealing the pages as they are added to the pool but
 986         * before they are reserved.
 987         */
 988        needed += allocated;
 989        h->resv_huge_pages += delta;
 990        ret = 0;
 991
 992        /* Free the needed pages to the hugetlb pool */
 993        list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
 994                if ((--needed) < 0)
 995                        break;
 996                list_del(&page->lru);
 997                /*
 998                 * This page is now managed by the hugetlb allocator and has
 999                 * no users -- drop the buddy allocator's reference.
1000                 */
1001                put_page_testzero(page);
1002                VM_BUG_ON(page_count(page));
1003                enqueue_huge_page(h, page);
1004        }
1005free:
1006        spin_unlock(&hugetlb_lock);
1007
1008        /* Free unnecessary surplus pages to the buddy allocator */
1009        if (!list_empty(&surplus_list)) {
1010                list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1011                        list_del(&page->lru);
1012                        put_page(page);
1013                }
1014        }
1015        spin_lock(&hugetlb_lock);
1016
1017        return ret;
1018}
1019
1020/*
1021 * When releasing a hugetlb pool reservation, any surplus pages that were
1022 * allocated to satisfy the reservation must be explicitly freed if they were
1023 * never used.
1024 * Called with hugetlb_lock held.
1025 */
1026static void return_unused_surplus_pages(struct hstate *h,
1027                                        unsigned long unused_resv_pages)
1028{
1029        unsigned long nr_pages;
1030
1031        /* Uncommit the reservation */
1032        h->resv_huge_pages -= unused_resv_pages;
1033
1034        /* Cannot return gigantic pages currently */
1035        if (h->order >= MAX_ORDER)
1036                return;
1037
1038        nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1039
1040        /*
1041         * We want to release as many surplus pages as possible, spread
1042         * evenly across all nodes with memory. Iterate across these nodes
1043         * until we can no longer free unreserved surplus pages. This occurs
1044         * when the nodes with surplus pages have no free pages.
1045         * free_pool_huge_page() will balance the the freed pages across the
1046         * on-line nodes with memory and will handle the hstate accounting.
1047         */
1048        while (nr_pages--) {
1049                if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1))
1050                        break;
1051        }
1052}
1053
1054/*
1055 * Determine if the huge page at addr within the vma has an associated
1056 * reservation.  Where it does not we will need to logically increase
1057 * reservation and actually increase subpool usage before an allocation
1058 * can occur.  Where any new reservation would be required the
1059 * reservation change is prepared, but not committed.  Once the page
1060 * has been allocated from the subpool and instantiated the change should
1061 * be committed via vma_commit_reservation.  No action is required on
1062 * failure.
1063 */
1064static long vma_needs_reservation(struct hstate *h,
1065                        struct vm_area_struct *vma, unsigned long addr)
1066{
1067        struct address_space *mapping = vma->vm_file->f_mapping;
1068        struct inode *inode = mapping->host;
1069
1070        if (vma->vm_flags & VM_MAYSHARE) {
1071                pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1072                return region_chg(&inode->i_mapping->private_list,
1073                                                        idx, idx + 1);
1074
1075        } else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1076                return 1;
1077
1078        } else  {
1079                long err;
1080                pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1081                struct resv_map *reservations = vma_resv_map(vma);
1082
1083                err = region_chg(&reservations->regions, idx, idx + 1);
1084                if (err < 0)
1085                        return err;
1086                return 0;
1087        }
1088}
1089static void vma_commit_reservation(struct hstate *h,
1090                        struct vm_area_struct *vma, unsigned long addr)
1091{
1092        struct address_space *mapping = vma->vm_file->f_mapping;
1093        struct inode *inode = mapping->host;
1094
1095        if (vma->vm_flags & VM_MAYSHARE) {
1096                pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1097                region_add(&inode->i_mapping->private_list, idx, idx + 1);
1098
1099        } else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1100                pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1101                struct resv_map *reservations = vma_resv_map(vma);
1102
1103                /* Mark this page used in the map. */
1104                region_add(&reservations->regions, idx, idx + 1);
1105        }
1106}
1107
1108static struct page *alloc_huge_page(struct vm_area_struct *vma,
1109                                    unsigned long addr, int avoid_reserve)
1110{
1111        struct hugepage_subpool *spool = subpool_vma(vma);
1112        struct hstate *h = hstate_vma(vma);
1113        struct page *page;
1114        long chg;
1115
1116        /*
1117         * Processes that did not create the mapping will have no
1118         * reserves and will not have accounted against subpool
1119         * limit. Check that the subpool limit can be made before
1120         * satisfying the allocation MAP_NORESERVE mappings may also
1121         * need pages and subpool limit allocated allocated if no reserve
1122         * mapping overlaps.
1123         */
1124        chg = vma_needs_reservation(h, vma, addr);
1125        if (chg < 0)
1126                return ERR_PTR(-VM_FAULT_OOM);
1127        if (chg)
1128                if (hugepage_subpool_get_pages(spool, chg))
1129                        return ERR_PTR(-VM_FAULT_SIGBUS);
1130
1131        spin_lock(&hugetlb_lock);
1132        page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve);
1133        spin_unlock(&hugetlb_lock);
1134
1135        if (!page) {
1136                page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1137                if (!page) {
1138                        hugepage_subpool_put_pages(spool, chg);
1139                        return ERR_PTR(-VM_FAULT_SIGBUS);
1140                }
1141        }
1142
1143        set_page_private(page, (unsigned long)spool);
1144
1145        vma_commit_reservation(h, vma, addr);
1146
1147        return page;
1148}
1149
1150int __weak alloc_bootmem_huge_page(struct hstate *h)
1151{
1152        struct huge_bootmem_page *m;
1153        int nr_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
1154
1155        while (nr_nodes) {
1156                void *addr;
1157
1158                addr = __alloc_bootmem_node_nopanic(
1159                                NODE_DATA(hstate_next_node_to_alloc(h,
1160                                                &node_states[N_HIGH_MEMORY])),
1161                                huge_page_size(h), huge_page_size(h), 0);
1162
1163                if (addr) {
1164                        /*
1165                         * Use the beginning of the huge page to store the
1166                         * huge_bootmem_page struct (until gather_bootmem
1167                         * puts them into the mem_map).
1168                         */
1169                        m = addr;
1170                        goto found;
1171                }
1172                nr_nodes--;
1173        }
1174        return 0;
1175
1176found:
1177        BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1178        /* Put them into a private list first because mem_map is not up yet */
1179        list_add(&m->list, &huge_boot_pages);
1180        m->hstate = h;
1181        return 1;
1182}
1183
1184static void prep_compound_huge_page(struct page *page, int order)
1185{
1186        if (unlikely(order > (MAX_ORDER - 1)))
1187                prep_compound_gigantic_page(page, order);
1188        else
1189                prep_compound_page(page, order);
1190}
1191
1192/* Put bootmem huge pages into the standard lists after mem_map is up */
1193static void __init gather_bootmem_prealloc(void)
1194{
1195        struct huge_bootmem_page *m;
1196
1197        list_for_each_entry(m, &huge_boot_pages, list) {
1198                struct hstate *h = m->hstate;
1199                struct page *page;
1200
1201#ifdef CONFIG_HIGHMEM
1202                page = pfn_to_page(m->phys >> PAGE_SHIFT);
1203                free_bootmem_late((unsigned long)m,
1204                                  sizeof(struct huge_bootmem_page));
1205#else
1206                page = virt_to_page(m);
1207#endif
1208                __ClearPageReserved(page);
1209                WARN_ON(page_count(page) != 1);
1210                prep_compound_huge_page(page, h->order);
1211                prep_new_huge_page(h, page, page_to_nid(page));
1212                /*
1213                 * If we had gigantic hugepages allocated at boot time, we need
1214                 * to restore the 'stolen' pages to totalram_pages in order to
1215                 * fix confusing memory reports from free(1) and another
1216                 * side-effects, like CommitLimit going negative.
1217                 */
1218                if (h->order > (MAX_ORDER - 1))
1219                        totalram_pages += 1 << h->order;
1220        }
1221}
1222
1223static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1224{
1225        unsigned long i;
1226
1227        for (i = 0; i < h->max_huge_pages; ++i) {
1228                if (h->order >= MAX_ORDER) {
1229                        if (!alloc_bootmem_huge_page(h))
1230                                break;
1231                } else if (!alloc_fresh_huge_page(h,
1232                                         &node_states[N_HIGH_MEMORY]))
1233                        break;
1234        }
1235        h->max_huge_pages = i;
1236}
1237
1238static void __init hugetlb_init_hstates(void)
1239{
1240        struct hstate *h;
1241
1242        for_each_hstate(h) {
1243                /* oversize hugepages were init'ed in early boot */
1244                if (h->order < MAX_ORDER)
1245                        hugetlb_hstate_alloc_pages(h);
1246        }
1247}
1248
1249static char * __init memfmt(char *buf, unsigned long n)
1250{
1251        if (n >= (1UL << 30))
1252                sprintf(buf, "%lu GB", n >> 30);
1253        else if (n >= (1UL << 20))
1254                sprintf(buf, "%lu MB", n >> 20);
1255        else
1256                sprintf(buf, "%lu KB", n >> 10);
1257        return buf;
1258}
1259
1260static void __init report_hugepages(void)
1261{
1262        struct hstate *h;
1263
1264        for_each_hstate(h) {
1265                char buf[32];
1266                printk(KERN_INFO "HugeTLB registered %s page size, "
1267                                 "pre-allocated %ld pages\n",
1268                        memfmt(buf, huge_page_size(h)),
1269                        h->free_huge_pages);
1270        }
1271}
1272
1273#ifdef CONFIG_HIGHMEM
1274static void try_to_free_low(struct hstate *h, unsigned long count,
1275                                                nodemask_t *nodes_allowed)
1276{
1277        int i;
1278
1279        if (h->order >= MAX_ORDER)
1280                return;
1281
1282        for_each_node_mask(i, *nodes_allowed) {
1283                struct page *page, *next;
1284                struct list_head *freel = &h->hugepage_freelists[i];
1285                list_for_each_entry_safe(page, next, freel, lru) {
1286                        if (count >= h->nr_huge_pages)
1287                                return;
1288                        if (PageHighMem(page))
1289                                continue;
1290                        list_del(&page->lru);
1291                        update_and_free_page(h, page);
1292                        h->free_huge_pages--;
1293                        h->free_huge_pages_node[page_to_nid(page)]--;
1294                }
1295        }
1296}
1297#else
1298static inline void try_to_free_low(struct hstate *h, unsigned long count,
1299                                                nodemask_t *nodes_allowed)
1300{
1301}
1302#endif
1303
1304/*
1305 * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1306 * balanced by operating on them in a round-robin fashion.
1307 * Returns 1 if an adjustment was made.
1308 */
1309static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1310                                int delta)
1311{
1312        int start_nid, next_nid;
1313        int ret = 0;
1314
1315        VM_BUG_ON(delta != -1 && delta != 1);
1316
1317        if (delta < 0)
1318                start_nid = hstate_next_node_to_alloc(h, nodes_allowed);
1319        else
1320                start_nid = hstate_next_node_to_free(h, nodes_allowed);
1321        next_nid = start_nid;
1322
1323        do {
1324                int nid = next_nid;
1325                if (delta < 0)  {
1326                        /*
1327                         * To shrink on this node, there must be a surplus page
1328                         */
1329                        if (!h->surplus_huge_pages_node[nid]) {
1330                                next_nid = hstate_next_node_to_alloc(h,
1331                                                                nodes_allowed);
1332                                continue;
1333                        }
1334                }
1335                if (delta > 0) {
1336                        /*
1337                         * Surplus cannot exceed the total number of pages
1338                         */
1339                        if (h->surplus_huge_pages_node[nid] >=
1340                                                h->nr_huge_pages_node[nid]) {
1341                                next_nid = hstate_next_node_to_free(h,
1342                                                                nodes_allowed);
1343                                continue;
1344                        }
1345                }
1346
1347                h->surplus_huge_pages += delta;
1348                h->surplus_huge_pages_node[nid] += delta;
1349                ret = 1;
1350                break;
1351        } while (next_nid != start_nid);
1352
1353        return ret;
1354}
1355
1356#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1357static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1358                                                nodemask_t *nodes_allowed)
1359{
1360        unsigned long min_count, ret;
1361
1362        if (h->order >= MAX_ORDER)
1363                return h->max_huge_pages;
1364
1365        /*
1366         * Increase the pool size
1367         * First take pages out of surplus state.  Then make up the
1368         * remaining difference by allocating fresh huge pages.
1369         *
1370         * We might race with alloc_buddy_huge_page() here and be unable
1371         * to convert a surplus huge page to a normal huge page. That is
1372         * not critical, though, it just means the overall size of the
1373         * pool might be one hugepage larger than it needs to be, but
1374         * within all the constraints specified by the sysctls.
1375         */
1376        spin_lock(&hugetlb_lock);
1377        while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1378                if (!adjust_pool_surplus(h, nodes_allowed, -1))
1379                        break;
1380        }
1381
1382        while (count > persistent_huge_pages(h)) {
1383                /*
1384                 * If this allocation races such that we no longer need the
1385                 * page, free_huge_page will handle it by freeing the page
1386                 * and reducing the surplus.
1387                 */
1388                spin_unlock(&hugetlb_lock);
1389                ret = alloc_fresh_huge_page(h, nodes_allowed);
1390                spin_lock(&hugetlb_lock);
1391                if (!ret)
1392                        goto out;
1393
1394                /* Bail for signals. Probably ctrl-c from user */
1395                if (signal_pending(current))
1396                        goto out;
1397        }
1398
1399        /*
1400         * Decrease the pool size
1401         * First return free pages to the buddy allocator (being careful
1402         * to keep enough around to satisfy reservations).  Then place
1403         * pages into surplus state as needed so the pool will shrink
1404         * to the desired size as pages become free.
1405         *
1406         * By placing pages into the surplus state independent of the
1407         * overcommit value, we are allowing the surplus pool size to
1408         * exceed overcommit. There are few sane options here. Since
1409         * alloc_buddy_huge_page() is checking the global counter,
1410         * though, we'll note that we're not allowed to exceed surplus
1411         * and won't grow the pool anywhere else. Not until one of the
1412         * sysctls are changed, or the surplus pages go out of use.
1413         */
1414        min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1415        min_count = max(count, min_count);
1416        try_to_free_low(h, min_count, nodes_allowed);
1417        while (min_count < persistent_huge_pages(h)) {
1418                if (!free_pool_huge_page(h, nodes_allowed, 0))
1419                        break;
1420        }
1421        while (count < persistent_huge_pages(h)) {
1422                if (!adjust_pool_surplus(h, nodes_allowed, 1))
1423                        break;
1424        }
1425out:
1426        ret = persistent_huge_pages(h);
1427        spin_unlock(&hugetlb_lock);
1428        return ret;
1429}
1430
1431#define HSTATE_ATTR_RO(_name) \
1432        static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1433
1434#define HSTATE_ATTR(_name) \
1435        static struct kobj_attribute _name##_attr = \
1436                __ATTR(_name, 0644, _name##_show, _name##_store)
1437
1438static struct kobject *hugepages_kobj;
1439static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1440
1441static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1442
1443static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1444{
1445        int i;
1446
1447        for (i = 0; i < HUGE_MAX_HSTATE; i++)
1448                if (hstate_kobjs[i] == kobj) {
1449                        if (nidp)
1450                                *nidp = NUMA_NO_NODE;
1451                        return &hstates[i];
1452                }
1453
1454        return kobj_to_node_hstate(kobj, nidp);
1455}
1456
1457static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1458                                        struct kobj_attribute *attr, char *buf)
1459{
1460        struct hstate *h;
1461        unsigned long nr_huge_pages;
1462        int nid;
1463
1464        h = kobj_to_hstate(kobj, &nid);
1465        if (nid == NUMA_NO_NODE)
1466                nr_huge_pages = h->nr_huge_pages;
1467        else
1468                nr_huge_pages = h->nr_huge_pages_node[nid];
1469
1470        return sprintf(buf, "%lu\n", nr_huge_pages);
1471}
1472
1473static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1474                        struct kobject *kobj, struct kobj_attribute *attr,
1475                        const char *buf, size_t len)
1476{
1477        int err;
1478        int nid;
1479        unsigned long count;
1480        struct hstate *h;
1481        NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1482
1483        err = strict_strtoul(buf, 10, &count);
1484        if (err)
1485                goto out;
1486
1487        h = kobj_to_hstate(kobj, &nid);
1488        if (h->order >= MAX_ORDER) {
1489                err = -EINVAL;
1490                goto out;
1491        }
1492
1493        if (nid == NUMA_NO_NODE) {
1494                /*
1495                 * global hstate attribute
1496                 */
1497                if (!(obey_mempolicy &&
1498                                init_nodemask_of_mempolicy(nodes_allowed))) {
1499                        NODEMASK_FREE(nodes_allowed);
1500                        nodes_allowed = &node_states[N_HIGH_MEMORY];
1501                }
1502        } else if (nodes_allowed) {
1503                /*
1504                 * per node hstate attribute: adjust count to global,
1505                 * but restrict alloc/free to the specified node.
1506                 */
1507                count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1508                init_nodemask_of_node(nodes_allowed, nid);
1509        } else
1510                nodes_allowed = &node_states[N_HIGH_MEMORY];
1511
1512        h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1513
1514        if (nodes_allowed != &node_states[N_HIGH_MEMORY])
1515                NODEMASK_FREE(nodes_allowed);
1516
1517        return len;
1518out:
1519        NODEMASK_FREE(nodes_allowed);
1520        return err;
1521}
1522
1523static ssize_t nr_hugepages_show(struct kobject *kobj,
1524                                       struct kobj_attribute *attr, char *buf)
1525{
1526        return nr_hugepages_show_common(kobj, attr, buf);
1527}
1528
1529static ssize_t nr_hugepages_store(struct kobject *kobj,
1530               struct kobj_attribute *attr, const char *buf, size_t len)
1531{
1532        return nr_hugepages_store_common(false, kobj, attr, buf, len);
1533}
1534HSTATE_ATTR(nr_hugepages);
1535
1536#ifdef CONFIG_NUMA
1537
1538/*
1539 * hstate attribute for optionally mempolicy-based constraint on persistent
1540 * huge page alloc/free.
1541 */
1542static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1543                                       struct kobj_attribute *attr, char *buf)
1544{
1545        return nr_hugepages_show_common(kobj, attr, buf);
1546}
1547
1548static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1549               struct kobj_attribute *attr, const char *buf, size_t len)
1550{
1551        return nr_hugepages_store_common(true, kobj, attr, buf, len);
1552}
1553HSTATE_ATTR(nr_hugepages_mempolicy);
1554#endif
1555
1556
1557static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1558                                        struct kobj_attribute *attr, char *buf)
1559{
1560        struct hstate *h = kobj_to_hstate(kobj, NULL);
1561        return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1562}
1563
1564static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1565                struct kobj_attribute *attr, const char *buf, size_t count)
1566{
1567        int err;
1568        unsigned long input;
1569        struct hstate *h = kobj_to_hstate(kobj, NULL);
1570
1571        if (h->order >= MAX_ORDER)
1572                return -EINVAL;
1573
1574        err = strict_strtoul(buf, 10, &input);
1575        if (err)
1576                return err;
1577
1578        spin_lock(&hugetlb_lock);
1579        h->nr_overcommit_huge_pages = input;
1580        spin_unlock(&hugetlb_lock);
1581
1582        return count;
1583}
1584HSTATE_ATTR(nr_overcommit_hugepages);
1585
1586static ssize_t free_hugepages_show(struct kobject *kobj,
1587                                        struct kobj_attribute *attr, char *buf)
1588{
1589        struct hstate *h;
1590        unsigned long free_huge_pages;
1591        int nid;
1592
1593        h = kobj_to_hstate(kobj, &nid);
1594        if (nid == NUMA_NO_NODE)
1595                free_huge_pages = h->free_huge_pages;
1596        else
1597                free_huge_pages = h->free_huge_pages_node[nid];
1598
1599        return sprintf(buf, "%lu\n", free_huge_pages);
1600}
1601HSTATE_ATTR_RO(free_hugepages);
1602
1603static ssize_t resv_hugepages_show(struct kobject *kobj,
1604                                        struct kobj_attribute *attr, char *buf)
1605{
1606        struct hstate *h = kobj_to_hstate(kobj, NULL);
1607        return sprintf(buf, "%lu\n", h->resv_huge_pages);
1608}
1609HSTATE_ATTR_RO(resv_hugepages);
1610
1611static ssize_t surplus_hugepages_show(struct kobject *kobj,
1612                                        struct kobj_attribute *attr, char *buf)
1613{
1614        struct hstate *h;
1615        unsigned long surplus_huge_pages;
1616        int nid;
1617
1618        h = kobj_to_hstate(kobj, &nid);
1619        if (nid == NUMA_NO_NODE)
1620                surplus_huge_pages = h->surplus_huge_pages;
1621        else
1622                surplus_huge_pages = h->surplus_huge_pages_node[nid];
1623
1624        return sprintf(buf, "%lu\n", surplus_huge_pages);
1625}
1626HSTATE_ATTR_RO(surplus_hugepages);
1627
1628static struct attribute *hstate_attrs[] = {
1629        &nr_hugepages_attr.attr,
1630        &nr_overcommit_hugepages_attr.attr,
1631        &free_hugepages_attr.attr,
1632        &resv_hugepages_attr.attr,
1633        &surplus_hugepages_attr.attr,
1634#ifdef CONFIG_NUMA
1635        &nr_hugepages_mempolicy_attr.attr,
1636#endif
1637        NULL,
1638};
1639
1640static struct attribute_group hstate_attr_group = {
1641        .attrs = hstate_attrs,
1642};
1643
1644static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1645                                    struct kobject **hstate_kobjs,
1646                                    struct attribute_group *hstate_attr_group)
1647{
1648        int retval;
1649        int hi = h - hstates;
1650
1651        hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1652        if (!hstate_kobjs[hi])
1653                return -ENOMEM;
1654
1655        retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1656        if (retval)
1657                kobject_put(hstate_kobjs[hi]);
1658
1659        return retval;
1660}
1661
1662static void __init hugetlb_sysfs_init(void)
1663{
1664        struct hstate *h;
1665        int err;
1666
1667        hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1668        if (!hugepages_kobj)
1669                return;
1670
1671        for_each_hstate(h) {
1672                err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1673                                         hstate_kobjs, &hstate_attr_group);
1674                if (err)
1675                        printk(KERN_ERR "Hugetlb: Unable to add hstate %s",
1676                                                                h->name);
1677        }
1678}
1679
1680#ifdef CONFIG_NUMA
1681
1682/*
1683 * node_hstate/s - associate per node hstate attributes, via their kobjects,
1684 * with node devices in node_devices[] using a parallel array.  The array
1685 * index of a node device or _hstate == node id.
1686 * This is here to avoid any static dependency of the node device driver, in
1687 * the base kernel, on the hugetlb module.
1688 */
1689struct node_hstate {
1690        struct kobject          *hugepages_kobj;
1691        struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
1692};
1693struct node_hstate node_hstates[MAX_NUMNODES];
1694
1695/*
1696 * A subset of global hstate attributes for node devices
1697 */
1698static struct attribute *per_node_hstate_attrs[] = {
1699        &nr_hugepages_attr.attr,
1700        &free_hugepages_attr.attr,
1701        &surplus_hugepages_attr.attr,
1702        NULL,
1703};
1704
1705static struct attribute_group per_node_hstate_attr_group = {
1706        .attrs = per_node_hstate_attrs,
1707};
1708
1709/*
1710 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1711 * Returns node id via non-NULL nidp.
1712 */
1713static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1714{
1715        int nid;
1716
1717        for (nid = 0; nid < nr_node_ids; nid++) {
1718                struct node_hstate *nhs = &node_hstates[nid];
1719                int i;
1720                for (i = 0; i < HUGE_MAX_HSTATE; i++)
1721                        if (nhs->hstate_kobjs[i] == kobj) {
1722                                if (nidp)
1723                                        *nidp = nid;
1724                                return &hstates[i];
1725                        }
1726        }
1727
1728        BUG();
1729        return NULL;
1730}
1731
1732/*
1733 * Unregister hstate attributes from a single node device.
1734 * No-op if no hstate attributes attached.
1735 */
1736void hugetlb_unregister_node(struct node *node)
1737{
1738        struct hstate *h;
1739        struct node_hstate *nhs = &node_hstates[node->dev.id];
1740
1741        if (!nhs->hugepages_kobj)
1742                return;         /* no hstate attributes */
1743
1744        for_each_hstate(h)
1745                if (nhs->hstate_kobjs[h - hstates]) {
1746                        kobject_put(nhs->hstate_kobjs[h - hstates]);
1747                        nhs->hstate_kobjs[h - hstates] = NULL;
1748                }
1749
1750        kobject_put(nhs->hugepages_kobj);
1751        nhs->hugepages_kobj = NULL;
1752}
1753
1754/*
1755 * hugetlb module exit:  unregister hstate attributes from node devices
1756 * that have them.
1757 */
1758static void hugetlb_unregister_all_nodes(void)
1759{
1760        int nid;
1761
1762        /*
1763         * disable node device registrations.
1764         */
1765        register_hugetlbfs_with_node(NULL, NULL);
1766
1767        /*
1768         * remove hstate attributes from any nodes that have them.
1769         */
1770        for (nid = 0; nid < nr_node_ids; nid++)
1771                hugetlb_unregister_node(&node_devices[nid]);
1772}
1773
1774/*
1775 * Register hstate attributes for a single node device.
1776 * No-op if attributes already registered.
1777 */
1778void hugetlb_register_node(struct node *node)
1779{
1780        struct hstate *h;
1781        struct node_hstate *nhs = &node_hstates[node->dev.id];
1782        int err;
1783
1784        if (nhs->hugepages_kobj)
1785                return;         /* already allocated */
1786
1787        nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1788                                                        &node->dev.kobj);
1789        if (!nhs->hugepages_kobj)
1790                return;
1791
1792        for_each_hstate(h) {
1793                err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1794                                                nhs->hstate_kobjs,
1795                                                &per_node_hstate_attr_group);
1796                if (err) {
1797                        printk(KERN_ERR "Hugetlb: Unable to add hstate %s"
1798                                        " for node %d\n",
1799                                                h->name, node->dev.id);
1800                        hugetlb_unregister_node(node);
1801                        break;
1802                }
1803        }
1804}
1805
1806/*
1807 * hugetlb init time:  register hstate attributes for all registered node
1808 * devices of nodes that have memory.  All on-line nodes should have
1809 * registered their associated device by this time.
1810 */
1811static void hugetlb_register_all_nodes(void)
1812{
1813        int nid;
1814
1815        for_each_node_state(nid, N_HIGH_MEMORY) {
1816                struct node *node = &node_devices[nid];
1817                if (node->dev.id == nid)
1818                        hugetlb_register_node(node);
1819        }
1820
1821        /*
1822         * Let the node device driver know we're here so it can
1823         * [un]register hstate attributes on node hotplug.
1824         */
1825        register_hugetlbfs_with_node(hugetlb_register_node,
1826                                     hugetlb_unregister_node);
1827}
1828#else   /* !CONFIG_NUMA */
1829
1830static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1831{
1832        BUG();
1833        if (nidp)
1834                *nidp = -1;
1835        return NULL;
1836}
1837
1838static void hugetlb_unregister_all_nodes(void) { }
1839
1840static void hugetlb_register_all_nodes(void) { }
1841
1842#endif
1843
1844static void __exit hugetlb_exit(void)
1845{
1846        struct hstate *h;
1847
1848        hugetlb_unregister_all_nodes();
1849
1850        for_each_hstate(h) {
1851                kobject_put(hstate_kobjs[h - hstates]);
1852        }
1853
1854        kobject_put(hugepages_kobj);
1855}
1856module_exit(hugetlb_exit);
1857
1858static int __init hugetlb_init(void)
1859{
1860        /* Some platform decide whether they support huge pages at boot
1861         * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1862         * there is no such support
1863         */
1864        if (HPAGE_SHIFT == 0)
1865                return 0;
1866
1867        if (!size_to_hstate(default_hstate_size)) {
1868                default_hstate_size = HPAGE_SIZE;
1869                if (!size_to_hstate(default_hstate_size))
1870                        hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1871        }
1872        default_hstate_idx = size_to_hstate(default_hstate_size) - hstates;
1873        if (default_hstate_max_huge_pages)
1874                default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1875
1876        hugetlb_init_hstates();
1877
1878        gather_bootmem_prealloc();
1879
1880        report_hugepages();
1881
1882        hugetlb_sysfs_init();
1883
1884        hugetlb_register_all_nodes();
1885
1886        return 0;
1887}
1888module_init(hugetlb_init);
1889
1890/* Should be called on processing a hugepagesz=... option */
1891void __init hugetlb_add_hstate(unsigned order)
1892{
1893        struct hstate *h;
1894        unsigned long i;
1895
1896        if (size_to_hstate(PAGE_SIZE << order)) {
1897                printk(KERN_WARNING "hugepagesz= specified twice, ignoring\n");
1898                return;
1899        }
1900        BUG_ON(max_hstate >= HUGE_MAX_HSTATE);
1901        BUG_ON(order == 0);
1902        h = &hstates[max_hstate++];
1903        h->order = order;
1904        h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
1905        h->nr_huge_pages = 0;
1906        h->free_huge_pages = 0;
1907        for (i = 0; i < MAX_NUMNODES; ++i)
1908                INIT_LIST_HEAD(&h->hugepage_freelists[i]);
1909        h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
1910        h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
1911        snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
1912                                        huge_page_size(h)/1024);
1913
1914        parsed_hstate = h;
1915}
1916
1917static int __init hugetlb_nrpages_setup(char *s)
1918{
1919        unsigned long *mhp;
1920        static unsigned long *last_mhp;
1921
1922        /*
1923         * !max_hstate means we haven't parsed a hugepagesz= parameter yet,
1924         * so this hugepages= parameter goes to the "default hstate".
1925         */
1926        if (!max_hstate)
1927                mhp = &default_hstate_max_huge_pages;
1928        else
1929                mhp = &parsed_hstate->max_huge_pages;
1930
1931        if (mhp == last_mhp) {
1932                printk(KERN_WARNING "hugepages= specified twice without "
1933                        "interleaving hugepagesz=, ignoring\n");
1934                return 1;
1935        }
1936
1937        if (sscanf(s, "%lu", mhp) <= 0)
1938                *mhp = 0;
1939
1940        /*
1941         * Global state is always initialized later in hugetlb_init.
1942         * But we need to allocate >= MAX_ORDER hstates here early to still
1943         * use the bootmem allocator.
1944         */
1945        if (max_hstate && parsed_hstate->order >= MAX_ORDER)
1946                hugetlb_hstate_alloc_pages(parsed_hstate);
1947
1948        last_mhp = mhp;
1949
1950        return 1;
1951}
1952__setup("hugepages=", hugetlb_nrpages_setup);
1953
1954static int __init hugetlb_default_setup(char *s)
1955{
1956        default_hstate_size = memparse(s, &s);
1957        return 1;
1958}
1959__setup("default_hugepagesz=", hugetlb_default_setup);
1960
1961static unsigned int cpuset_mems_nr(unsigned int *array)
1962{
1963        int node;
1964        unsigned int nr = 0;
1965
1966        for_each_node_mask(node, cpuset_current_mems_allowed)
1967                nr += array[node];
1968
1969        return nr;
1970}
1971
1972#ifdef CONFIG_SYSCTL
1973static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
1974                         struct ctl_table *table, int write,
1975                         void __user *buffer, size_t *length, loff_t *ppos)
1976{
1977        struct hstate *h = &default_hstate;
1978        unsigned long tmp;
1979        int ret;
1980
1981        tmp = h->max_huge_pages;
1982
1983        if (write && h->order >= MAX_ORDER)
1984                return -EINVAL;
1985
1986        table->data = &tmp;
1987        table->maxlen = sizeof(unsigned long);
1988        ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
1989        if (ret)
1990                goto out;
1991
1992        if (write) {
1993                NODEMASK_ALLOC(nodemask_t, nodes_allowed,
1994                                                GFP_KERNEL | __GFP_NORETRY);
1995                if (!(obey_mempolicy &&
1996                               init_nodemask_of_mempolicy(nodes_allowed))) {
1997                        NODEMASK_FREE(nodes_allowed);
1998                        nodes_allowed = &node_states[N_HIGH_MEMORY];
1999                }
2000                h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2001
2002                if (nodes_allowed != &node_states[N_HIGH_MEMORY])
2003                        NODEMASK_FREE(nodes_allowed);
2004        }
2005out:
2006        return ret;
2007}
2008
2009int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2010                          void __user *buffer, size_t *length, loff_t *ppos)
2011{
2012
2013        return hugetlb_sysctl_handler_common(false, table, write,
2014                                                        buffer, length, ppos);
2015}
2016
2017#ifdef CONFIG_NUMA
2018int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2019                          void __user *buffer, size_t *length, loff_t *ppos)
2020{
2021        return hugetlb_sysctl_handler_common(true, table, write,
2022                                                        buffer, length, ppos);
2023}
2024#endif /* CONFIG_NUMA */
2025
2026int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
2027                        void __user *buffer,
2028                        size_t *length, loff_t *ppos)
2029{
2030        proc_dointvec(table, write, buffer, length, ppos);
2031        if (hugepages_treat_as_movable)
2032                htlb_alloc_mask = GFP_HIGHUSER_MOVABLE;
2033        else
2034                htlb_alloc_mask = GFP_HIGHUSER;
2035        return 0;
2036}
2037
2038int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2039                        void __user *buffer,
2040                        size_t *length, loff_t *ppos)
2041{
2042        struct hstate *h = &default_hstate;
2043        unsigned long tmp;
2044        int ret;
2045
2046        tmp = h->nr_overcommit_huge_pages;
2047
2048        if (write && h->order >= MAX_ORDER)
2049                return -EINVAL;
2050
2051        table->data = &tmp;
2052        table->maxlen = sizeof(unsigned long);
2053        ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2054        if (ret)
2055                goto out;
2056
2057        if (write) {
2058                spin_lock(&hugetlb_lock);
2059                h->nr_overcommit_huge_pages = tmp;
2060                spin_unlock(&hugetlb_lock);
2061        }
2062out:
2063        return ret;
2064}
2065
2066#endif /* CONFIG_SYSCTL */
2067
2068void hugetlb_report_meminfo(struct seq_file *m)
2069{
2070        struct hstate *h = &default_hstate;
2071        seq_printf(m,
2072                        "HugePages_Total:   %5lu\n"
2073                        "HugePages_Free:    %5lu\n"
2074                        "HugePages_Rsvd:    %5lu\n"
2075                        "HugePages_Surp:    %5lu\n"
2076                        "Hugepagesize:   %8lu kB\n",
2077                        h->nr_huge_pages,
2078                        h->free_huge_pages,
2079                        h->resv_huge_pages,
2080                        h->surplus_huge_pages,
2081                        1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2082}
2083
2084int hugetlb_report_node_meminfo(int nid, char *buf)
2085{
2086        struct hstate *h = &default_hstate;
2087        return sprintf(buf,
2088                "Node %d HugePages_Total: %5u\n"
2089                "Node %d HugePages_Free:  %5u\n"
2090                "Node %d HugePages_Surp:  %5u\n",
2091                nid, h->nr_huge_pages_node[nid],
2092                nid, h->free_huge_pages_node[nid],
2093                nid, h->surplus_huge_pages_node[nid]);
2094}
2095
2096/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2097unsigned long hugetlb_total_pages(void)
2098{
2099        struct hstate *h = &default_hstate;
2100        return h->nr_huge_pages * pages_per_huge_page(h);
2101}
2102
2103static int hugetlb_acct_memory(struct hstate *h, long delta)
2104{
2105        int ret = -ENOMEM;
2106
2107        spin_lock(&hugetlb_lock);
2108        /*
2109         * When cpuset is configured, it breaks the strict hugetlb page
2110         * reservation as the accounting is done on a global variable. Such
2111         * reservation is completely rubbish in the presence of cpuset because
2112         * the reservation is not checked against page availability for the
2113         * current cpuset. Application can still potentially OOM'ed by kernel
2114         * with lack of free htlb page in cpuset that the task is in.
2115         * Attempt to enforce strict accounting with cpuset is almost
2116         * impossible (or too ugly) because cpuset is too fluid that
2117         * task or memory node can be dynamically moved between cpusets.
2118         *
2119         * The change of semantics for shared hugetlb mapping with cpuset is
2120         * undesirable. However, in order to preserve some of the semantics,
2121         * we fall back to check against current free page availability as
2122         * a best attempt and hopefully to minimize the impact of changing
2123         * semantics that cpuset has.
2124         */
2125        if (delta > 0) {
2126                if (gather_surplus_pages(h, delta) < 0)
2127                        goto out;
2128
2129                if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2130                        return_unused_surplus_pages(h, delta);
2131                        goto out;
2132                }
2133        }
2134
2135        ret = 0;
2136        if (delta < 0)
2137                return_unused_surplus_pages(h, (unsigned long) -delta);
2138
2139out:
2140        spin_unlock(&hugetlb_lock);
2141        return ret;
2142}
2143
2144static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2145{
2146        struct resv_map *reservations = vma_resv_map(vma);
2147
2148        /*
2149         * This new VMA should share its siblings reservation map if present.
2150         * The VMA will only ever have a valid reservation map pointer where
2151         * it is being copied for another still existing VMA.  As that VMA
2152         * has a reference to the reservation map it cannot disappear until
2153         * after this open call completes.  It is therefore safe to take a
2154         * new reference here without additional locking.
2155         */
2156        if (reservations)
2157                kref_get(&reservations->refs);
2158}
2159
2160static void resv_map_put(struct vm_area_struct *vma)
2161{
2162        struct resv_map *reservations = vma_resv_map(vma);
2163
2164        if (!reservations)
2165                return;
2166        kref_put(&reservations->refs, resv_map_release);
2167}
2168
2169static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2170{
2171        struct hstate *h = hstate_vma(vma);
2172        struct resv_map *reservations = vma_resv_map(vma);
2173        struct hugepage_subpool *spool = subpool_vma(vma);
2174        unsigned long reserve;
2175        unsigned long start;
2176        unsigned long end;
2177
2178        if (reservations) {
2179                start = vma_hugecache_offset(h, vma, vma->vm_start);
2180                end = vma_hugecache_offset(h, vma, vma->vm_end);
2181
2182                reserve = (end - start) -
2183                        region_count(&reservations->regions, start, end);
2184
2185                resv_map_put(vma);
2186
2187                if (reserve) {
2188                        hugetlb_acct_memory(h, -reserve);
2189                        hugepage_subpool_put_pages(spool, reserve);
2190                }
2191        }
2192}
2193
2194/*
2195 * We cannot handle pagefaults against hugetlb pages at all.  They cause
2196 * handle_mm_fault() to try to instantiate regular-sized pages in the
2197 * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2198 * this far.
2199 */
2200static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2201{
2202        BUG();
2203        return 0;
2204}
2205
2206const struct vm_operations_struct hugetlb_vm_ops = {
2207        .fault = hugetlb_vm_op_fault,
2208        .open = hugetlb_vm_op_open,
2209        .close = hugetlb_vm_op_close,
2210};
2211
2212static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2213                                int writable)
2214{
2215        pte_t entry;
2216
2217        if (writable) {
2218                entry =
2219                    pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
2220        } else {
2221                entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
2222        }
2223        entry = pte_mkyoung(entry);
2224        entry = pte_mkhuge(entry);
2225        entry = arch_make_huge_pte(entry, vma, page, writable);
2226
2227        return entry;
2228}
2229
2230static void set_huge_ptep_writable(struct vm_area_struct *vma,
2231                                   unsigned long address, pte_t *ptep)
2232{
2233        pte_t entry;
2234
2235        entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
2236        if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2237                update_mmu_cache(vma, address, ptep);
2238}
2239
2240
2241int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2242                            struct vm_area_struct *vma)
2243{
2244        pte_t *src_pte, *dst_pte, entry;
2245        struct page *ptepage;
2246        unsigned long addr;
2247        int cow;
2248        struct hstate *h = hstate_vma(vma);
2249        unsigned long sz = huge_page_size(h);
2250
2251        cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2252
2253        for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2254                src_pte = huge_pte_offset(src, addr);
2255                if (!src_pte)
2256                        continue;
2257                dst_pte = huge_pte_alloc(dst, addr, sz);
2258                if (!dst_pte)
2259                        goto nomem;
2260
2261                /* If the pagetables are shared don't copy or take references */
2262                if (dst_pte == src_pte)
2263                        continue;
2264
2265                spin_lock(&dst->page_table_lock);
2266                spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
2267                if (!huge_pte_none(huge_ptep_get(src_pte))) {
2268                        if (cow)
2269                                huge_ptep_set_wrprotect(src, addr, src_pte);
2270                        entry = huge_ptep_get(src_pte);
2271                        ptepage = pte_page(entry);
2272                        get_page(ptepage);
2273                        page_dup_rmap(ptepage);
2274                        set_huge_pte_at(dst, addr, dst_pte, entry);
2275                }
2276                spin_unlock(&src->page_table_lock);
2277                spin_unlock(&dst->page_table_lock);
2278        }
2279        return 0;
2280
2281nomem:
2282        return -ENOMEM;
2283}
2284
2285static int is_hugetlb_entry_migration(pte_t pte)
2286{
2287        swp_entry_t swp;
2288
2289        if (huge_pte_none(pte) || pte_present(pte))
2290                return 0;
2291        swp = pte_to_swp_entry(pte);
2292        if (non_swap_entry(swp) && is_migration_entry(swp))
2293                return 1;
2294        else
2295                return 0;
2296}
2297
2298static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2299{
2300        swp_entry_t swp;
2301
2302        if (huge_pte_none(pte) || pte_present(pte))
2303                return 0;
2304        swp = pte_to_swp_entry(pte);
2305        if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2306                return 1;
2307        else
2308                return 0;
2309}
2310
2311void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2312                            unsigned long end, struct page *ref_page)
2313{
2314        struct mm_struct *mm = vma->vm_mm;
2315        unsigned long address;
2316        pte_t *ptep;
2317        pte_t pte;
2318        struct page *page;
2319        struct page *tmp;
2320        struct hstate *h = hstate_vma(vma);
2321        unsigned long sz = huge_page_size(h);
2322
2323        /*
2324         * A page gathering list, protected by per file i_mmap_mutex. The
2325         * lock is used to avoid list corruption from multiple unmapping
2326         * of the same page since we are using page->lru.
2327         */
2328        LIST_HEAD(page_list);
2329
2330        WARN_ON(!is_vm_hugetlb_page(vma));
2331        BUG_ON(start & ~huge_page_mask(h));
2332        BUG_ON(end & ~huge_page_mask(h));
2333
2334        mmu_notifier_invalidate_range_start(mm, start, end);
2335        spin_lock(&mm->page_table_lock);
2336        for (address = start; address < end; address += sz) {
2337                ptep = huge_pte_offset(mm, address);
2338                if (!ptep)
2339                        continue;
2340
2341                if (huge_pmd_unshare(mm, &address, ptep))
2342                        continue;
2343
2344                pte = huge_ptep_get(ptep);
2345                if (huge_pte_none(pte))
2346                        continue;
2347
2348                /*
2349                 * HWPoisoned hugepage is already unmapped and dropped reference
2350                 */
2351                if (unlikely(is_hugetlb_entry_hwpoisoned(pte)))
2352                        continue;
2353
2354                page = pte_page(pte);
2355                /*
2356                 * If a reference page is supplied, it is because a specific
2357                 * page is being unmapped, not a range. Ensure the page we
2358                 * are about to unmap is the actual page of interest.
2359                 */
2360                if (ref_page) {
2361                        if (page != ref_page)
2362                                continue;
2363
2364                        /*
2365                         * Mark the VMA as having unmapped its page so that
2366                         * future faults in this VMA will fail rather than
2367                         * looking like data was lost
2368                         */
2369                        set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2370                }
2371
2372                pte = huge_ptep_get_and_clear(mm, address, ptep);
2373                if (pte_dirty(pte))
2374                        set_page_dirty(page);
2375                list_add(&page->lru, &page_list);
2376
2377                /* Bail out after unmapping reference page if supplied */
2378                if (ref_page)
2379                        break;
2380        }
2381        flush_tlb_range(vma, start, end);
2382        spin_unlock(&mm->page_table_lock);
2383        mmu_notifier_invalidate_range_end(mm, start, end);
2384        list_for_each_entry_safe(page, tmp, &page_list, lru) {
2385                page_remove_rmap(page);
2386                list_del(&page->lru);
2387                put_page(page);
2388        }
2389}
2390
2391void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2392                          unsigned long end, struct page *ref_page)
2393{
2394        mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
2395        __unmap_hugepage_range(vma, start, end, ref_page);
2396        /*
2397         * Clear this flag so that x86's huge_pmd_share page_table_shareable
2398         * test will fail on a vma being torn down, and not grab a page table
2399         * on its way out.  We're lucky that the flag has such an appropriate
2400         * name, and can in fact be safely cleared here. We could clear it
2401         * before the __unmap_hugepage_range above, but all that's necessary
2402         * is to clear it before releasing the i_mmap_mutex below.
2403         *
2404         * This works because in the contexts this is called, the VMA is
2405         * going to be destroyed. It is not vunerable to madvise(DONTNEED)
2406         * because madvise is not supported on hugetlbfs. The same applies
2407         * for direct IO. unmap_hugepage_range() is only being called just
2408         * before free_pgtables() so clearing VM_MAYSHARE will not cause
2409         * surprises later.
2410         */
2411        vma->vm_flags &= ~VM_MAYSHARE;
2412        mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
2413}
2414
2415/*
2416 * This is called when the original mapper is failing to COW a MAP_PRIVATE
2417 * mappping it owns the reserve page for. The intention is to unmap the page
2418 * from other VMAs and let the children be SIGKILLed if they are faulting the
2419 * same region.
2420 */
2421static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2422                                struct page *page, unsigned long address)
2423{
2424        struct hstate *h = hstate_vma(vma);
2425        struct vm_area_struct *iter_vma;
2426        struct address_space *mapping;
2427        struct prio_tree_iter iter;
2428        pgoff_t pgoff;
2429
2430        /*
2431         * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2432         * from page cache lookup which is in HPAGE_SIZE units.
2433         */
2434        address = address & huge_page_mask(h);
2435        pgoff = vma_hugecache_offset(h, vma, address);
2436        mapping = vma->vm_file->f_dentry->d_inode->i_mapping;
2437
2438        /*
2439         * Take the mapping lock for the duration of the table walk. As
2440         * this mapping should be shared between all the VMAs,
2441         * __unmap_hugepage_range() is called as the lock is already held
2442         */
2443        mutex_lock(&mapping->i_mmap_mutex);
2444        vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
2445                /* Do not unmap the current VMA */
2446                if (iter_vma == vma)
2447                        continue;
2448
2449                /*
2450                 * Unmap the page from other VMAs without their own reserves.
2451                 * They get marked to be SIGKILLed if they fault in these
2452                 * areas. This is because a future no-page fault on this VMA
2453                 * could insert a zeroed page instead of the data existing
2454                 * from the time of fork. This would look like data corruption
2455                 */
2456                if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2457                        __unmap_hugepage_range(iter_vma,
2458                                address, address + huge_page_size(h),
2459                                page);
2460        }
2461        mutex_unlock(&mapping->i_mmap_mutex);
2462
2463        return 1;
2464}
2465
2466/*
2467 * Hugetlb_cow() should be called with page lock of the original hugepage held.
2468 * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2469 * cannot race with other handlers or page migration.
2470 * Keep the pte_same checks anyway to make transition from the mutex easier.
2471 */
2472static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2473                        unsigned long address, pte_t *ptep, pte_t pte,
2474                        struct page *pagecache_page)
2475{
2476        struct hstate *h = hstate_vma(vma);
2477        struct page *old_page, *new_page;
2478        int avoidcopy;
2479        int outside_reserve = 0;
2480
2481        old_page = pte_page(pte);
2482
2483retry_avoidcopy:
2484        /* If no-one else is actually using this page, avoid the copy
2485         * and just make the page writable */
2486        avoidcopy = (page_mapcount(old_page) == 1);
2487        if (avoidcopy) {
2488                if (PageAnon(old_page))
2489                        page_move_anon_rmap(old_page, vma, address);
2490                set_huge_ptep_writable(vma, address, ptep);
2491                return 0;
2492        }
2493
2494        /*
2495         * If the process that created a MAP_PRIVATE mapping is about to
2496         * perform a COW due to a shared page count, attempt to satisfy
2497         * the allocation without using the existing reserves. The pagecache
2498         * page is used to determine if the reserve at this address was
2499         * consumed or not. If reserves were used, a partial faulted mapping
2500         * at the time of fork() could consume its reserves on COW instead
2501         * of the full address range.
2502         */
2503        if (!(vma->vm_flags & VM_MAYSHARE) &&
2504                        is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2505                        old_page != pagecache_page)
2506                outside_reserve = 1;
2507
2508        page_cache_get(old_page);
2509
2510        /* Drop page_table_lock as buddy allocator may be called */
2511        spin_unlock(&mm->page_table_lock);
2512        new_page = alloc_huge_page(vma, address, outside_reserve);
2513
2514        if (IS_ERR(new_page)) {
2515                page_cache_release(old_page);
2516
2517                /*
2518                 * If a process owning a MAP_PRIVATE mapping fails to COW,
2519                 * it is due to references held by a child and an insufficient
2520                 * huge page pool. To guarantee the original mappers
2521                 * reliability, unmap the page from child processes. The child
2522                 * may get SIGKILLed if it later faults.
2523                 */
2524                if (outside_reserve) {
2525                        BUG_ON(huge_pte_none(pte));
2526                        if (unmap_ref_private(mm, vma, old_page, address)) {
2527                                BUG_ON(huge_pte_none(pte));
2528                                spin_lock(&mm->page_table_lock);
2529                                ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2530                                if (likely(pte_same(huge_ptep_get(ptep), pte)))
2531                                        goto retry_avoidcopy;
2532                                /*
2533                                 * race occurs while re-acquiring page_table_lock, and
2534                                 * our job is done.
2535                                 */
2536                                return 0;
2537                        }
2538                        WARN_ON_ONCE(1);
2539                }
2540
2541                /* Caller expects lock to be held */
2542                spin_lock(&mm->page_table_lock);
2543                return -PTR_ERR(new_page);
2544        }
2545
2546        /*
2547         * When the original hugepage is shared one, it does not have
2548         * anon_vma prepared.
2549         */
2550        if (unlikely(anon_vma_prepare(vma))) {
2551                page_cache_release(new_page);
2552                page_cache_release(old_page);
2553                /* Caller expects lock to be held */
2554                spin_lock(&mm->page_table_lock);
2555                return VM_FAULT_OOM;
2556        }
2557
2558        copy_user_huge_page(new_page, old_page, address, vma,
2559                            pages_per_huge_page(h));
2560        __SetPageUptodate(new_page);
2561
2562        /*
2563         * Retake the page_table_lock to check for racing updates
2564         * before the page tables are altered
2565         */
2566        spin_lock(&mm->page_table_lock);
2567        ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2568        if (likely(pte_same(huge_ptep_get(ptep), pte))) {
2569                /* Break COW */
2570                mmu_notifier_invalidate_range_start(mm,
2571                        address & huge_page_mask(h),
2572                        (address & huge_page_mask(h)) + huge_page_size(h));
2573                huge_ptep_clear_flush(vma, address, ptep);
2574                set_huge_pte_at(mm, address, ptep,
2575                                make_huge_pte(vma, new_page, 1));
2576                page_remove_rmap(old_page);
2577                hugepage_add_new_anon_rmap(new_page, vma, address);
2578                /* Make the old page be freed below */
2579                new_page = old_page;
2580                mmu_notifier_invalidate_range_end(mm,
2581                        address & huge_page_mask(h),
2582                        (address & huge_page_mask(h)) + huge_page_size(h));
2583        }
2584        page_cache_release(new_page);
2585        page_cache_release(old_page);
2586        return 0;
2587}
2588
2589/* Return the pagecache page at a given address within a VMA */
2590static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2591                        struct vm_area_struct *vma, unsigned long address)
2592{
2593        struct address_space *mapping;
2594        pgoff_t idx;
2595
2596        mapping = vma->vm_file->f_mapping;
2597        idx = vma_hugecache_offset(h, vma, address);
2598
2599        return find_lock_page(mapping, idx);
2600}
2601
2602/*
2603 * Return whether there is a pagecache page to back given address within VMA.
2604 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2605 */
2606static bool hugetlbfs_pagecache_present(struct hstate *h,
2607                        struct vm_area_struct *vma, unsigned long address)
2608{
2609        struct address_space *mapping;
2610        pgoff_t idx;
2611        struct page *page;
2612
2613        mapping = vma->vm_file->f_mapping;
2614        idx = vma_hugecache_offset(h, vma, address);
2615
2616        page = find_get_page(mapping, idx);
2617        if (page)
2618                put_page(page);
2619        return page != NULL;
2620}
2621
2622static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2623                        unsigned long address, pte_t *ptep, unsigned int flags)
2624{
2625        struct hstate *h = hstate_vma(vma);
2626        int ret = VM_FAULT_SIGBUS;
2627        int anon_rmap = 0;
2628        pgoff_t idx;
2629        unsigned long size;
2630        struct page *page;
2631        struct address_space *mapping;
2632        pte_t new_pte;
2633
2634        /*
2635         * Currently, we are forced to kill the process in the event the
2636         * original mapper has unmapped pages from the child due to a failed
2637         * COW. Warn that such a situation has occurred as it may not be obvious
2638         */
2639        if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2640                printk(KERN_WARNING
2641                        "PID %d killed due to inadequate hugepage pool\n",
2642                        current->pid);
2643                return ret;
2644        }
2645
2646        mapping = vma->vm_file->f_mapping;
2647        idx = vma_hugecache_offset(h, vma, address);
2648
2649        /*
2650         * Use page lock to guard against racing truncation
2651         * before we get page_table_lock.
2652         */
2653retry:
2654        page = find_lock_page(mapping, idx);
2655        if (!page) {
2656                size = i_size_read(mapping->host) >> huge_page_shift(h);
2657                if (idx >= size)
2658                        goto out;
2659                page = alloc_huge_page(vma, address, 0);
2660                if (IS_ERR(page)) {
2661                        ret = -PTR_ERR(page);
2662                        goto out;
2663                }
2664                clear_huge_page(page, address, pages_per_huge_page(h));
2665                __SetPageUptodate(page);
2666
2667                if (vma->vm_flags & VM_MAYSHARE) {
2668                        int err;
2669                        struct inode *inode = mapping->host;
2670
2671                        err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2672                        if (err) {
2673                                put_page(page);
2674                                if (err == -EEXIST)
2675                                        goto retry;
2676                                goto out;
2677                        }
2678
2679                        spin_lock(&inode->i_lock);
2680                        inode->i_blocks += blocks_per_huge_page(h);
2681                        spin_unlock(&inode->i_lock);
2682                } else {
2683                        lock_page(page);
2684                        if (unlikely(anon_vma_prepare(vma))) {
2685                                ret = VM_FAULT_OOM;
2686                                goto backout_unlocked;
2687                        }
2688                        anon_rmap = 1;
2689                }
2690        } else {
2691                /*
2692                 * If memory error occurs between mmap() and fault, some process
2693                 * don't have hwpoisoned swap entry for errored virtual address.
2694                 * So we need to block hugepage fault by PG_hwpoison bit check.
2695                 */
2696                if (unlikely(PageHWPoison(page))) {
2697                        ret = VM_FAULT_HWPOISON |
2698                              VM_FAULT_SET_HINDEX(h - hstates);
2699                        goto backout_unlocked;
2700                }
2701        }
2702
2703        /*
2704         * If we are going to COW a private mapping later, we examine the
2705         * pending reservations for this page now. This will ensure that
2706         * any allocations necessary to record that reservation occur outside
2707         * the spinlock.
2708         */
2709        if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2710                if (vma_needs_reservation(h, vma, address) < 0) {
2711                        ret = VM_FAULT_OOM;
2712                        goto backout_unlocked;
2713                }
2714
2715        spin_lock(&mm->page_table_lock);
2716        size = i_size_read(mapping->host) >> huge_page_shift(h);
2717        if (idx >= size)
2718                goto backout;
2719
2720        ret = 0;
2721        if (!huge_pte_none(huge_ptep_get(ptep)))
2722                goto backout;
2723
2724        if (anon_rmap)
2725                hugepage_add_new_anon_rmap(page, vma, address);
2726        else
2727                page_dup_rmap(page);
2728        new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2729                                && (vma->vm_flags & VM_SHARED)));
2730        set_huge_pte_at(mm, address, ptep, new_pte);
2731
2732        if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2733                /* Optimization, do the COW without a second fault */
2734                ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
2735        }
2736
2737        spin_unlock(&mm->page_table_lock);
2738        unlock_page(page);
2739out:
2740        return ret;
2741
2742backout:
2743        spin_unlock(&mm->page_table_lock);
2744backout_unlocked:
2745        unlock_page(page);
2746        put_page(page);
2747        goto out;
2748}
2749
2750int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2751                        unsigned long address, unsigned int flags)
2752{
2753        pte_t *ptep;
2754        pte_t entry;
2755        int ret;
2756        struct page *page = NULL;
2757        struct page *pagecache_page = NULL;
2758        static DEFINE_MUTEX(hugetlb_instantiation_mutex);
2759        struct hstate *h = hstate_vma(vma);
2760
2761        address &= huge_page_mask(h);
2762
2763        ptep = huge_pte_offset(mm, address);
2764        if (ptep) {
2765                entry = huge_ptep_get(ptep);
2766                if (unlikely(is_hugetlb_entry_migration(entry))) {
2767                        migration_entry_wait(mm, (pmd_t *)ptep, address);
2768                        return 0;
2769                } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
2770                        return VM_FAULT_HWPOISON_LARGE |
2771                               VM_FAULT_SET_HINDEX(h - hstates);
2772        }
2773
2774        ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2775        if (!ptep)
2776                return VM_FAULT_OOM;
2777
2778        /*
2779         * Serialize hugepage allocation and instantiation, so that we don't
2780         * get spurious allocation failures if two CPUs race to instantiate
2781         * the same page in the page cache.
2782         */
2783        mutex_lock(&hugetlb_instantiation_mutex);
2784        entry = huge_ptep_get(ptep);
2785        if (huge_pte_none(entry)) {
2786                ret = hugetlb_no_page(mm, vma, address, ptep, flags);
2787                goto out_mutex;
2788        }
2789
2790        ret = 0;
2791
2792        /*
2793         * If we are going to COW the mapping later, we examine the pending
2794         * reservations for this page now. This will ensure that any
2795         * allocations necessary to record that reservation occur outside the
2796         * spinlock. For private mappings, we also lookup the pagecache
2797         * page now as it is used to determine if a reservation has been
2798         * consumed.
2799         */
2800        if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
2801                if (vma_needs_reservation(h, vma, address) < 0) {
2802                        ret = VM_FAULT_OOM;
2803                        goto out_mutex;
2804                }
2805
2806                if (!(vma->vm_flags & VM_MAYSHARE))
2807                        pagecache_page = hugetlbfs_pagecache_page(h,
2808                                                                vma, address);
2809        }
2810
2811        /*
2812         * hugetlb_cow() requires page locks of pte_page(entry) and
2813         * pagecache_page, so here we need take the former one
2814         * when page != pagecache_page or !pagecache_page.
2815         * Note that locking order is always pagecache_page -> page,
2816         * so no worry about deadlock.
2817         */
2818        page = pte_page(entry);
2819        get_page(page);
2820        if (page != pagecache_page)
2821                lock_page(page);
2822
2823        spin_lock(&mm->page_table_lock);
2824        /* Check for a racing update before calling hugetlb_cow */
2825        if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2826                goto out_page_table_lock;
2827
2828
2829        if (flags & FAULT_FLAG_WRITE) {
2830                if (!pte_write(entry)) {
2831                        ret = hugetlb_cow(mm, vma, address, ptep, entry,
2832                                                        pagecache_page);
2833                        goto out_page_table_lock;
2834                }
2835                entry = pte_mkdirty(entry);
2836        }
2837        entry = pte_mkyoung(entry);
2838        if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2839                                                flags & FAULT_FLAG_WRITE))
2840                update_mmu_cache(vma, address, ptep);
2841
2842out_page_table_lock:
2843        spin_unlock(&mm->page_table_lock);
2844
2845        if (pagecache_page) {
2846                unlock_page(pagecache_page);
2847                put_page(pagecache_page);
2848        }
2849        if (page != pagecache_page)
2850                unlock_page(page);
2851        put_page(page);
2852
2853out_mutex:
2854        mutex_unlock(&hugetlb_instantiation_mutex);
2855
2856        return ret;
2857}
2858
2859/* Can be overriden by architectures */
2860__attribute__((weak)) struct page *
2861follow_huge_pud(struct mm_struct *mm, unsigned long address,
2862               pud_t *pud, int write)
2863{
2864        BUG();
2865        return NULL;
2866}
2867
2868int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
2869                        struct page **pages, struct vm_area_struct **vmas,
2870                        unsigned long *position, int *length, int i,
2871                        unsigned int flags)
2872{
2873        unsigned long pfn_offset;
2874        unsigned long vaddr = *position;
2875        int remainder = *length;
2876        struct hstate *h = hstate_vma(vma);
2877
2878        spin_lock(&mm->page_table_lock);
2879        while (vaddr < vma->vm_end && remainder) {
2880                pte_t *pte;
2881                int absent;
2882                struct page *page;
2883
2884                /*
2885                 * Some archs (sparc64, sh*) have multiple pte_ts to
2886                 * each hugepage.  We have to make sure we get the
2887                 * first, for the page indexing below to work.
2888                 */
2889                pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
2890                absent = !pte || huge_pte_none(huge_ptep_get(pte));
2891
2892                /*
2893                 * When coredumping, it suits get_dump_page if we just return
2894                 * an error where there's an empty slot with no huge pagecache
2895                 * to back it.  This way, we avoid allocating a hugepage, and
2896                 * the sparse dumpfile avoids allocating disk blocks, but its
2897                 * huge holes still show up with zeroes where they need to be.
2898                 */
2899                if (absent && (flags & FOLL_DUMP) &&
2900                    !hugetlbfs_pagecache_present(h, vma, vaddr)) {
2901                        remainder = 0;
2902                        break;
2903                }
2904
2905                if (absent ||
2906                    ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
2907                        int ret;
2908
2909                        spin_unlock(&mm->page_table_lock);
2910                        ret = hugetlb_fault(mm, vma, vaddr,
2911                                (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
2912                        spin_lock(&mm->page_table_lock);
2913                        if (!(ret & VM_FAULT_ERROR))
2914                                continue;
2915
2916                        remainder = 0;
2917                        break;
2918                }
2919
2920                pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
2921                page = pte_page(huge_ptep_get(pte));
2922same_page:
2923                if (pages) {
2924                        pages[i] = mem_map_offset(page, pfn_offset);
2925                        get_page(pages[i]);
2926                }
2927
2928                if (vmas)
2929                        vmas[i] = vma;
2930
2931                vaddr += PAGE_SIZE;
2932                ++pfn_offset;
2933                --remainder;
2934                ++i;
2935                if (vaddr < vma->vm_end && remainder &&
2936                                pfn_offset < pages_per_huge_page(h)) {
2937                        /*
2938                         * We use pfn_offset to avoid touching the pageframes
2939                         * of this compound page.
2940                         */
2941                        goto same_page;
2942                }
2943        }
2944        spin_unlock(&mm->page_table_lock);
2945        *length = remainder;
2946        *position = vaddr;
2947
2948        return i ? i : -EFAULT;
2949}
2950
2951void hugetlb_change_protection(struct vm_area_struct *vma,
2952                unsigned long address, unsigned long end, pgprot_t newprot)
2953{
2954        struct mm_struct *mm = vma->vm_mm;
2955        unsigned long start = address;
2956        pte_t *ptep;
2957        pte_t pte;
2958        struct hstate *h = hstate_vma(vma);
2959
2960        BUG_ON(address >= end);
2961        flush_cache_range(vma, address, end);
2962
2963        mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
2964        spin_lock(&mm->page_table_lock);
2965        for (; address < end; address += huge_page_size(h)) {
2966                ptep = huge_pte_offset(mm, address);
2967                if (!ptep)
2968                        continue;
2969                if (huge_pmd_unshare(mm, &address, ptep))
2970                        continue;
2971                if (!huge_pte_none(huge_ptep_get(ptep))) {
2972                        pte = huge_ptep_get_and_clear(mm, address, ptep);
2973                        pte = pte_mkhuge(pte_modify(pte, newprot));
2974                        set_huge_pte_at(mm, address, ptep, pte);
2975                }
2976        }
2977        spin_unlock(&mm->page_table_lock);
2978        /*
2979         * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
2980         * may have cleared our pud entry and done put_page on the page table:
2981         * once we release i_mmap_mutex, another task can do the final put_page
2982         * and that page table be reused and filled with junk.
2983         */
2984        flush_tlb_range(vma, start, end);
2985        mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
2986}
2987
2988int hugetlb_reserve_pages(struct inode *inode,
2989                                        long from, long to,
2990                                        struct vm_area_struct *vma,
2991                                        vm_flags_t vm_flags)
2992{
2993        long ret, chg;
2994        struct hstate *h = hstate_inode(inode);
2995        struct hugepage_subpool *spool = subpool_inode(inode);
2996
2997        /*
2998         * Only apply hugepage reservation if asked. At fault time, an
2999         * attempt will be made for VM_NORESERVE to allocate a page
3000         * without using reserves
3001         */
3002        if (vm_flags & VM_NORESERVE)
3003                return 0;
3004
3005        /*
3006         * Shared mappings base their reservation on the number of pages that
3007         * are already allocated on behalf of the file. Private mappings need
3008         * to reserve the full area even if read-only as mprotect() may be
3009         * called to make the mapping read-write. Assume !vma is a shm mapping
3010         */
3011        if (!vma || vma->vm_flags & VM_MAYSHARE)
3012                chg = region_chg(&inode->i_mapping->private_list, from, to);
3013        else {
3014                struct resv_map *resv_map = resv_map_alloc();
3015                if (!resv_map)
3016                        return -ENOMEM;
3017
3018                chg = to - from;
3019
3020                set_vma_resv_map(vma, resv_map);
3021                set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3022        }
3023
3024        if (chg < 0) {
3025                ret = chg;
3026                goto out_err;
3027        }
3028
3029        /* There must be enough pages in the subpool for the mapping */
3030        if (hugepage_subpool_get_pages(spool, chg)) {
3031                ret = -ENOSPC;
3032                goto out_err;
3033        }
3034
3035        /*
3036         * Check enough hugepages are available for the reservation.
3037         * Hand the pages back to the subpool if there are not
3038         */
3039        ret = hugetlb_acct_memory(h, chg);
3040        if (ret < 0) {
3041                hugepage_subpool_put_pages(spool, chg);
3042                goto out_err;
3043        }
3044
3045        /*
3046         * Account for the reservations made. Shared mappings record regions
3047         * that have reservations as they are shared by multiple VMAs.
3048         * When the last VMA disappears, the region map says how much
3049         * the reservation was and the page cache tells how much of
3050         * the reservation was consumed. Private mappings are per-VMA and
3051         * only the consumed reservations are tracked. When the VMA
3052         * disappears, the original reservation is the VMA size and the
3053         * consumed reservations are stored in the map. Hence, nothing
3054         * else has to be done for private mappings here
3055         */
3056        if (!vma || vma->vm_flags & VM_MAYSHARE)
3057                region_add(&inode->i_mapping->private_list, from, to);
3058        return 0;
3059out_err:
3060        if (vma)
3061                resv_map_put(vma);
3062        return ret;
3063}
3064
3065void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3066{
3067        struct hstate *h = hstate_inode(inode);
3068        long chg = region_truncate(&inode->i_mapping->private_list, offset);
3069        struct hugepage_subpool *spool = subpool_inode(inode);
3070
3071        spin_lock(&inode->i_lock);
3072        inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3073        spin_unlock(&inode->i_lock);
3074
3075        hugepage_subpool_put_pages(spool, (chg - freed));
3076        hugetlb_acct_memory(h, -(chg - freed));
3077}
3078
3079#ifdef CONFIG_MEMORY_FAILURE
3080
3081/* Should be called in hugetlb_lock */
3082static int is_hugepage_on_freelist(struct page *hpage)
3083{
3084        struct page *page;
3085        struct page *tmp;
3086        struct hstate *h = page_hstate(hpage);
3087        int nid = page_to_nid(hpage);
3088
3089        list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3090                if (page == hpage)
3091                        return 1;
3092        return 0;
3093}
3094
3095/*
3096 * This function is called from memory failure code.
3097 * Assume the caller holds page lock of the head page.
3098 */
3099int dequeue_hwpoisoned_huge_page(struct page *hpage)
3100{
3101        struct hstate *h = page_hstate(hpage);
3102        int nid = page_to_nid(hpage);
3103        int ret = -EBUSY;
3104
3105        spin_lock(&hugetlb_lock);
3106        if (is_hugepage_on_freelist(hpage)) {
3107                list_del(&hpage->lru);
3108                set_page_refcounted(hpage);
3109                h->free_huge_pages--;
3110                h->free_huge_pages_node[nid]--;
3111                ret = 0;
3112        }
3113        spin_unlock(&hugetlb_lock);
3114        return ret;
3115}
3116#endif
3117
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.