linux/arch/x86/mm/pat/set_memory.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright 2002 Andi Kleen, SuSE Labs.
   4 * Thanks to Ben LaHaise for precious feedback.
   5 */
   6#include <linux/highmem.h>
   7#include <linux/memblock.h>
   8#include <linux/sched.h>
   9#include <linux/mm.h>
  10#include <linux/interrupt.h>
  11#include <linux/seq_file.h>
  12#include <linux/debugfs.h>
  13#include <linux/pfn.h>
  14#include <linux/percpu.h>
  15#include <linux/gfp.h>
  16#include <linux/pci.h>
  17#include <linux/vmalloc.h>
  18#include <linux/libnvdimm.h>
  19#include <linux/vmstat.h>
  20#include <linux/kernel.h>
  21
  22#include <asm/e820/api.h>
  23#include <asm/processor.h>
  24#include <asm/tlbflush.h>
  25#include <asm/sections.h>
  26#include <asm/setup.h>
  27#include <linux/uaccess.h>
  28#include <asm/pgalloc.h>
  29#include <asm/proto.h>
  30#include <asm/memtype.h>
  31#include <asm/set_memory.h>
  32
  33#include "../mm_internal.h"
  34
  35/*
  36 * The current flushing context - we pass it instead of 5 arguments:
  37 */
  38struct cpa_data {
  39        unsigned long   *vaddr;
  40        pgd_t           *pgd;
  41        pgprot_t        mask_set;
  42        pgprot_t        mask_clr;
  43        unsigned long   numpages;
  44        unsigned long   curpage;
  45        unsigned long   pfn;
  46        unsigned int    flags;
  47        unsigned int    force_split             : 1,
  48                        force_static_prot       : 1,
  49                        force_flush_all         : 1;
  50        struct page     **pages;
  51};
  52
  53enum cpa_warn {
  54        CPA_CONFLICT,
  55        CPA_PROTECT,
  56        CPA_DETECT,
  57};
  58
  59static const int cpa_warn_level = CPA_PROTECT;
  60
  61/*
  62 * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
  63 * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
  64 * entries change the page attribute in parallel to some other cpu
  65 * splitting a large page entry along with changing the attribute.
  66 */
  67static DEFINE_SPINLOCK(cpa_lock);
  68
  69#define CPA_FLUSHTLB 1
  70#define CPA_ARRAY 2
  71#define CPA_PAGES_ARRAY 4
  72#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */
  73
  74static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm)
  75{
  76        return __pgprot(cachemode2protval(pcm));
  77}
  78
  79#ifdef CONFIG_PROC_FS
  80static unsigned long direct_pages_count[PG_LEVEL_NUM];
  81
  82void update_page_count(int level, unsigned long pages)
  83{
  84        /* Protect against CPA */
  85        spin_lock(&pgd_lock);
  86        direct_pages_count[level] += pages;
  87        spin_unlock(&pgd_lock);
  88}
  89
  90static void split_page_count(int level)
  91{
  92        if (direct_pages_count[level] == 0)
  93                return;
  94
  95        direct_pages_count[level]--;
  96        if (system_state == SYSTEM_RUNNING) {
  97                if (level == PG_LEVEL_2M)
  98                        count_vm_event(DIRECT_MAP_LEVEL2_SPLIT);
  99                else if (level == PG_LEVEL_1G)
 100                        count_vm_event(DIRECT_MAP_LEVEL3_SPLIT);
 101        }
 102        direct_pages_count[level - 1] += PTRS_PER_PTE;
 103}
 104
 105void arch_report_meminfo(struct seq_file *m)
 106{
 107        seq_printf(m, "DirectMap4k:    %8lu kB\n",
 108                        direct_pages_count[PG_LEVEL_4K] << 2);
 109#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 110        seq_printf(m, "DirectMap2M:    %8lu kB\n",
 111                        direct_pages_count[PG_LEVEL_2M] << 11);
 112#else
 113        seq_printf(m, "DirectMap4M:    %8lu kB\n",
 114                        direct_pages_count[PG_LEVEL_2M] << 12);
 115#endif
 116        if (direct_gbpages)
 117                seq_printf(m, "DirectMap1G:    %8lu kB\n",
 118                        direct_pages_count[PG_LEVEL_1G] << 20);
 119}
 120#else
 121static inline void split_page_count(int level) { }
 122#endif
 123
 124#ifdef CONFIG_X86_CPA_STATISTICS
 125
 126static unsigned long cpa_1g_checked;
 127static unsigned long cpa_1g_sameprot;
 128static unsigned long cpa_1g_preserved;
 129static unsigned long cpa_2m_checked;
 130static unsigned long cpa_2m_sameprot;
 131static unsigned long cpa_2m_preserved;
 132static unsigned long cpa_4k_install;
 133
 134static inline void cpa_inc_1g_checked(void)
 135{
 136        cpa_1g_checked++;
 137}
 138
 139static inline void cpa_inc_2m_checked(void)
 140{
 141        cpa_2m_checked++;
 142}
 143
 144static inline void cpa_inc_4k_install(void)
 145{
 146        data_race(cpa_4k_install++);
 147}
 148
 149static inline void cpa_inc_lp_sameprot(int level)
 150{
 151        if (level == PG_LEVEL_1G)
 152                cpa_1g_sameprot++;
 153        else
 154                cpa_2m_sameprot++;
 155}
 156
 157static inline void cpa_inc_lp_preserved(int level)
 158{
 159        if (level == PG_LEVEL_1G)
 160                cpa_1g_preserved++;
 161        else
 162                cpa_2m_preserved++;
 163}
 164
 165static int cpastats_show(struct seq_file *m, void *p)
 166{
 167        seq_printf(m, "1G pages checked:     %16lu\n", cpa_1g_checked);
 168        seq_printf(m, "1G pages sameprot:    %16lu\n", cpa_1g_sameprot);
 169        seq_printf(m, "1G pages preserved:   %16lu\n", cpa_1g_preserved);
 170        seq_printf(m, "2M pages checked:     %16lu\n", cpa_2m_checked);
 171        seq_printf(m, "2M pages sameprot:    %16lu\n", cpa_2m_sameprot);
 172        seq_printf(m, "2M pages preserved:   %16lu\n", cpa_2m_preserved);
 173        seq_printf(m, "4K pages set-checked: %16lu\n", cpa_4k_install);
 174        return 0;
 175}
 176
 177static int cpastats_open(struct inode *inode, struct file *file)
 178{
 179        return single_open(file, cpastats_show, NULL);
 180}
 181
 182static const struct file_operations cpastats_fops = {
 183        .open           = cpastats_open,
 184        .read           = seq_read,
 185        .llseek         = seq_lseek,
 186        .release        = single_release,
 187};
 188
 189static int __init cpa_stats_init(void)
 190{
 191        debugfs_create_file("cpa_stats", S_IRUSR, arch_debugfs_dir, NULL,
 192                            &cpastats_fops);
 193        return 0;
 194}
 195late_initcall(cpa_stats_init);
 196#else
 197static inline void cpa_inc_1g_checked(void) { }
 198static inline void cpa_inc_2m_checked(void) { }
 199static inline void cpa_inc_4k_install(void) { }
 200static inline void cpa_inc_lp_sameprot(int level) { }
 201static inline void cpa_inc_lp_preserved(int level) { }
 202#endif
 203
 204
 205static inline int
 206within(unsigned long addr, unsigned long start, unsigned long end)
 207{
 208        return addr >= start && addr < end;
 209}
 210
 211static inline int
 212within_inclusive(unsigned long addr, unsigned long start, unsigned long end)
 213{
 214        return addr >= start && addr <= end;
 215}
 216
 217#ifdef CONFIG_X86_64
 218
 219static inline unsigned long highmap_start_pfn(void)
 220{
 221        return __pa_symbol(_text) >> PAGE_SHIFT;
 222}
 223
 224static inline unsigned long highmap_end_pfn(void)
 225{
 226        /* Do not reference physical address outside the kernel. */
 227        return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
 228}
 229
 230static bool __cpa_pfn_in_highmap(unsigned long pfn)
 231{
 232        /*
 233         * Kernel text has an alias mapping at a high address, known
 234         * here as "highmap".
 235         */
 236        return within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn());
 237}
 238
 239#else
 240
 241static bool __cpa_pfn_in_highmap(unsigned long pfn)
 242{
 243        /* There is no highmap on 32-bit */
 244        return false;
 245}
 246
 247#endif
 248
 249/*
 250 * See set_mce_nospec().
 251 *
 252 * Machine check recovery code needs to change cache mode of poisoned pages to
 253 * UC to avoid speculative access logging another error. But passing the
 254 * address of the 1:1 mapping to set_memory_uc() is a fine way to encourage a
 255 * speculative access. So we cheat and flip the top bit of the address. This
 256 * works fine for the code that updates the page tables. But at the end of the
 257 * process we need to flush the TLB and cache and the non-canonical address
 258 * causes a #GP fault when used by the INVLPG and CLFLUSH instructions.
 259 *
 260 * But in the common case we already have a canonical address. This code
 261 * will fix the top bit if needed and is a no-op otherwise.
 262 */
 263static inline unsigned long fix_addr(unsigned long addr)
 264{
 265#ifdef CONFIG_X86_64
 266        return (long)(addr << 1) >> 1;
 267#else
 268        return addr;
 269#endif
 270}
 271
 272static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx)
 273{
 274        if (cpa->flags & CPA_PAGES_ARRAY) {
 275                struct page *page = cpa->pages[idx];
 276
 277                if (unlikely(PageHighMem(page)))
 278                        return 0;
 279
 280                return (unsigned long)page_address(page);
 281        }
 282
 283        if (cpa->flags & CPA_ARRAY)
 284                return cpa->vaddr[idx];
 285
 286        return *cpa->vaddr + idx * PAGE_SIZE;
 287}
 288
 289/*
 290 * Flushing functions
 291 */
 292
 293static void clflush_cache_range_opt(void *vaddr, unsigned int size)
 294{
 295        const unsigned long clflush_size = boot_cpu_data.x86_clflush_size;
 296        void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1));
 297        void *vend = vaddr + size;
 298
 299        if (p >= vend)
 300                return;
 301
 302        for (; p < vend; p += clflush_size)
 303                clflushopt(p);
 304}
 305
 306/**
 307 * clflush_cache_range - flush a cache range with clflush
 308 * @vaddr:      virtual start address
 309 * @size:       number of bytes to flush
 310 *
 311 * CLFLUSHOPT is an unordered instruction which needs fencing with MFENCE or
 312 * SFENCE to avoid ordering issues.
 313 */
 314void clflush_cache_range(void *vaddr, unsigned int size)
 315{
 316        mb();
 317        clflush_cache_range_opt(vaddr, size);
 318        mb();
 319}
 320EXPORT_SYMBOL_GPL(clflush_cache_range);
 321
 322#ifdef CONFIG_ARCH_HAS_PMEM_API
 323void arch_invalidate_pmem(void *addr, size_t size)
 324{
 325        clflush_cache_range(addr, size);
 326}
 327EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
 328#endif
 329
 330static void __cpa_flush_all(void *arg)
 331{
 332        unsigned long cache = (unsigned long)arg;
 333
 334        /*
 335         * Flush all to work around Errata in early athlons regarding
 336         * large page flushing.
 337         */
 338        __flush_tlb_all();
 339
 340        if (cache && boot_cpu_data.x86 >= 4)
 341                wbinvd();
 342}
 343
 344static void cpa_flush_all(unsigned long cache)
 345{
 346        BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
 347
 348        on_each_cpu(__cpa_flush_all, (void *) cache, 1);
 349}
 350
 351static void __cpa_flush_tlb(void *data)
 352{
 353        struct cpa_data *cpa = data;
 354        unsigned int i;
 355
 356        for (i = 0; i < cpa->numpages; i++)
 357                flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i)));
 358}
 359
 360static void cpa_flush(struct cpa_data *data, int cache)
 361{
 362        struct cpa_data *cpa = data;
 363        unsigned int i;
 364
 365        BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
 366
 367        if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 368                cpa_flush_all(cache);
 369                return;
 370        }
 371
 372        if (cpa->force_flush_all || cpa->numpages > tlb_single_page_flush_ceiling)
 373                flush_tlb_all();
 374        else
 375                on_each_cpu(__cpa_flush_tlb, cpa, 1);
 376
 377        if (!cache)
 378                return;
 379
 380        mb();
 381        for (i = 0; i < cpa->numpages; i++) {
 382                unsigned long addr = __cpa_addr(cpa, i);
 383                unsigned int level;
 384
 385                pte_t *pte = lookup_address(addr, &level);
 386
 387                /*
 388                 * Only flush present addresses:
 389                 */
 390                if (pte && (pte_val(*pte) & _PAGE_PRESENT))
 391                        clflush_cache_range_opt((void *)fix_addr(addr), PAGE_SIZE);
 392        }
 393        mb();
 394}
 395
 396static bool overlaps(unsigned long r1_start, unsigned long r1_end,
 397                     unsigned long r2_start, unsigned long r2_end)
 398{
 399        return (r1_start <= r2_end && r1_end >= r2_start) ||
 400                (r2_start <= r1_end && r2_end >= r1_start);
 401}
 402
 403#ifdef CONFIG_PCI_BIOS
 404/*
 405 * The BIOS area between 640k and 1Mb needs to be executable for PCI BIOS
 406 * based config access (CONFIG_PCI_GOBIOS) support.
 407 */
 408#define BIOS_PFN        PFN_DOWN(BIOS_BEGIN)
 409#define BIOS_PFN_END    PFN_DOWN(BIOS_END - 1)
 410
 411static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn)
 412{
 413        if (pcibios_enabled && overlaps(spfn, epfn, BIOS_PFN, BIOS_PFN_END))
 414                return _PAGE_NX;
 415        return 0;
 416}
 417#else
 418static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn)
 419{
 420        return 0;
 421}
 422#endif
 423
 424/*
 425 * The .rodata section needs to be read-only. Using the pfn catches all
 426 * aliases.  This also includes __ro_after_init, so do not enforce until
 427 * kernel_set_to_readonly is true.
 428 */
 429static pgprotval_t protect_rodata(unsigned long spfn, unsigned long epfn)
 430{
 431        unsigned long epfn_ro, spfn_ro = PFN_DOWN(__pa_symbol(__start_rodata));
 432
 433        /*
 434         * Note: __end_rodata is at page aligned and not inclusive, so
 435         * subtract 1 to get the last enforced PFN in the rodata area.
 436         */
 437        epfn_ro = PFN_DOWN(__pa_symbol(__end_rodata)) - 1;
 438
 439        if (kernel_set_to_readonly && overlaps(spfn, epfn, spfn_ro, epfn_ro))
 440                return _PAGE_RW;
 441        return 0;
 442}
 443
 444/*
 445 * Protect kernel text against becoming non executable by forbidding
 446 * _PAGE_NX.  This protects only the high kernel mapping (_text -> _etext)
 447 * out of which the kernel actually executes.  Do not protect the low
 448 * mapping.
 449 *
 450 * This does not cover __inittext since that is gone after boot.
 451 */
 452static pgprotval_t protect_kernel_text(unsigned long start, unsigned long end)
 453{
 454        unsigned long t_end = (unsigned long)_etext - 1;
 455        unsigned long t_start = (unsigned long)_text;
 456
 457        if (overlaps(start, end, t_start, t_end))
 458                return _PAGE_NX;
 459        return 0;
 460}
 461
 462#if defined(CONFIG_X86_64)
 463/*
 464 * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
 465 * kernel text mappings for the large page aligned text, rodata sections
 466 * will be always read-only. For the kernel identity mappings covering the
 467 * holes caused by this alignment can be anything that user asks.
 468 *
 469 * This will preserve the large page mappings for kernel text/data at no
 470 * extra cost.
 471 */
 472static pgprotval_t protect_kernel_text_ro(unsigned long start,
 473                                          unsigned long end)
 474{
 475        unsigned long t_end = (unsigned long)__end_rodata_hpage_align - 1;
 476        unsigned long t_start = (unsigned long)_text;
 477        unsigned int level;
 478
 479        if (!kernel_set_to_readonly || !overlaps(start, end, t_start, t_end))
 480                return 0;
 481        /*
 482         * Don't enforce the !RW mapping for the kernel text mapping, if
 483         * the current mapping is already using small page mapping.  No
 484         * need to work hard to preserve large page mappings in this case.
 485         *
 486         * This also fixes the Linux Xen paravirt guest boot failure caused
 487         * by unexpected read-only mappings for kernel identity
 488         * mappings. In this paravirt guest case, the kernel text mapping
 489         * and the kernel identity mapping share the same page-table pages,
 490         * so the protections for kernel text and identity mappings have to
 491         * be the same.
 492         */
 493        if (lookup_address(start, &level) && (level != PG_LEVEL_4K))
 494                return _PAGE_RW;
 495        return 0;
 496}
 497#else
 498static pgprotval_t protect_kernel_text_ro(unsigned long start,
 499                                          unsigned long end)
 500{
 501        return 0;
 502}
 503#endif
 504
 505static inline bool conflicts(pgprot_t prot, pgprotval_t val)
 506{
 507        return (pgprot_val(prot) & ~val) != pgprot_val(prot);
 508}
 509
 510static inline void check_conflict(int warnlvl, pgprot_t prot, pgprotval_t val,
 511                                  unsigned long start, unsigned long end,
 512                                  unsigned long pfn, const char *txt)
 513{
 514        static const char *lvltxt[] = {
 515                [CPA_CONFLICT]  = "conflict",
 516                [CPA_PROTECT]   = "protect",
 517                [CPA_DETECT]    = "detect",
 518        };
 519
 520        if (warnlvl > cpa_warn_level || !conflicts(prot, val))
 521                return;
 522
 523        pr_warn("CPA %8s %10s: 0x%016lx - 0x%016lx PFN %lx req %016llx prevent %016llx\n",
 524                lvltxt[warnlvl], txt, start, end, pfn, (unsigned long long)pgprot_val(prot),
 525                (unsigned long long)val);
 526}
 527
 528/*
 529 * Certain areas of memory on x86 require very specific protection flags,
 530 * for example the BIOS area or kernel text. Callers don't always get this
 531 * right (again, ioremap() on BIOS memory is not uncommon) so this function
 532 * checks and fixes these known static required protection bits.
 533 */
 534static inline pgprot_t static_protections(pgprot_t prot, unsigned long start,
 535                                          unsigned long pfn, unsigned long npg,
 536                                          unsigned long lpsize, int warnlvl)
 537{
 538        pgprotval_t forbidden, res;
 539        unsigned long end;
 540
 541        /*
 542         * There is no point in checking RW/NX conflicts when the requested
 543         * mapping is setting the page !PRESENT.
 544         */
 545        if (!(pgprot_val(prot) & _PAGE_PRESENT))
 546                return prot;
 547
 548        /* Operate on the virtual address */
 549        end = start + npg * PAGE_SIZE - 1;
 550
 551        res = protect_kernel_text(start, end);
 552        check_conflict(warnlvl, prot, res, start, end, pfn, "Text NX");
 553        forbidden = res;
 554
 555        /*
 556         * Special case to preserve a large page. If the change spawns the
 557         * full large page mapping then there is no point to split it
 558         * up. Happens with ftrace and is going to be removed once ftrace
 559         * switched to text_poke().
 560         */
 561        if (lpsize != (npg * PAGE_SIZE) || (start & (lpsize - 1))) {
 562                res = protect_kernel_text_ro(start, end);
 563                check_conflict(warnlvl, prot, res, start, end, pfn, "Text RO");
 564                forbidden |= res;
 565        }
 566
 567        /* Check the PFN directly */
 568        res = protect_pci_bios(pfn, pfn + npg - 1);
 569        check_conflict(warnlvl, prot, res, start, end, pfn, "PCIBIOS NX");
 570        forbidden |= res;
 571
 572        res = protect_rodata(pfn, pfn + npg - 1);
 573        check_conflict(warnlvl, prot, res, start, end, pfn, "Rodata RO");
 574        forbidden |= res;
 575
 576        return __pgprot(pgprot_val(prot) & ~forbidden);
 577}
 578
 579/*
 580 * Lookup the page table entry for a virtual address in a specific pgd.
 581 * Return a pointer to the entry and the level of the mapping.
 582 */
 583pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
 584                             unsigned int *level)
 585{
 586        p4d_t *p4d;
 587        pud_t *pud;
 588        pmd_t *pmd;
 589
 590        *level = PG_LEVEL_NONE;
 591
 592        if (pgd_none(*pgd))
 593                return NULL;
 594
 595        p4d = p4d_offset(pgd, address);
 596        if (p4d_none(*p4d))
 597                return NULL;
 598
 599        *level = PG_LEVEL_512G;
 600        if (p4d_large(*p4d) || !p4d_present(*p4d))
 601                return (pte_t *)p4d;
 602
 603        pud = pud_offset(p4d, address);
 604        if (pud_none(*pud))
 605                return NULL;
 606
 607        *level = PG_LEVEL_1G;
 608        if (pud_large(*pud) || !pud_present(*pud))
 609                return (pte_t *)pud;
 610
 611        pmd = pmd_offset(pud, address);
 612        if (pmd_none(*pmd))
 613                return NULL;
 614
 615        *level = PG_LEVEL_2M;
 616        if (pmd_large(*pmd) || !pmd_present(*pmd))
 617                return (pte_t *)pmd;
 618
 619        *level = PG_LEVEL_4K;
 620
 621        return pte_offset_kernel(pmd, address);
 622}
 623
 624/*
 625 * Lookup the page table entry for a virtual address. Return a pointer
 626 * to the entry and the level of the mapping.
 627 *
 628 * Note: We return pud and pmd either when the entry is marked large
 629 * or when the present bit is not set. Otherwise we would return a
 630 * pointer to a nonexisting mapping.
 631 */
 632pte_t *lookup_address(unsigned long address, unsigned int *level)
 633{
 634        return lookup_address_in_pgd(pgd_offset_k(address), address, level);
 635}
 636EXPORT_SYMBOL_GPL(lookup_address);
 637
 638/*
 639 * Lookup the page table entry for a virtual address in a given mm. Return a
 640 * pointer to the entry and the level of the mapping.
 641 */
 642pte_t *lookup_address_in_mm(struct mm_struct *mm, unsigned long address,
 643                            unsigned int *level)
 644{
 645        return lookup_address_in_pgd(pgd_offset(mm, address), address, level);
 646}
 647EXPORT_SYMBOL_GPL(lookup_address_in_mm);
 648
 649static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
 650                                  unsigned int *level)
 651{
 652        if (cpa->pgd)
 653                return lookup_address_in_pgd(cpa->pgd + pgd_index(address),
 654                                               address, level);
 655
 656        return lookup_address(address, level);
 657}
 658
 659/*
 660 * Lookup the PMD entry for a virtual address. Return a pointer to the entry
 661 * or NULL if not present.
 662 */
 663pmd_t *lookup_pmd_address(unsigned long address)
 664{
 665        pgd_t *pgd;
 666        p4d_t *p4d;
 667        pud_t *pud;
 668
 669        pgd = pgd_offset_k(address);
 670        if (pgd_none(*pgd))
 671                return NULL;
 672
 673        p4d = p4d_offset(pgd, address);
 674        if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d))
 675                return NULL;
 676
 677        pud = pud_offset(p4d, address);
 678        if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud))
 679                return NULL;
 680
 681        return pmd_offset(pud, address);
 682}
 683
 684/*
 685 * This is necessary because __pa() does not work on some
 686 * kinds of memory, like vmalloc() or the alloc_remap()
 687 * areas on 32-bit NUMA systems.  The percpu areas can
 688 * end up in this kind of memory, for instance.
 689 *
 690 * This could be optimized, but it is only intended to be
 691 * used at initialization time, and keeping it
 692 * unoptimized should increase the testing coverage for
 693 * the more obscure platforms.
 694 */
 695phys_addr_t slow_virt_to_phys(void *__virt_addr)
 696{
 697        unsigned long virt_addr = (unsigned long)__virt_addr;
 698        phys_addr_t phys_addr;
 699        unsigned long offset;
 700        enum pg_level level;
 701        pte_t *pte;
 702
 703        pte = lookup_address(virt_addr, &level);
 704        BUG_ON(!pte);
 705
 706        /*
 707         * pXX_pfn() returns unsigned long, which must be cast to phys_addr_t
 708         * before being left-shifted PAGE_SHIFT bits -- this trick is to
 709         * make 32-PAE kernel work correctly.
 710         */
 711        switch (level) {
 712        case PG_LEVEL_1G:
 713                phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT;
 714                offset = virt_addr & ~PUD_PAGE_MASK;
 715                break;
 716        case PG_LEVEL_2M:
 717                phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT;
 718                offset = virt_addr & ~PMD_PAGE_MASK;
 719                break;
 720        default:
 721                phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
 722                offset = virt_addr & ~PAGE_MASK;
 723        }
 724
 725        return (phys_addr_t)(phys_addr | offset);
 726}
 727EXPORT_SYMBOL_GPL(slow_virt_to_phys);
 728
 729/*
 730 * Set the new pmd in all the pgds we know about:
 731 */
 732static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
 733{
 734        /* change init_mm */
 735        set_pte_atomic(kpte, pte);
 736#ifdef CONFIG_X86_32
 737        if (!SHARED_KERNEL_PMD) {
 738                struct page *page;
 739
 740                list_for_each_entry(page, &pgd_list, lru) {
 741                        pgd_t *pgd;
 742                        p4d_t *p4d;
 743                        pud_t *pud;
 744                        pmd_t *pmd;
 745
 746                        pgd = (pgd_t *)page_address(page) + pgd_index(address);
 747                        p4d = p4d_offset(pgd, address);
 748                        pud = pud_offset(p4d, address);
 749                        pmd = pmd_offset(pud, address);
 750                        set_pte_atomic((pte_t *)pmd, pte);
 751                }
 752        }
 753#endif
 754}
 755
 756static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot)
 757{
 758        /*
 759         * _PAGE_GLOBAL means "global page" for present PTEs.
 760         * But, it is also used to indicate _PAGE_PROTNONE
 761         * for non-present PTEs.
 762         *
 763         * This ensures that a _PAGE_GLOBAL PTE going from
 764         * present to non-present is not confused as
 765         * _PAGE_PROTNONE.
 766         */
 767        if (!(pgprot_val(prot) & _PAGE_PRESENT))
 768                pgprot_val(prot) &= ~_PAGE_GLOBAL;
 769
 770        return prot;
 771}
 772
 773static int __should_split_large_page(pte_t *kpte, unsigned long address,
 774                                     struct cpa_data *cpa)
 775{
 776        unsigned long numpages, pmask, psize, lpaddr, pfn, old_pfn;
 777        pgprot_t old_prot, new_prot, req_prot, chk_prot;
 778        pte_t new_pte, *tmp;
 779        enum pg_level level;
 780
 781        /*
 782         * Check for races, another CPU might have split this page
 783         * up already:
 784         */
 785        tmp = _lookup_address_cpa(cpa, address, &level);
 786        if (tmp != kpte)
 787                return 1;
 788
 789        switch (level) {
 790        case PG_LEVEL_2M:
 791                old_prot = pmd_pgprot(*(pmd_t *)kpte);
 792                old_pfn = pmd_pfn(*(pmd_t *)kpte);
 793                cpa_inc_2m_checked();
 794                break;
 795        case PG_LEVEL_1G:
 796                old_prot = pud_pgprot(*(pud_t *)kpte);
 797                old_pfn = pud_pfn(*(pud_t *)kpte);
 798                cpa_inc_1g_checked();
 799                break;
 800        default:
 801                return -EINVAL;
 802        }
 803
 804        psize = page_level_size(level);
 805        pmask = page_level_mask(level);
 806
 807        /*
 808         * Calculate the number of pages, which fit into this large
 809         * page starting at address:
 810         */
 811        lpaddr = (address + psize) & pmask;
 812        numpages = (lpaddr - address) >> PAGE_SHIFT;
 813        if (numpages < cpa->numpages)
 814                cpa->numpages = numpages;
 815
 816        /*
 817         * We are safe now. Check whether the new pgprot is the same:
 818         * Convert protection attributes to 4k-format, as cpa->mask* are set
 819         * up accordingly.
 820         */
 821
 822        /* Clear PSE (aka _PAGE_PAT) and move PAT bit to correct position */
 823        req_prot = pgprot_large_2_4k(old_prot);
 824
 825        pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
 826        pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
 827
 828        /*
 829         * req_prot is in format of 4k pages. It must be converted to large
 830         * page format: the caching mode includes the PAT bit located at
 831         * different bit positions in the two formats.
 832         */
 833        req_prot = pgprot_4k_2_large(req_prot);
 834        req_prot = pgprot_clear_protnone_bits(req_prot);
 835        if (pgprot_val(req_prot) & _PAGE_PRESENT)
 836                pgprot_val(req_prot) |= _PAGE_PSE;
 837
 838        /*
 839         * old_pfn points to the large page base pfn. So we need to add the
 840         * offset of the virtual address:
 841         */
 842        pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT);
 843        cpa->pfn = pfn;
 844
 845        /*
 846         * Calculate the large page base address and the number of 4K pages
 847         * in the large page
 848         */
 849        lpaddr = address & pmask;
 850        numpages = psize >> PAGE_SHIFT;
 851
 852        /*
 853         * Sanity check that the existing mapping is correct versus the static
 854         * protections. static_protections() guards against !PRESENT, so no
 855         * extra conditional required here.
 856         */
 857        chk_prot = static_protections(old_prot, lpaddr, old_pfn, numpages,
 858                                      psize, CPA_CONFLICT);
 859
 860        if (WARN_ON_ONCE(pgprot_val(chk_prot) != pgprot_val(old_prot))) {
 861                /*
 862                 * Split the large page and tell the split code to
 863                 * enforce static protections.
 864                 */
 865                cpa->force_static_prot = 1;
 866                return 1;
 867        }
 868
 869        /*
 870         * Optimization: If the requested pgprot is the same as the current
 871         * pgprot, then the large page can be preserved and no updates are
 872         * required independent of alignment and length of the requested
 873         * range. The above already established that the current pgprot is
 874         * correct, which in consequence makes the requested pgprot correct
 875         * as well if it is the same. The static protection scan below will
 876         * not come to a different conclusion.
 877         */
 878        if (pgprot_val(req_prot) == pgprot_val(old_prot)) {
 879                cpa_inc_lp_sameprot(level);
 880                return 0;
 881        }
 882
 883        /*
 884         * If the requested range does not cover the full page, split it up
 885         */
 886        if (address != lpaddr || cpa->numpages != numpages)
 887                return 1;
 888
 889        /*
 890         * Check whether the requested pgprot is conflicting with a static
 891         * protection requirement in the large page.
 892         */
 893        new_prot = static_protections(req_prot, lpaddr, old_pfn, numpages,
 894                                      psize, CPA_DETECT);
 895
 896        /*
 897         * If there is a conflict, split the large page.
 898         *
 899         * There used to be a 4k wise evaluation trying really hard to
 900         * preserve the large pages, but experimentation has shown, that this
 901         * does not help at all. There might be corner cases which would
 902         * preserve one large page occasionally, but it's really not worth the
 903         * extra code and cycles for the common case.
 904         */
 905        if (pgprot_val(req_prot) != pgprot_val(new_prot))
 906                return 1;
 907
 908        /* All checks passed. Update the large page mapping. */
 909        new_pte = pfn_pte(old_pfn, new_prot);
 910        __set_pmd_pte(kpte, address, new_pte);
 911        cpa->flags |= CPA_FLUSHTLB;
 912        cpa_inc_lp_preserved(level);
 913        return 0;
 914}
 915
 916static int should_split_large_page(pte_t *kpte, unsigned long address,
 917                                   struct cpa_data *cpa)
 918{
 919        int do_split;
 920
 921        if (cpa->force_split)
 922                return 1;
 923
 924        spin_lock(&pgd_lock);
 925        do_split = __should_split_large_page(kpte, address, cpa);
 926        spin_unlock(&pgd_lock);
 927
 928        return do_split;
 929}
 930
 931static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
 932                          pgprot_t ref_prot, unsigned long address,
 933                          unsigned long size)
 934{
 935        unsigned int npg = PFN_DOWN(size);
 936        pgprot_t prot;
 937
 938        /*
 939         * If should_split_large_page() discovered an inconsistent mapping,
 940         * remove the invalid protection in the split mapping.
 941         */
 942        if (!cpa->force_static_prot)
 943                goto set;
 944
 945        /* Hand in lpsize = 0 to enforce the protection mechanism */
 946        prot = static_protections(ref_prot, address, pfn, npg, 0, CPA_PROTECT);
 947
 948        if (pgprot_val(prot) == pgprot_val(ref_prot))
 949                goto set;
 950
 951        /*
 952         * If this is splitting a PMD, fix it up. PUD splits cannot be
 953         * fixed trivially as that would require to rescan the newly
 954         * installed PMD mappings after returning from split_large_page()
 955         * so an eventual further split can allocate the necessary PTE
 956         * pages. Warn for now and revisit it in case this actually
 957         * happens.
 958         */
 959        if (size == PAGE_SIZE)
 960                ref_prot = prot;
 961        else
 962                pr_warn_once("CPA: Cannot fixup static protections for PUD split\n");
 963set:
 964        set_pte(pte, pfn_pte(pfn, ref_prot));
 965}
 966
 967static int
 968__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
 969                   struct page *base)
 970{
 971        unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
 972        pte_t *pbase = (pte_t *)page_address(base);
 973        unsigned int i, level;
 974        pgprot_t ref_prot;
 975        pte_t *tmp;
 976
 977        spin_lock(&pgd_lock);
 978        /*
 979         * Check for races, another CPU might have split this page
 980         * up for us already:
 981         */
 982        tmp = _lookup_address_cpa(cpa, address, &level);
 983        if (tmp != kpte) {
 984                spin_unlock(&pgd_lock);
 985                return 1;
 986        }
 987
 988        paravirt_alloc_pte(&init_mm, page_to_pfn(base));
 989
 990        switch (level) {
 991        case PG_LEVEL_2M:
 992                ref_prot = pmd_pgprot(*(pmd_t *)kpte);
 993                /*
 994                 * Clear PSE (aka _PAGE_PAT) and move
 995                 * PAT bit to correct position.
 996                 */
 997                ref_prot = pgprot_large_2_4k(ref_prot);
 998                ref_pfn = pmd_pfn(*(pmd_t *)kpte);
 999                lpaddr = address & PMD_MASK;
1000                lpinc = PAGE_SIZE;
1001                break;
1002
1003        case PG_LEVEL_1G:
1004                ref_prot = pud_pgprot(*(pud_t *)kpte);
1005                ref_pfn = pud_pfn(*(pud_t *)kpte);
1006                pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
1007                lpaddr = address & PUD_MASK;
1008                lpinc = PMD_SIZE;
1009                /*
1010                 * Clear the PSE flags if the PRESENT flag is not set
1011                 * otherwise pmd_present/pmd_huge will return true
1012                 * even on a non present pmd.
1013                 */
1014                if (!(pgprot_val(ref_prot) & _PAGE_PRESENT))
1015                        pgprot_val(ref_prot) &= ~_PAGE_PSE;
1016                break;
1017
1018        default:
1019                spin_unlock(&pgd_lock);
1020                return 1;
1021        }
1022
1023        ref_prot = pgprot_clear_protnone_bits(ref_prot);
1024
1025        /*
1026         * Get the target pfn from the original entry:
1027         */
1028        pfn = ref_pfn;
1029        for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc, lpaddr += lpinc)
1030                split_set_pte(cpa, pbase + i, pfn, ref_prot, lpaddr, lpinc);
1031
1032        if (virt_addr_valid(address)) {
1033                unsigned long pfn = PFN_DOWN(__pa(address));
1034
1035                if (pfn_range_is_mapped(pfn, pfn + 1))
1036                        split_page_count(level);
1037        }
1038
1039        /*
1040         * Install the new, split up pagetable.
1041         *
1042         * We use the standard kernel pagetable protections for the new
1043         * pagetable protections, the actual ptes set above control the
1044         * primary protection behavior:
1045         */
1046        __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
1047
1048        /*
1049         * Do a global flush tlb after splitting the large page
1050         * and before we do the actual change page attribute in the PTE.
1051         *
1052         * Without this, we violate the TLB application note, that says:
1053         * "The TLBs may contain both ordinary and large-page
1054         *  translations for a 4-KByte range of linear addresses. This
1055         *  may occur if software modifies the paging structures so that
1056         *  the page size used for the address range changes. If the two
1057         *  translations differ with respect to page frame or attributes
1058         *  (e.g., permissions), processor behavior is undefined and may
1059         *  be implementation-specific."
1060         *
1061         * We do this global tlb flush inside the cpa_lock, so that we
1062         * don't allow any other cpu, with stale tlb entries change the
1063         * page attribute in parallel, that also falls into the
1064         * just split large page entry.
1065         */
1066        flush_tlb_all();
1067        spin_unlock(&pgd_lock);
1068
1069        return 0;
1070}
1071
1072static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
1073                            unsigned long address)
1074{
1075        struct page *base;
1076
1077        if (!debug_pagealloc_enabled())
1078                spin_unlock(&cpa_lock);
1079        base = alloc_pages(GFP_KERNEL, 0);
1080        if (!debug_pagealloc_enabled())
1081                spin_lock(&cpa_lock);
1082        if (!base)
1083                return -ENOMEM;
1084
1085        if (__split_large_page(cpa, kpte, address, base))
1086                __free_page(base);
1087
1088        return 0;
1089}
1090
1091static bool try_to_free_pte_page(pte_t *pte)
1092{
1093        int i;
1094
1095        for (i = 0; i < PTRS_PER_PTE; i++)
1096                if (!pte_none(pte[i]))
1097                        return false;
1098
1099        free_page((unsigned long)pte);
1100        return true;
1101}
1102
1103static bool try_to_free_pmd_page(pmd_t *pmd)
1104{
1105        int i;
1106
1107        for (i = 0; i < PTRS_PER_PMD; i++)
1108                if (!pmd_none(pmd[i]))
1109                        return false;
1110
1111        free_page((unsigned long)pmd);
1112        return true;
1113}
1114
1115static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
1116{
1117        pte_t *pte = pte_offset_kernel(pmd, start);
1118
1119        while (start < end) {
1120                set_pte(pte, __pte(0));
1121
1122                start += PAGE_SIZE;
1123                pte++;
1124        }
1125
1126        if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
1127                pmd_clear(pmd);
1128                return true;
1129        }
1130        return false;
1131}
1132
1133static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
1134                              unsigned long start, unsigned long end)
1135{
1136        if (unmap_pte_range(pmd, start, end))
1137                if (try_to_free_pmd_page(pud_pgtable(*pud)))
1138                        pud_clear(pud);
1139}
1140
1141static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
1142{
1143        pmd_t *pmd = pmd_offset(pud, start);
1144
1145        /*
1146         * Not on a 2MB page boundary?
1147         */
1148        if (start & (PMD_SIZE - 1)) {
1149                unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
1150                unsigned long pre_end = min_t(unsigned long, end, next_page);
1151
1152                __unmap_pmd_range(pud, pmd, start, pre_end);
1153
1154                start = pre_end;
1155                pmd++;
1156        }
1157
1158        /*
1159         * Try to unmap in 2M chunks.
1160         */
1161        while (end - start >= PMD_SIZE) {
1162                if (pmd_large(*pmd))
1163                        pmd_clear(pmd);
1164                else
1165                        __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);
1166
1167                start += PMD_SIZE;
1168                pmd++;
1169        }
1170
1171        /*
1172         * 4K leftovers?
1173         */
1174        if (start < end)
1175                return __unmap_pmd_range(pud, pmd, start, end);
1176
1177        /*
1178         * Try again to free the PMD page if haven't succeeded above.
1179         */
1180        if (!pud_none(*pud))
1181                if (try_to_free_pmd_page(pud_pgtable(*pud)))
1182                        pud_clear(pud);
1183}
1184
1185static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end)
1186{
1187        pud_t *pud = pud_offset(p4d, start);
1188
1189        /*
1190         * Not on a GB page boundary?
1191         */
1192        if (start & (PUD_SIZE - 1)) {
1193                unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
1194                unsigned long pre_end   = min_t(unsigned long, end, next_page);
1195
1196                unmap_pmd_range(pud, start, pre_end);
1197
1198                start = pre_end;
1199                pud++;
1200        }
1201
1202        /*
1203         * Try to unmap in 1G chunks?
1204         */
1205        while (end - start >= PUD_SIZE) {
1206
1207                if (pud_large(*pud))
1208                        pud_clear(pud);
1209                else
1210                        unmap_pmd_range(pud, start, start + PUD_SIZE);
1211
1212                start += PUD_SIZE;
1213                pud++;
1214        }
1215
1216        /*
1217         * 2M leftovers?
1218         */
1219        if (start < end)
1220                unmap_pmd_range(pud, start, end);
1221
1222        /*
1223         * No need to try to free the PUD page because we'll free it in
1224         * populate_pgd's error path
1225         */
1226}
1227
1228static int alloc_pte_page(pmd_t *pmd)
1229{
1230        pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
1231        if (!pte)
1232                return -1;
1233
1234        set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
1235        return 0;
1236}
1237
1238static int alloc_pmd_page(pud_t *pud)
1239{
1240        pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
1241        if (!pmd)
1242                return -1;
1243
1244        set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
1245        return 0;
1246}
1247
1248static void populate_pte(struct cpa_data *cpa,
1249                         unsigned long start, unsigned long end,
1250                         unsigned num_pages, pmd_t *pmd, pgprot_t pgprot)
1251{
1252        pte_t *pte;
1253
1254        pte = pte_offset_kernel(pmd, start);
1255
1256        pgprot = pgprot_clear_protnone_bits(pgprot);
1257
1258        while (num_pages-- && start < end) {
1259                set_pte(pte, pfn_pte(cpa->pfn, pgprot));
1260
1261                start    += PAGE_SIZE;
1262                cpa->pfn++;
1263                pte++;
1264        }
1265}
1266
1267static long populate_pmd(struct cpa_data *cpa,
1268                         unsigned long start, unsigned long end,
1269                         unsigned num_pages, pud_t *pud, pgprot_t pgprot)
1270{
1271        long cur_pages = 0;
1272        pmd_t *pmd;
1273        pgprot_t pmd_pgprot;
1274
1275        /*
1276         * Not on a 2M boundary?
1277         */
1278        if (start & (PMD_SIZE - 1)) {
1279                unsigned long pre_end = start + (num_pages << PAGE_SHIFT);
1280                unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
1281
1282                pre_end   = min_t(unsigned long, pre_end, next_page);
1283                cur_pages = (pre_end - start) >> PAGE_SHIFT;
1284                cur_pages = min_t(unsigned int, num_pages, cur_pages);
1285
1286                /*
1287                 * Need a PTE page?
1288                 */
1289                pmd = pmd_offset(pud, start);
1290                if (pmd_none(*pmd))
1291                        if (alloc_pte_page(pmd))
1292                                return -1;
1293
1294                populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);
1295
1296                start = pre_end;
1297        }
1298
1299        /*
1300         * We mapped them all?
1301         */
1302        if (num_pages == cur_pages)
1303                return cur_pages;
1304
1305        pmd_pgprot = pgprot_4k_2_large(pgprot);
1306
1307        while (end - start >= PMD_SIZE) {
1308
1309                /*
1310                 * We cannot use a 1G page so allocate a PMD page if needed.
1311                 */
1312                if (pud_none(*pud))
1313                        if (alloc_pmd_page(pud))
1314                                return -1;
1315
1316                pmd = pmd_offset(pud, start);
1317
1318                set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn,
1319                                        canon_pgprot(pmd_pgprot))));
1320
1321                start     += PMD_SIZE;
1322                cpa->pfn  += PMD_SIZE >> PAGE_SHIFT;
1323                cur_pages += PMD_SIZE >> PAGE_SHIFT;
1324        }
1325
1326        /*
1327         * Map trailing 4K pages.
1328         */
1329        if (start < end) {
1330                pmd = pmd_offset(pud, start);
1331                if (pmd_none(*pmd))
1332                        if (alloc_pte_page(pmd))
1333                                return -1;
1334
1335                populate_pte(cpa, start, end, num_pages - cur_pages,
1336                             pmd, pgprot);
1337        }
1338        return num_pages;
1339}
1340
1341static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d,
1342                        pgprot_t pgprot)
1343{
1344        pud_t *pud;
1345        unsigned long end;
1346        long cur_pages = 0;
1347        pgprot_t pud_pgprot;
1348
1349        end = start + (cpa->numpages << PAGE_SHIFT);
1350
1351        /*
1352         * Not on a Gb page boundary? => map everything up to it with
1353         * smaller pages.
1354         */
1355        if (start & (PUD_SIZE - 1)) {
1356                unsigned long pre_end;
1357                unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
1358
1359                pre_end   = min_t(unsigned long, end, next_page);
1360                cur_pages = (pre_end - start) >> PAGE_SHIFT;
1361                cur_pages = min_t(int, (int)cpa->numpages, cur_pages);
1362
1363                pud = pud_offset(p4d, start);
1364
1365                /*
1366                 * Need a PMD page?
1367                 */
1368                if (pud_none(*pud))
1369                        if (alloc_pmd_page(pud))
1370                                return -1;
1371
1372                cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
1373                                         pud, pgprot);
1374                if (cur_pages < 0)
1375                        return cur_pages;
1376
1377                start = pre_end;
1378        }
1379
1380        /* We mapped them all? */
1381        if (cpa->numpages == cur_pages)
1382                return cur_pages;
1383
1384        pud = pud_offset(p4d, start);
1385        pud_pgprot = pgprot_4k_2_large(pgprot);
1386
1387        /*
1388         * Map everything starting from the Gb boundary, possibly with 1G pages
1389         */
1390        while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) {
1391                set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn,
1392                                   canon_pgprot(pud_pgprot))));
1393
1394                start     += PUD_SIZE;
1395                cpa->pfn  += PUD_SIZE >> PAGE_SHIFT;
1396                cur_pages += PUD_SIZE >> PAGE_SHIFT;
1397                pud++;
1398        }
1399
1400        /* Map trailing leftover */
1401        if (start < end) {
1402                long tmp;
1403
1404                pud = pud_offset(p4d, start);
1405                if (pud_none(*pud))
1406                        if (alloc_pmd_page(pud))
1407                                return -1;
1408
1409                tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
1410                                   pud, pgprot);
1411                if (tmp < 0)
1412                        return cur_pages;
1413
1414                cur_pages += tmp;
1415        }
1416        return cur_pages;
1417}
1418
1419/*
1420 * Restrictions for kernel page table do not necessarily apply when mapping in
1421 * an alternate PGD.
1422 */
1423static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
1424{
1425        pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
1426        pud_t *pud = NULL;      /* shut up gcc */
1427        p4d_t *p4d;
1428        pgd_t *pgd_entry;
1429        long ret;
1430
1431        pgd_entry = cpa->pgd + pgd_index(addr);
1432
1433        if (pgd_none(*pgd_entry)) {
1434                p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
1435                if (!p4d)
1436                        return -1;
1437
1438                set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE));
1439        }
1440
1441        /*
1442         * Allocate a PUD page and hand it down for mapping.
1443         */
1444        p4d = p4d_offset(pgd_entry, addr);
1445        if (p4d_none(*p4d)) {
1446                pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
1447                if (!pud)
1448                        return -1;
1449
1450                set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
1451        }
1452
1453        pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
1454        pgprot_val(pgprot) |=  pgprot_val(cpa->mask_set);
1455
1456        ret = populate_pud(cpa, addr, p4d, pgprot);
1457        if (ret < 0) {
1458                /*
1459                 * Leave the PUD page in place in case some other CPU or thread
1460                 * already found it, but remove any useless entries we just
1461                 * added to it.
1462                 */
1463                unmap_pud_range(p4d, addr,
1464                                addr + (cpa->numpages << PAGE_SHIFT));
1465                return ret;
1466        }
1467
1468        cpa->numpages = ret;
1469        return 0;
1470}
1471
1472static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
1473                               int primary)
1474{
1475        if (cpa->pgd) {
1476                /*
1477                 * Right now, we only execute this code path when mapping
1478                 * the EFI virtual memory map regions, no other users
1479                 * provide a ->pgd value. This may change in the future.
1480                 */
1481                return populate_pgd(cpa, vaddr);
1482        }
1483
1484        /*
1485         * Ignore all non primary paths.
1486         */
1487        if (!primary) {
1488                cpa->numpages = 1;
1489                return 0;
1490        }
1491
1492        /*
1493         * Ignore the NULL PTE for kernel identity mapping, as it is expected
1494         * to have holes.
1495         * Also set numpages to '1' indicating that we processed cpa req for
1496         * one virtual address page and its pfn. TBD: numpages can be set based
1497         * on the initial value and the level returned by lookup_address().
1498         */
1499        if (within(vaddr, PAGE_OFFSET,
1500                   PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
1501                cpa->numpages = 1;
1502                cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
1503                return 0;
1504
1505        } else if (__cpa_pfn_in_highmap(cpa->pfn)) {
1506                /* Faults in the highmap are OK, so do not warn: */
1507                return -EFAULT;
1508        } else {
1509                WARN(1, KERN_WARNING "CPA: called for zero pte. "
1510                        "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
1511                        *cpa->vaddr);
1512
1513                return -EFAULT;
1514        }
1515}
1516
1517static int __change_page_attr(struct cpa_data *cpa, int primary)
1518{
1519        unsigned long address;
1520        int do_split, err;
1521        unsigned int level;
1522        pte_t *kpte, old_pte;
1523
1524        address = __cpa_addr(cpa, cpa->curpage);
1525repeat:
1526        kpte = _lookup_address_cpa(cpa, address, &level);
1527        if (!kpte)
1528                return __cpa_process_fault(cpa, address, primary);
1529
1530        old_pte = *kpte;
1531        if (pte_none(old_pte))
1532                return __cpa_process_fault(cpa, address, primary);
1533
1534        if (level == PG_LEVEL_4K) {
1535                pte_t new_pte;
1536                pgprot_t new_prot = pte_pgprot(old_pte);
1537                unsigned long pfn = pte_pfn(old_pte);
1538
1539                pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
1540                pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
1541
1542                cpa_inc_4k_install();
1543                /* Hand in lpsize = 0 to enforce the protection mechanism */
1544                new_prot = static_protections(new_prot, address, pfn, 1, 0,
1545                                              CPA_PROTECT);
1546
1547                new_prot = pgprot_clear_protnone_bits(new_prot);
1548
1549                /*
1550                 * We need to keep the pfn from the existing PTE,
1551                 * after all we're only going to change it's attributes
1552                 * not the memory it points to
1553                 */
1554                new_pte = pfn_pte(pfn, new_prot);
1555                cpa->pfn = pfn;
1556                /*
1557                 * Do we really change anything ?
1558                 */
1559                if (pte_val(old_pte) != pte_val(new_pte)) {
1560                        set_pte_atomic(kpte, new_pte);
1561                        cpa->flags |= CPA_FLUSHTLB;
1562                }
1563                cpa->numpages = 1;
1564                return 0;
1565        }
1566
1567        /*
1568         * Check, whether we can keep the large page intact
1569         * and just change the pte:
1570         */
1571        do_split = should_split_large_page(kpte, address, cpa);
1572        /*
1573         * When the range fits into the existing large page,
1574         * return. cp->numpages and cpa->tlbflush have been updated in
1575         * try_large_page:
1576         */
1577        if (do_split <= 0)
1578                return do_split;
1579
1580        /*
1581         * We have to split the large page:
1582         */
1583        err = split_large_page(cpa, kpte, address);
1584        if (!err)
1585                goto repeat;
1586
1587        return err;
1588}
1589
1590static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
1591
1592static int cpa_process_alias(struct cpa_data *cpa)
1593{
1594        struct cpa_data alias_cpa;
1595        unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
1596        unsigned long vaddr;
1597        int ret;
1598
1599        if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
1600                return 0;
1601
1602        /*
1603         * No need to redo, when the primary call touched the direct
1604         * mapping already:
1605         */
1606        vaddr = __cpa_addr(cpa, cpa->curpage);
1607        if (!(within(vaddr, PAGE_OFFSET,
1608                    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
1609
1610                alias_cpa = *cpa;
1611                alias_cpa.vaddr = &laddr;
1612                alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1613                alias_cpa.curpage = 0;
1614
1615                cpa->force_flush_all = 1;
1616
1617                ret = __change_page_attr_set_clr(&alias_cpa, 0);
1618                if (ret)
1619                        return ret;
1620        }
1621
1622#ifdef CONFIG_X86_64
1623        /*
1624         * If the primary call didn't touch the high mapping already
1625         * and the physical address is inside the kernel map, we need
1626         * to touch the high mapped kernel as well:
1627         */
1628        if (!within(vaddr, (unsigned long)_text, _brk_end) &&
1629            __cpa_pfn_in_highmap(cpa->pfn)) {
1630                unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
1631                                               __START_KERNEL_map - phys_base;
1632                alias_cpa = *cpa;
1633                alias_cpa.vaddr = &temp_cpa_vaddr;
1634                alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1635                alias_cpa.curpage = 0;
1636
1637                cpa->force_flush_all = 1;
1638                /*
1639                 * The high mapping range is imprecise, so ignore the
1640                 * return value.
1641                 */
1642                __change_page_attr_set_clr(&alias_cpa, 0);
1643        }
1644#endif
1645
1646        return 0;
1647}
1648
1649static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
1650{
1651        unsigned long numpages = cpa->numpages;
1652        unsigned long rempages = numpages;
1653        int ret = 0;
1654
1655        while (rempages) {
1656                /*
1657                 * Store the remaining nr of pages for the large page
1658                 * preservation check.
1659                 */
1660                cpa->numpages = rempages;
1661                /* for array changes, we can't use large page */
1662                if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
1663                        cpa->numpages = 1;
1664
1665                if (!debug_pagealloc_enabled())
1666                        spin_lock(&cpa_lock);
1667                ret = __change_page_attr(cpa, checkalias);
1668                if (!debug_pagealloc_enabled())
1669                        spin_unlock(&cpa_lock);
1670                if (ret)
1671                        goto out;
1672
1673                if (checkalias) {
1674                        ret = cpa_process_alias(cpa);
1675                        if (ret)
1676                                goto out;
1677                }
1678
1679                /*
1680                 * Adjust the number of pages with the result of the
1681                 * CPA operation. Either a large page has been
1682                 * preserved or a single page update happened.
1683                 */
1684                BUG_ON(cpa->numpages > rempages || !cpa->numpages);
1685                rempages -= cpa->numpages;
1686                cpa->curpage += cpa->numpages;
1687        }
1688
1689out:
1690        /* Restore the original numpages */
1691        cpa->numpages = numpages;
1692        return ret;
1693}
1694
1695static int change_page_attr_set_clr(unsigned long *addr, int numpages,
1696                                    pgprot_t mask_set, pgprot_t mask_clr,
1697                                    int force_split, int in_flag,
1698                                    struct page **pages)
1699{
1700        struct cpa_data cpa;
1701        int ret, cache, checkalias;
1702
1703        memset(&cpa, 0, sizeof(cpa));
1704
1705        /*
1706         * Check, if we are requested to set a not supported
1707         * feature.  Clearing non-supported features is OK.
1708         */
1709        mask_set = canon_pgprot(mask_set);
1710
1711        if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
1712                return 0;
1713
1714        /* Ensure we are PAGE_SIZE aligned */
1715        if (in_flag & CPA_ARRAY) {
1716                int i;
1717                for (i = 0; i < numpages; i++) {
1718                        if (addr[i] & ~PAGE_MASK) {
1719                                addr[i] &= PAGE_MASK;
1720                                WARN_ON_ONCE(1);
1721                        }
1722                }
1723        } else if (!(in_flag & CPA_PAGES_ARRAY)) {
1724                /*
1725                 * in_flag of CPA_PAGES_ARRAY implies it is aligned.
1726                 * No need to check in that case
1727                 */
1728                if (*addr & ~PAGE_MASK) {
1729                        *addr &= PAGE_MASK;
1730                        /*
1731                         * People should not be passing in unaligned addresses:
1732                         */
1733                        WARN_ON_ONCE(1);
1734                }
1735        }
1736
1737        /* Must avoid aliasing mappings in the highmem code */
1738        kmap_flush_unused();
1739
1740        vm_unmap_aliases();
1741
1742        cpa.vaddr = addr;
1743        cpa.pages = pages;
1744        cpa.numpages = numpages;
1745        cpa.mask_set = mask_set;
1746        cpa.mask_clr = mask_clr;
1747        cpa.flags = 0;
1748        cpa.curpage = 0;
1749        cpa.force_split = force_split;
1750
1751        if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
1752                cpa.flags |= in_flag;
1753
1754        /* No alias checking for _NX bit modifications */
1755        checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
1756        /* Has caller explicitly disabled alias checking? */
1757        if (in_flag & CPA_NO_CHECK_ALIAS)
1758                checkalias = 0;
1759
1760        ret = __change_page_attr_set_clr(&cpa, checkalias);
1761
1762        /*
1763         * Check whether we really changed something:
1764         */
1765        if (!(cpa.flags & CPA_FLUSHTLB))
1766                goto out;
1767
1768        /*
1769         * No need to flush, when we did not set any of the caching
1770         * attributes:
1771         */
1772        cache = !!pgprot2cachemode(mask_set);
1773
1774        /*
1775         * On error; flush everything to be sure.
1776         */
1777        if (ret) {
1778                cpa_flush_all(cache);
1779                goto out;
1780        }
1781
1782        cpa_flush(&cpa, cache);
1783out:
1784        return ret;
1785}
1786
1787static inline int change_page_attr_set(unsigned long *addr, int numpages,
1788                                       pgprot_t mask, int array)
1789{
1790        return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
1791                (array ? CPA_ARRAY : 0), NULL);
1792}
1793
1794static inline int change_page_attr_clear(unsigned long *addr, int numpages,
1795                                         pgprot_t mask, int array)
1796{
1797        return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
1798                (array ? CPA_ARRAY : 0), NULL);
1799}
1800
1801static inline int cpa_set_pages_array(struct page **pages, int numpages,
1802                                       pgprot_t mask)
1803{
1804        return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
1805                CPA_PAGES_ARRAY, pages);
1806}
1807
1808static inline int cpa_clear_pages_array(struct page **pages, int numpages,
1809                                         pgprot_t mask)
1810{
1811        return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
1812                CPA_PAGES_ARRAY, pages);
1813}
1814
1815/*
1816 * _set_memory_prot is an internal helper for callers that have been passed
1817 * a pgprot_t value from upper layers and a reservation has already been taken.
1818 * If you want to set the pgprot to a specific page protocol, use the
1819 * set_memory_xx() functions.
1820 */
1821int __set_memory_prot(unsigned long addr, int numpages, pgprot_t prot)
1822{
1823        return change_page_attr_set_clr(&addr, numpages, prot,
1824                                        __pgprot(~pgprot_val(prot)), 0, 0,
1825                                        NULL);
1826}
1827
1828int _set_memory_uc(unsigned long addr, int numpages)
1829{
1830        /*
1831         * for now UC MINUS. see comments in ioremap()
1832         * If you really need strong UC use ioremap_uc(), but note
1833         * that you cannot override IO areas with set_memory_*() as
1834         * these helpers cannot work with IO memory.
1835         */
1836        return change_page_attr_set(&addr, numpages,
1837                                    cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1838                                    0);
1839}
1840
1841int set_memory_uc(unsigned long addr, int numpages)
1842{
1843        int ret;
1844
1845        /*
1846         * for now UC MINUS. see comments in ioremap()
1847         */
1848        ret = memtype_reserve(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1849                              _PAGE_CACHE_MODE_UC_MINUS, NULL);
1850        if (ret)
1851                goto out_err;
1852
1853        ret = _set_memory_uc(addr, numpages);
1854        if (ret)
1855                goto out_free;
1856
1857        return 0;
1858
1859out_free:
1860        memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1861out_err:
1862        return ret;
1863}
1864EXPORT_SYMBOL(set_memory_uc);
1865
1866int _set_memory_wc(unsigned long addr, int numpages)
1867{
1868        int ret;
1869
1870        ret = change_page_attr_set(&addr, numpages,
1871                                   cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1872                                   0);
1873        if (!ret) {
1874                ret = change_page_attr_set_clr(&addr, numpages,
1875                                               cachemode2pgprot(_PAGE_CACHE_MODE_WC),
1876                                               __pgprot(_PAGE_CACHE_MASK),
1877                                               0, 0, NULL);
1878        }
1879        return ret;
1880}
1881
1882int set_memory_wc(unsigned long addr, int numpages)
1883{
1884        int ret;
1885
1886        ret = memtype_reserve(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1887                _PAGE_CACHE_MODE_WC, NULL);
1888        if (ret)
1889                return ret;
1890
1891        ret = _set_memory_wc(addr, numpages);
1892        if (ret)
1893                memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1894
1895        return ret;
1896}
1897EXPORT_SYMBOL(set_memory_wc);
1898
1899int _set_memory_wt(unsigned long addr, int numpages)
1900{
1901        return change_page_attr_set(&addr, numpages,
1902                                    cachemode2pgprot(_PAGE_CACHE_MODE_WT), 0);
1903}
1904
1905int _set_memory_wb(unsigned long addr, int numpages)
1906{
1907        /* WB cache mode is hard wired to all cache attribute bits being 0 */
1908        return change_page_attr_clear(&addr, numpages,
1909                                      __pgprot(_PAGE_CACHE_MASK), 0);
1910}
1911
1912int set_memory_wb(unsigned long addr, int numpages)
1913{
1914        int ret;
1915
1916        ret = _set_memory_wb(addr, numpages);
1917        if (ret)
1918                return ret;
1919
1920        memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1921        return 0;
1922}
1923EXPORT_SYMBOL(set_memory_wb);
1924
1925int set_memory_x(unsigned long addr, int numpages)
1926{
1927        if (!(__supported_pte_mask & _PAGE_NX))
1928                return 0;
1929
1930        return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
1931}
1932
1933int set_memory_nx(unsigned long addr, int numpages)
1934{
1935        if (!(__supported_pte_mask & _PAGE_NX))
1936                return 0;
1937
1938        return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
1939}
1940
1941int set_memory_ro(unsigned long addr, int numpages)
1942{
1943        return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
1944}
1945
1946int set_memory_rw(unsigned long addr, int numpages)
1947{
1948        return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
1949}
1950
1951int set_memory_np(unsigned long addr, int numpages)
1952{
1953        return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
1954}
1955
1956int set_memory_np_noalias(unsigned long addr, int numpages)
1957{
1958        int cpa_flags = CPA_NO_CHECK_ALIAS;
1959
1960        return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
1961                                        __pgprot(_PAGE_PRESENT), 0,
1962                                        cpa_flags, NULL);
1963}
1964
1965int set_memory_4k(unsigned long addr, int numpages)
1966{
1967        return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
1968                                        __pgprot(0), 1, 0, NULL);
1969}
1970
1971int set_memory_nonglobal(unsigned long addr, int numpages)
1972{
1973        return change_page_attr_clear(&addr, numpages,
1974                                      __pgprot(_PAGE_GLOBAL), 0);
1975}
1976
1977int set_memory_global(unsigned long addr, int numpages)
1978{
1979        return change_page_attr_set(&addr, numpages,
1980                                    __pgprot(_PAGE_GLOBAL), 0);
1981}
1982
1983static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
1984{
1985        struct cpa_data cpa;
1986        int ret;
1987
1988        /* Nothing to do if memory encryption is not active */
1989        if (!mem_encrypt_active())
1990                return 0;
1991
1992        /* Should not be working on unaligned addresses */
1993        if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr))
1994                addr &= PAGE_MASK;
1995
1996        memset(&cpa, 0, sizeof(cpa));
1997        cpa.vaddr = &addr;
1998        cpa.numpages = numpages;
1999        cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0);
2000        cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC);
2001        cpa.pgd = init_mm.pgd;
2002
2003        /* Must avoid aliasing mappings in the highmem code */
2004        kmap_flush_unused();
2005        vm_unmap_aliases();
2006
2007        /*
2008         * Before changing the encryption attribute, we need to flush caches.
2009         */
2010        cpa_flush(&cpa, !this_cpu_has(X86_FEATURE_SME_COHERENT));
2011
2012        ret = __change_page_attr_set_clr(&cpa, 1);
2013
2014        /*
2015         * After changing the encryption attribute, we need to flush TLBs again
2016         * in case any speculative TLB caching occurred (but no need to flush
2017         * caches again).  We could just use cpa_flush_all(), but in case TLB
2018         * flushing gets optimized in the cpa_flush() path use the same logic
2019         * as above.
2020         */
2021        cpa_flush(&cpa, 0);
2022
2023        return ret;
2024}
2025
2026int set_memory_encrypted(unsigned long addr, int numpages)
2027{
2028        return __set_memory_enc_dec(addr, numpages, true);
2029}
2030EXPORT_SYMBOL_GPL(set_memory_encrypted);
2031
2032int set_memory_decrypted(unsigned long addr, int numpages)
2033{
2034        return __set_memory_enc_dec(addr, numpages, false);
2035}
2036EXPORT_SYMBOL_GPL(set_memory_decrypted);
2037
2038int set_pages_uc(struct page *page, int numpages)
2039{
2040        unsigned long addr = (unsigned long)page_address(page);
2041
2042        return set_memory_uc(addr, numpages);
2043}
2044EXPORT_SYMBOL(set_pages_uc);
2045
2046static int _set_pages_array(struct page **pages, int numpages,
2047                enum page_cache_mode new_type)
2048{
2049        unsigned long start;
2050        unsigned long end;
2051        enum page_cache_mode set_type;
2052        int i;
2053        int free_idx;
2054        int ret;
2055
2056        for (i = 0; i < numpages; i++) {
2057                if (PageHighMem(pages[i]))
2058                        continue;
2059                start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2060                end = start + PAGE_SIZE;
2061                if (memtype_reserve(start, end, new_type, NULL))
2062                        goto err_out;
2063        }
2064
2065        /* If WC, set to UC- first and then WC */
2066        set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
2067                                _PAGE_CACHE_MODE_UC_MINUS : new_type;
2068
2069        ret = cpa_set_pages_array(pages, numpages,
2070                                  cachemode2pgprot(set_type));
2071        if (!ret && new_type == _PAGE_CACHE_MODE_WC)
2072                ret = change_page_attr_set_clr(NULL, numpages,
2073                                               cachemode2pgprot(
2074                                                _PAGE_CACHE_MODE_WC),
2075                                               __pgprot(_PAGE_CACHE_MASK),
2076                                               0, CPA_PAGES_ARRAY, pages);
2077        if (ret)
2078                goto err_out;
2079        return 0; /* Success */
2080err_out:
2081        free_idx = i;
2082        for (i = 0; i < free_idx; i++) {
2083                if (PageHighMem(pages[i]))
2084                        continue;
2085                start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2086                end = start + PAGE_SIZE;
2087                memtype_free(start, end);
2088        }
2089        return -EINVAL;
2090}
2091
2092int set_pages_array_uc(struct page **pages, int numpages)
2093{
2094        return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_UC_MINUS);
2095}
2096EXPORT_SYMBOL(set_pages_array_uc);
2097
2098int set_pages_array_wc(struct page **pages, int numpages)
2099{
2100        return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WC);
2101}
2102EXPORT_SYMBOL(set_pages_array_wc);
2103
2104int set_pages_array_wt(struct page **pages, int numpages)
2105{
2106        return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WT);
2107}
2108EXPORT_SYMBOL_GPL(set_pages_array_wt);
2109
2110int set_pages_wb(struct page *page, int numpages)
2111{
2112        unsigned long addr = (unsigned long)page_address(page);
2113
2114        return set_memory_wb(addr, numpages);
2115}
2116EXPORT_SYMBOL(set_pages_wb);
2117
2118int set_pages_array_wb(struct page **pages, int numpages)
2119{
2120        int retval;
2121        unsigned long start;
2122        unsigned long end;
2123        int i;
2124
2125        /* WB cache mode is hard wired to all cache attribute bits being 0 */
2126        retval = cpa_clear_pages_array(pages, numpages,
2127                        __pgprot(_PAGE_CACHE_MASK));
2128        if (retval)
2129                return retval;
2130
2131        for (i = 0; i < numpages; i++) {
2132                if (PageHighMem(pages[i]))
2133                        continue;
2134                start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2135                end = start + PAGE_SIZE;
2136                memtype_free(start, end);
2137        }
2138
2139        return 0;
2140}
2141EXPORT_SYMBOL(set_pages_array_wb);
2142
2143int set_pages_ro(struct page *page, int numpages)
2144{
2145        unsigned long addr = (unsigned long)page_address(page);
2146
2147        return set_memory_ro(addr, numpages);
2148}
2149
2150int set_pages_rw(struct page *page, int numpages)
2151{
2152        unsigned long addr = (unsigned long)page_address(page);
2153
2154        return set_memory_rw(addr, numpages);
2155}
2156
2157static int __set_pages_p(struct page *page, int numpages)
2158{
2159        unsigned long tempaddr = (unsigned long) page_address(page);
2160        struct cpa_data cpa = { .vaddr = &tempaddr,
2161                                .pgd = NULL,
2162                                .numpages = numpages,
2163                                .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2164                                .mask_clr = __pgprot(0),
2165                                .flags = 0};
2166
2167        /*
2168         * No alias checking needed for setting present flag. otherwise,
2169         * we may need to break large pages for 64-bit kernel text
2170         * mappings (this adds to complexity if we want to do this from
2171         * atomic context especially). Let's keep it simple!
2172         */
2173        return __change_page_attr_set_clr(&cpa, 0);
2174}
2175
2176static int __set_pages_np(struct page *page, int numpages)
2177{
2178        unsigned long tempaddr = (unsigned long) page_address(page);
2179        struct cpa_data cpa = { .vaddr = &tempaddr,
2180                                .pgd = NULL,
2181                                .numpages = numpages,
2182                                .mask_set = __pgprot(0),
2183                                .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2184                                .flags = 0};
2185
2186        /*
2187         * No alias checking needed for setting not present flag. otherwise,
2188         * we may need to break large pages for 64-bit kernel text
2189         * mappings (this adds to complexity if we want to do this from
2190         * atomic context especially). Let's keep it simple!
2191         */
2192        return __change_page_attr_set_clr(&cpa, 0);
2193}
2194
2195int set_direct_map_invalid_noflush(struct page *page)
2196{
2197        return __set_pages_np(page, 1);
2198}
2199
2200int set_direct_map_default_noflush(struct page *page)
2201{
2202        return __set_pages_p(page, 1);
2203}
2204
2205#ifdef CONFIG_DEBUG_PAGEALLOC
2206void __kernel_map_pages(struct page *page, int numpages, int enable)
2207{
2208        if (PageHighMem(page))
2209                return;
2210        if (!enable) {
2211                debug_check_no_locks_freed(page_address(page),
2212                                           numpages * PAGE_SIZE);
2213        }
2214
2215        /*
2216         * The return value is ignored as the calls cannot fail.
2217         * Large pages for identity mappings are not used at boot time
2218         * and hence no memory allocations during large page split.
2219         */
2220        if (enable)
2221                __set_pages_p(page, numpages);
2222        else
2223                __set_pages_np(page, numpages);
2224
2225        /*
2226         * We should perform an IPI and flush all tlbs,
2227         * but that can deadlock->flush only current cpu.
2228         * Preemption needs to be disabled around __flush_tlb_all() due to
2229         * CR3 reload in __native_flush_tlb().
2230         */
2231        preempt_disable();
2232        __flush_tlb_all();
2233        preempt_enable();
2234
2235        arch_flush_lazy_mmu_mode();
2236}
2237#endif /* CONFIG_DEBUG_PAGEALLOC */
2238
2239bool kernel_page_present(struct page *page)
2240{
2241        unsigned int level;
2242        pte_t *pte;
2243
2244        if (PageHighMem(page))
2245                return false;
2246
2247        pte = lookup_address((unsigned long)page_address(page), &level);
2248        return (pte_val(*pte) & _PAGE_PRESENT);
2249}
2250
2251int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address,
2252                                   unsigned numpages, unsigned long page_flags)
2253{
2254        int retval = -EINVAL;
2255
2256        struct cpa_data cpa = {
2257                .vaddr = &address,
2258                .pfn = pfn,
2259                .pgd = pgd,
2260                .numpages = numpages,
2261                .mask_set = __pgprot(0),
2262                .mask_clr = __pgprot(~page_flags & (_PAGE_NX|_PAGE_RW)),
2263                .flags = 0,
2264        };
2265
2266        WARN_ONCE(num_online_cpus() > 1, "Don't call after initializing SMP");
2267
2268        if (!(__supported_pte_mask & _PAGE_NX))
2269                goto out;
2270
2271        if (!(page_flags & _PAGE_ENC))
2272                cpa.mask_clr = pgprot_encrypted(cpa.mask_clr);
2273
2274        cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);
2275
2276        retval = __change_page_attr_set_clr(&cpa, 0);
2277        __flush_tlb_all();
2278
2279out:
2280        return retval;
2281}
2282
2283/*
2284 * __flush_tlb_all() flushes mappings only on current CPU and hence this
2285 * function shouldn't be used in an SMP environment. Presently, it's used only
2286 * during boot (way before smp_init()) by EFI subsystem and hence is ok.
2287 */
2288int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address,
2289                                     unsigned long numpages)
2290{
2291        int retval;
2292
2293        /*
2294         * The typical sequence for unmapping is to find a pte through
2295         * lookup_address_in_pgd() (ideally, it should never return NULL because
2296         * the address is already mapped) and change it's protections. As pfn is
2297         * the *target* of a mapping, it's not useful while unmapping.
2298         */
2299        struct cpa_data cpa = {
2300                .vaddr          = &address,
2301                .pfn            = 0,
2302                .pgd            = pgd,
2303                .numpages       = numpages,
2304                .mask_set       = __pgprot(0),
2305                .mask_clr       = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2306                .flags          = 0,
2307        };
2308
2309        WARN_ONCE(num_online_cpus() > 1, "Don't call after initializing SMP");
2310
2311        retval = __change_page_attr_set_clr(&cpa, 0);
2312        __flush_tlb_all();
2313
2314        return retval;
2315}
2316
2317/*
2318 * The testcases use internal knowledge of the implementation that shouldn't
2319 * be exposed to the rest of the kernel. Include these directly here.
2320 */
2321#ifdef CONFIG_CPA_DEBUG
2322#include "cpa-test.c"
2323#endif
2324
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.