linux/arch/x86/xen/mmu.c
<<
>>
Prefs
   1/*
   2 * Xen mmu operations
   3 *
   4 * This file contains the various mmu fetch and update operations.
   5 * The most important job they must perform is the mapping between the
   6 * domain's pfn and the overall machine mfns.
   7 *
   8 * Xen allows guests to directly update the pagetable, in a controlled
   9 * fashion.  In other words, the guest modifies the same pagetable
  10 * that the CPU actually uses, which eliminates the overhead of having
  11 * a separate shadow pagetable.
  12 *
  13 * In order to allow this, it falls on the guest domain to map its
  14 * notion of a "physical" pfn - which is just a domain-local linear
  15 * address - into a real "machine address" which the CPU's MMU can
  16 * use.
  17 *
  18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
  19 * inserted directly into the pagetable.  When creating a new
  20 * pte/pmd/pgd, it converts the passed pfn into an mfn.  Conversely,
  21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
  22 * the mfn back into a pfn.
  23 *
  24 * The other constraint is that all pages which make up a pagetable
  25 * must be mapped read-only in the guest.  This prevents uncontrolled
  26 * guest updates to the pagetable.  Xen strictly enforces this, and
  27 * will disallow any pagetable update which will end up mapping a
  28 * pagetable page RW, and will disallow using any writable page as a
  29 * pagetable.
  30 *
  31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
  32 * would need to validate the whole pagetable before going on.
  33 * Naturally, this is quite slow.  The solution is to "pin" a
  34 * pagetable, which enforces all the constraints on the pagetable even
  35 * when it is not actively in use.  This menas that Xen can be assured
  36 * that it is still valid when you do load it into %cr3, and doesn't
  37 * need to revalidate it.
  38 *
  39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
  40 */
  41#include <linux/sched.h>
  42#include <linux/highmem.h>
  43#include <linux/debugfs.h>
  44#include <linux/bug.h>
  45
  46#include <asm/pgtable.h>
  47#include <asm/tlbflush.h>
  48#include <asm/fixmap.h>
  49#include <asm/mmu_context.h>
  50#include <asm/paravirt.h>
  51#include <asm/linkage.h>
  52
  53#include <asm/xen/hypercall.h>
  54#include <asm/xen/hypervisor.h>
  55
  56#include <xen/page.h>
  57#include <xen/interface/xen.h>
  58
  59#include "multicalls.h"
  60#include "mmu.h"
  61#include "debugfs.h"
  62
  63#define MMU_UPDATE_HISTO        30
  64
  65#ifdef CONFIG_XEN_DEBUG_FS
  66
  67static struct {
  68        u32 pgd_update;
  69        u32 pgd_update_pinned;
  70        u32 pgd_update_batched;
  71
  72        u32 pud_update;
  73        u32 pud_update_pinned;
  74        u32 pud_update_batched;
  75
  76        u32 pmd_update;
  77        u32 pmd_update_pinned;
  78        u32 pmd_update_batched;
  79
  80        u32 pte_update;
  81        u32 pte_update_pinned;
  82        u32 pte_update_batched;
  83
  84        u32 mmu_update;
  85        u32 mmu_update_extended;
  86        u32 mmu_update_histo[MMU_UPDATE_HISTO];
  87
  88        u32 prot_commit;
  89        u32 prot_commit_batched;
  90
  91        u32 set_pte_at;
  92        u32 set_pte_at_batched;
  93        u32 set_pte_at_pinned;
  94        u32 set_pte_at_current;
  95        u32 set_pte_at_kernel;
  96} mmu_stats;
  97
  98static u8 zero_stats;
  99
 100static inline void check_zero(void)
 101{
 102        if (unlikely(zero_stats)) {
 103                memset(&mmu_stats, 0, sizeof(mmu_stats));
 104                zero_stats = 0;
 105        }
 106}
 107
 108#define ADD_STATS(elem, val)                    \
 109        do { check_zero(); mmu_stats.elem += (val); } while(0)
 110
 111#else  /* !CONFIG_XEN_DEBUG_FS */
 112
 113#define ADD_STATS(elem, val)    do { (void)(val); } while(0)
 114
 115#endif /* CONFIG_XEN_DEBUG_FS */
 116
 117/*
 118 * Just beyond the highest usermode address.  STACK_TOP_MAX has a
 119 * redzone above it, so round it up to a PGD boundary.
 120 */
 121#define USER_LIMIT      ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
 122
 123
 124#define P2M_ENTRIES_PER_PAGE    (PAGE_SIZE / sizeof(unsigned long))
 125#define TOP_ENTRIES             (MAX_DOMAIN_PAGES / P2M_ENTRIES_PER_PAGE)
 126
 127/* Placeholder for holes in the address space */
 128static unsigned long p2m_missing[P2M_ENTRIES_PER_PAGE] __page_aligned_data =
 129                { [ 0 ... P2M_ENTRIES_PER_PAGE-1 ] = ~0UL };
 130
 131 /* Array of pointers to pages containing p2m entries */
 132static unsigned long *p2m_top[TOP_ENTRIES] __page_aligned_data =
 133                { [ 0 ... TOP_ENTRIES - 1] = &p2m_missing[0] };
 134
 135/* Arrays of p2m arrays expressed in mfns used for save/restore */
 136static unsigned long p2m_top_mfn[TOP_ENTRIES] __page_aligned_bss;
 137
 138static unsigned long p2m_top_mfn_list[TOP_ENTRIES / P2M_ENTRIES_PER_PAGE]
 139        __page_aligned_bss;
 140
 141static inline unsigned p2m_top_index(unsigned long pfn)
 142{
 143        BUG_ON(pfn >= MAX_DOMAIN_PAGES);
 144        return pfn / P2M_ENTRIES_PER_PAGE;
 145}
 146
 147static inline unsigned p2m_index(unsigned long pfn)
 148{
 149        return pfn % P2M_ENTRIES_PER_PAGE;
 150}
 151
 152/* Build the parallel p2m_top_mfn structures */
 153void xen_setup_mfn_list_list(void)
 154{
 155        unsigned pfn, idx;
 156
 157        for(pfn = 0; pfn < MAX_DOMAIN_PAGES; pfn += P2M_ENTRIES_PER_PAGE) {
 158                unsigned topidx = p2m_top_index(pfn);
 159
 160                p2m_top_mfn[topidx] = virt_to_mfn(p2m_top[topidx]);
 161        }
 162
 163        for(idx = 0; idx < ARRAY_SIZE(p2m_top_mfn_list); idx++) {
 164                unsigned topidx = idx * P2M_ENTRIES_PER_PAGE;
 165                p2m_top_mfn_list[idx] = virt_to_mfn(&p2m_top_mfn[topidx]);
 166        }
 167
 168        BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
 169
 170        HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
 171                virt_to_mfn(p2m_top_mfn_list);
 172        HYPERVISOR_shared_info->arch.max_pfn = xen_start_info->nr_pages;
 173}
 174
 175/* Set up p2m_top to point to the domain-builder provided p2m pages */
 176void __init xen_build_dynamic_phys_to_machine(void)
 177{
 178        unsigned long *mfn_list = (unsigned long *)xen_start_info->mfn_list;
 179        unsigned long max_pfn = min(MAX_DOMAIN_PAGES, xen_start_info->nr_pages);
 180        unsigned pfn;
 181
 182        for(pfn = 0; pfn < max_pfn; pfn += P2M_ENTRIES_PER_PAGE) {
 183                unsigned topidx = p2m_top_index(pfn);
 184
 185                p2m_top[topidx] = &mfn_list[pfn];
 186        }
 187}
 188
 189unsigned long get_phys_to_machine(unsigned long pfn)
 190{
 191        unsigned topidx, idx;
 192
 193        if (unlikely(pfn >= MAX_DOMAIN_PAGES))
 194                return INVALID_P2M_ENTRY;
 195
 196        topidx = p2m_top_index(pfn);
 197        idx = p2m_index(pfn);
 198        return p2m_top[topidx][idx];
 199}
 200EXPORT_SYMBOL_GPL(get_phys_to_machine);
 201
 202static void alloc_p2m(unsigned long **pp, unsigned long *mfnp)
 203{
 204        unsigned long *p;
 205        unsigned i;
 206
 207        p = (void *)__get_free_page(GFP_KERNEL | __GFP_NOFAIL);
 208        BUG_ON(p == NULL);
 209
 210        for(i = 0; i < P2M_ENTRIES_PER_PAGE; i++)
 211                p[i] = INVALID_P2M_ENTRY;
 212
 213        if (cmpxchg(pp, p2m_missing, p) != p2m_missing)
 214                free_page((unsigned long)p);
 215        else
 216                *mfnp = virt_to_mfn(p);
 217}
 218
 219void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
 220{
 221        unsigned topidx, idx;
 222
 223        if (unlikely(xen_feature(XENFEAT_auto_translated_physmap))) {
 224                BUG_ON(pfn != mfn && mfn != INVALID_P2M_ENTRY);
 225                return;
 226        }
 227
 228        if (unlikely(pfn >= MAX_DOMAIN_PAGES)) {
 229                BUG_ON(mfn != INVALID_P2M_ENTRY);
 230                return;
 231        }
 232
 233        topidx = p2m_top_index(pfn);
 234        if (p2m_top[topidx] == p2m_missing) {
 235                /* no need to allocate a page to store an invalid entry */
 236                if (mfn == INVALID_P2M_ENTRY)
 237                        return;
 238                alloc_p2m(&p2m_top[topidx], &p2m_top_mfn[topidx]);
 239        }
 240
 241        idx = p2m_index(pfn);
 242        p2m_top[topidx][idx] = mfn;
 243}
 244
 245xmaddr_t arbitrary_virt_to_machine(void *vaddr)
 246{
 247        unsigned long address = (unsigned long)vaddr;
 248        unsigned int level;
 249        pte_t *pte;
 250        unsigned offset;
 251
 252        /*
 253         * if the PFN is in the linear mapped vaddr range, we can just use
 254         * the (quick) virt_to_machine() p2m lookup
 255         */
 256        if (virt_addr_valid(vaddr))
 257                return virt_to_machine(vaddr);
 258
 259        /* otherwise we have to do a (slower) full page-table walk */
 260
 261        pte = lookup_address(address, &level);
 262        BUG_ON(pte == NULL);
 263        offset = address & ~PAGE_MASK;
 264        return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
 265}
 266
 267void make_lowmem_page_readonly(void *vaddr)
 268{
 269        pte_t *pte, ptev;
 270        unsigned long address = (unsigned long)vaddr;
 271        unsigned int level;
 272
 273        pte = lookup_address(address, &level);
 274        BUG_ON(pte == NULL);
 275
 276        ptev = pte_wrprotect(*pte);
 277
 278        if (HYPERVISOR_update_va_mapping(address, ptev, 0))
 279                BUG();
 280}
 281
 282void make_lowmem_page_readwrite(void *vaddr)
 283{
 284        pte_t *pte, ptev;
 285        unsigned long address = (unsigned long)vaddr;
 286        unsigned int level;
 287
 288        pte = lookup_address(address, &level);
 289        BUG_ON(pte == NULL);
 290
 291        ptev = pte_mkwrite(*pte);
 292
 293        if (HYPERVISOR_update_va_mapping(address, ptev, 0))
 294                BUG();
 295}
 296
 297
 298static bool xen_page_pinned(void *ptr)
 299{
 300        struct page *page = virt_to_page(ptr);
 301
 302        return PagePinned(page);
 303}
 304
 305static void xen_extend_mmu_update(const struct mmu_update *update)
 306{
 307        struct multicall_space mcs;
 308        struct mmu_update *u;
 309
 310        mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
 311
 312        if (mcs.mc != NULL) {
 313                ADD_STATS(mmu_update_extended, 1);
 314                ADD_STATS(mmu_update_histo[mcs.mc->args[1]], -1);
 315
 316                mcs.mc->args[1]++;
 317
 318                if (mcs.mc->args[1] < MMU_UPDATE_HISTO)
 319                        ADD_STATS(mmu_update_histo[mcs.mc->args[1]], 1);
 320                else
 321                        ADD_STATS(mmu_update_histo[0], 1);
 322        } else {
 323                ADD_STATS(mmu_update, 1);
 324                mcs = __xen_mc_entry(sizeof(*u));
 325                MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
 326                ADD_STATS(mmu_update_histo[1], 1);
 327        }
 328
 329        u = mcs.args;
 330        *u = *update;
 331}
 332
 333void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
 334{
 335        struct mmu_update u;
 336
 337        preempt_disable();
 338
 339        xen_mc_batch();
 340
 341        /* ptr may be ioremapped for 64-bit pagetable setup */
 342        u.ptr = arbitrary_virt_to_machine(ptr).maddr;
 343        u.val = pmd_val_ma(val);
 344        xen_extend_mmu_update(&u);
 345
 346        ADD_STATS(pmd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
 347
 348        xen_mc_issue(PARAVIRT_LAZY_MMU);
 349
 350        preempt_enable();
 351}
 352
 353void xen_set_pmd(pmd_t *ptr, pmd_t val)
 354{
 355        ADD_STATS(pmd_update, 1);
 356
 357        /* If page is not pinned, we can just update the entry
 358           directly */
 359        if (!xen_page_pinned(ptr)) {
 360                *ptr = val;
 361                return;
 362        }
 363
 364        ADD_STATS(pmd_update_pinned, 1);
 365
 366        xen_set_pmd_hyper(ptr, val);
 367}
 368
 369/*
 370 * Associate a virtual page frame with a given physical page frame
 371 * and protection flags for that frame.
 372 */
 373void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
 374{
 375        set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
 376}
 377
 378void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
 379                    pte_t *ptep, pte_t pteval)
 380{
 381        /* updates to init_mm may be done without lock */
 382        if (mm == &init_mm)
 383                preempt_disable();
 384
 385        ADD_STATS(set_pte_at, 1);
 386//      ADD_STATS(set_pte_at_pinned, xen_page_pinned(ptep));
 387        ADD_STATS(set_pte_at_current, mm == current->mm);
 388        ADD_STATS(set_pte_at_kernel, mm == &init_mm);
 389
 390        if (mm == current->mm || mm == &init_mm) {
 391                if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
 392                        struct multicall_space mcs;
 393                        mcs = xen_mc_entry(0);
 394
 395                        MULTI_update_va_mapping(mcs.mc, addr, pteval, 0);
 396                        ADD_STATS(set_pte_at_batched, 1);
 397                        xen_mc_issue(PARAVIRT_LAZY_MMU);
 398                        goto out;
 399                } else
 400                        if (HYPERVISOR_update_va_mapping(addr, pteval, 0) == 0)
 401                                goto out;
 402        }
 403        xen_set_pte(ptep, pteval);
 404
 405out:
 406        if (mm == &init_mm)
 407                preempt_enable();
 408}
 409
 410pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 411{
 412        /* Just return the pte as-is.  We preserve the bits on commit */
 413        return *ptep;
 414}
 415
 416void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
 417                                 pte_t *ptep, pte_t pte)
 418{
 419        struct mmu_update u;
 420
 421        xen_mc_batch();
 422
 423        u.ptr = arbitrary_virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
 424        u.val = pte_val_ma(pte);
 425        xen_extend_mmu_update(&u);
 426
 427        ADD_STATS(prot_commit, 1);
 428        ADD_STATS(prot_commit_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
 429
 430        xen_mc_issue(PARAVIRT_LAZY_MMU);
 431}
 432
 433/* Assume pteval_t is equivalent to all the other *val_t types. */
 434static pteval_t pte_mfn_to_pfn(pteval_t val)
 435{
 436        if (val & _PAGE_PRESENT) {
 437                unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
 438                pteval_t flags = val & PTE_FLAGS_MASK;
 439                val = ((pteval_t)mfn_to_pfn(mfn) << PAGE_SHIFT) | flags;
 440        }
 441
 442        return val;
 443}
 444
 445static pteval_t pte_pfn_to_mfn(pteval_t val)
 446{
 447        if (val & _PAGE_PRESENT) {
 448                unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
 449                pteval_t flags = val & PTE_FLAGS_MASK;
 450                val = ((pteval_t)pfn_to_mfn(pfn) << PAGE_SHIFT) | flags;
 451        }
 452
 453        return val;
 454}
 455
 456pteval_t xen_pte_val(pte_t pte)
 457{
 458        return pte_mfn_to_pfn(pte.pte);
 459}
 460
 461pgdval_t xen_pgd_val(pgd_t pgd)
 462{
 463        return pte_mfn_to_pfn(pgd.pgd);
 464}
 465
 466pte_t xen_make_pte(pteval_t pte)
 467{
 468        pte = pte_pfn_to_mfn(pte);
 469        return native_make_pte(pte);
 470}
 471
 472pgd_t xen_make_pgd(pgdval_t pgd)
 473{
 474        pgd = pte_pfn_to_mfn(pgd);
 475        return native_make_pgd(pgd);
 476}
 477
 478pmdval_t xen_pmd_val(pmd_t pmd)
 479{
 480        return pte_mfn_to_pfn(pmd.pmd);
 481}
 482
 483void xen_set_pud_hyper(pud_t *ptr, pud_t val)
 484{
 485        struct mmu_update u;
 486
 487        preempt_disable();
 488
 489        xen_mc_batch();
 490
 491        /* ptr may be ioremapped for 64-bit pagetable setup */
 492        u.ptr = arbitrary_virt_to_machine(ptr).maddr;
 493        u.val = pud_val_ma(val);
 494        xen_extend_mmu_update(&u);
 495
 496        ADD_STATS(pud_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
 497
 498        xen_mc_issue(PARAVIRT_LAZY_MMU);
 499
 500        preempt_enable();
 501}
 502
 503void xen_set_pud(pud_t *ptr, pud_t val)
 504{
 505        ADD_STATS(pud_update, 1);
 506
 507        /* If page is not pinned, we can just update the entry
 508           directly */
 509        if (!xen_page_pinned(ptr)) {
 510                *ptr = val;
 511                return;
 512        }
 513
 514        ADD_STATS(pud_update_pinned, 1);
 515
 516        xen_set_pud_hyper(ptr, val);
 517}
 518
 519void xen_set_pte(pte_t *ptep, pte_t pte)
 520{
 521        ADD_STATS(pte_update, 1);
 522//      ADD_STATS(pte_update_pinned, xen_page_pinned(ptep));
 523        ADD_STATS(pte_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
 524
 525#ifdef CONFIG_X86_PAE
 526        ptep->pte_high = pte.pte_high;
 527        smp_wmb();
 528        ptep->pte_low = pte.pte_low;
 529#else
 530        *ptep = pte;
 531#endif
 532}
 533
 534#ifdef CONFIG_X86_PAE
 535void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
 536{
 537        set_64bit((u64 *)ptep, native_pte_val(pte));
 538}
 539
 540void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 541{
 542        ptep->pte_low = 0;
 543        smp_wmb();              /* make sure low gets written first */
 544        ptep->pte_high = 0;
 545}
 546
 547void xen_pmd_clear(pmd_t *pmdp)
 548{
 549        set_pmd(pmdp, __pmd(0));
 550}
 551#endif  /* CONFIG_X86_PAE */
 552
 553pmd_t xen_make_pmd(pmdval_t pmd)
 554{
 555        pmd = pte_pfn_to_mfn(pmd);
 556        return native_make_pmd(pmd);
 557}
 558
 559#if PAGETABLE_LEVELS == 4
 560pudval_t xen_pud_val(pud_t pud)
 561{
 562        return pte_mfn_to_pfn(pud.pud);
 563}
 564
 565pud_t xen_make_pud(pudval_t pud)
 566{
 567        pud = pte_pfn_to_mfn(pud);
 568
 569        return native_make_pud(pud);
 570}
 571
 572pgd_t *xen_get_user_pgd(pgd_t *pgd)
 573{
 574        pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
 575        unsigned offset = pgd - pgd_page;
 576        pgd_t *user_ptr = NULL;
 577
 578        if (offset < pgd_index(USER_LIMIT)) {
 579                struct page *page = virt_to_page(pgd_page);
 580                user_ptr = (pgd_t *)page->private;
 581                if (user_ptr)
 582                        user_ptr += offset;
 583        }
 584
 585        return user_ptr;
 586}
 587
 588static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
 589{
 590        struct mmu_update u;
 591
 592        u.ptr = virt_to_machine(ptr).maddr;
 593        u.val = pgd_val_ma(val);
 594        xen_extend_mmu_update(&u);
 595}
 596
 597/*
 598 * Raw hypercall-based set_pgd, intended for in early boot before
 599 * there's a page structure.  This implies:
 600 *  1. The only existing pagetable is the kernel's
 601 *  2. It is always pinned
 602 *  3. It has no user pagetable attached to it
 603 */
 604void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
 605{
 606        preempt_disable();
 607
 608        xen_mc_batch();
 609
 610        __xen_set_pgd_hyper(ptr, val);
 611
 612        xen_mc_issue(PARAVIRT_LAZY_MMU);
 613
 614        preempt_enable();
 615}
 616
 617void xen_set_pgd(pgd_t *ptr, pgd_t val)
 618{
 619        pgd_t *user_ptr = xen_get_user_pgd(ptr);
 620
 621        ADD_STATS(pgd_update, 1);
 622
 623        /* If page is not pinned, we can just update the entry
 624           directly */
 625        if (!xen_page_pinned(ptr)) {
 626                *ptr = val;
 627                if (user_ptr) {
 628                        WARN_ON(xen_page_pinned(user_ptr));
 629                        *user_ptr = val;
 630                }
 631                return;
 632        }
 633
 634        ADD_STATS(pgd_update_pinned, 1);
 635        ADD_STATS(pgd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
 636
 637        /* If it's pinned, then we can at least batch the kernel and
 638           user updates together. */
 639        xen_mc_batch();
 640
 641        __xen_set_pgd_hyper(ptr, val);
 642        if (user_ptr)
 643                __xen_set_pgd_hyper(user_ptr, val);
 644
 645        xen_mc_issue(PARAVIRT_LAZY_MMU);
 646}
 647#endif  /* PAGETABLE_LEVELS == 4 */
 648
 649/*
 650 * (Yet another) pagetable walker.  This one is intended for pinning a
 651 * pagetable.  This means that it walks a pagetable and calls the
 652 * callback function on each page it finds making up the page table,
 653 * at every level.  It walks the entire pagetable, but it only bothers
 654 * pinning pte pages which are below limit.  In the normal case this
 655 * will be STACK_TOP_MAX, but at boot we need to pin up to
 656 * FIXADDR_TOP.
 657 *
 658 * For 32-bit the important bit is that we don't pin beyond there,
 659 * because then we start getting into Xen's ptes.
 660 *
 661 * For 64-bit, we must skip the Xen hole in the middle of the address
 662 * space, just after the big x86-64 virtual hole.
 663 */
 664static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
 665                          int (*func)(struct mm_struct *mm, struct page *,
 666                                      enum pt_level),
 667                          unsigned long limit)
 668{
 669        int flush = 0;
 670        unsigned hole_low, hole_high;
 671        unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
 672        unsigned pgdidx, pudidx, pmdidx;
 673
 674        /* The limit is the last byte to be touched */
 675        limit--;
 676        BUG_ON(limit >= FIXADDR_TOP);
 677
 678        if (xen_feature(XENFEAT_auto_translated_physmap))
 679                return 0;
 680
 681        /*
 682         * 64-bit has a great big hole in the middle of the address
 683         * space, which contains the Xen mappings.  On 32-bit these
 684         * will end up making a zero-sized hole and so is a no-op.
 685         */
 686        hole_low = pgd_index(USER_LIMIT);
 687        hole_high = pgd_index(PAGE_OFFSET);
 688
 689        pgdidx_limit = pgd_index(limit);
 690#if PTRS_PER_PUD > 1
 691        pudidx_limit = pud_index(limit);
 692#else
 693        pudidx_limit = 0;
 694#endif
 695#if PTRS_PER_PMD > 1
 696        pmdidx_limit = pmd_index(limit);
 697#else
 698        pmdidx_limit = 0;
 699#endif
 700
 701        for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
 702                pud_t *pud;
 703
 704                if (pgdidx >= hole_low && pgdidx < hole_high)
 705                        continue;
 706
 707                if (!pgd_val(pgd[pgdidx]))
 708                        continue;
 709
 710                pud = pud_offset(&pgd[pgdidx], 0);
 711
 712                if (PTRS_PER_PUD > 1) /* not folded */
 713                        flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
 714
 715                for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
 716                        pmd_t *pmd;
 717
 718                        if (pgdidx == pgdidx_limit &&
 719                            pudidx > pudidx_limit)
 720                                goto out;
 721
 722                        if (pud_none(pud[pudidx]))
 723                                continue;
 724
 725                        pmd = pmd_offset(&pud[pudidx], 0);
 726
 727                        if (PTRS_PER_PMD > 1) /* not folded */
 728                                flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
 729
 730                        for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
 731                                struct page *pte;
 732
 733                                if (pgdidx == pgdidx_limit &&
 734                                    pudidx == pudidx_limit &&
 735                                    pmdidx > pmdidx_limit)
 736                                        goto out;
 737
 738                                if (pmd_none(pmd[pmdidx]))
 739                                        continue;
 740
 741                                pte = pmd_page(pmd[pmdidx]);
 742                                flush |= (*func)(mm, pte, PT_PTE);
 743                        }
 744                }
 745        }
 746
 747out:
 748        /* Do the top level last, so that the callbacks can use it as
 749           a cue to do final things like tlb flushes. */
 750        flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
 751
 752        return flush;
 753}
 754
 755static int xen_pgd_walk(struct mm_struct *mm,
 756                        int (*func)(struct mm_struct *mm, struct page *,
 757                                    enum pt_level),
 758                        unsigned long limit)
 759{
 760        return __xen_pgd_walk(mm, mm->pgd, func, limit);
 761}
 762
 763/* If we're using split pte locks, then take the page's lock and
 764   return a pointer to it.  Otherwise return NULL. */
 765static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
 766{
 767        spinlock_t *ptl = NULL;
 768
 769#if USE_SPLIT_PTLOCKS
 770        ptl = __pte_lockptr(page);
 771        spin_lock_nest_lock(ptl, &mm->page_table_lock);
 772#endif
 773
 774        return ptl;
 775}
 776
 777static void xen_pte_unlock(void *v)
 778{
 779        spinlock_t *ptl = v;
 780        spin_unlock(ptl);
 781}
 782
 783static void xen_do_pin(unsigned level, unsigned long pfn)
 784{
 785        struct mmuext_op *op;
 786        struct multicall_space mcs;
 787
 788        mcs = __xen_mc_entry(sizeof(*op));
 789        op = mcs.args;
 790        op->cmd = level;
 791        op->arg1.mfn = pfn_to_mfn(pfn);
 792        MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
 793}
 794
 795static int xen_pin_page(struct mm_struct *mm, struct page *page,
 796                        enum pt_level level)
 797{
 798        unsigned pgfl = TestSetPagePinned(page);
 799        int flush;
 800
 801        if (pgfl)
 802                flush = 0;              /* already pinned */
 803        else if (PageHighMem(page))
 804                /* kmaps need flushing if we found an unpinned
 805                   highpage */
 806                flush = 1;
 807        else {
 808                void *pt = lowmem_page_address(page);
 809                unsigned long pfn = page_to_pfn(page);
 810                struct multicall_space mcs = __xen_mc_entry(0);
 811                spinlock_t *ptl;
 812
 813                flush = 0;
 814
 815                /*
 816                 * We need to hold the pagetable lock between the time
 817                 * we make the pagetable RO and when we actually pin
 818                 * it.  If we don't, then other users may come in and
 819                 * attempt to update the pagetable by writing it,
 820                 * which will fail because the memory is RO but not
 821                 * pinned, so Xen won't do the trap'n'emulate.
 822                 *
 823                 * If we're using split pte locks, we can't hold the
 824                 * entire pagetable's worth of locks during the
 825                 * traverse, because we may wrap the preempt count (8
 826                 * bits).  The solution is to mark RO and pin each PTE
 827                 * page while holding the lock.  This means the number
 828                 * of locks we end up holding is never more than a
 829                 * batch size (~32 entries, at present).
 830                 *
 831                 * If we're not using split pte locks, we needn't pin
 832                 * the PTE pages independently, because we're
 833                 * protected by the overall pagetable lock.
 834                 */
 835                ptl = NULL;
 836                if (level == PT_PTE)
 837                        ptl = xen_pte_lock(page, mm);
 838
 839                MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
 840                                        pfn_pte(pfn, PAGE_KERNEL_RO),
 841                                        level == PT_PGD ? UVMF_TLB_FLUSH : 0);
 842
 843                if (ptl) {
 844                        xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
 845
 846                        /* Queue a deferred unlock for when this batch
 847                           is completed. */
 848                        xen_mc_callback(xen_pte_unlock, ptl);
 849                }
 850        }
 851
 852        return flush;
 853}
 854
 855/* This is called just after a mm has been created, but it has not
 856   been used yet.  We need to make sure that its pagetable is all
 857   read-only, and can be pinned. */
 858static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
 859{
 860        vm_unmap_aliases();
 861
 862        xen_mc_batch();
 863
 864        if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
 865                /* re-enable interrupts for flushing */
 866                xen_mc_issue(0);
 867
 868                kmap_flush_unused();
 869
 870                xen_mc_batch();
 871        }
 872
 873#ifdef CONFIG_X86_64
 874        {
 875                pgd_t *user_pgd = xen_get_user_pgd(pgd);
 876
 877                xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
 878
 879                if (user_pgd) {
 880                        xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
 881                        xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(user_pgd)));
 882                }
 883        }
 884#else /* CONFIG_X86_32 */
 885#ifdef CONFIG_X86_PAE
 886        /* Need to make sure unshared kernel PMD is pinnable */
 887        xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
 888                     PT_PMD);
 889#endif
 890        xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
 891#endif /* CONFIG_X86_64 */
 892        xen_mc_issue(0);
 893}
 894
 895static void xen_pgd_pin(struct mm_struct *mm)
 896{
 897        __xen_pgd_pin(mm, mm->pgd);
 898}
 899
 900/*
 901 * On save, we need to pin all pagetables to make sure they get their
 902 * mfns turned into pfns.  Search the list for any unpinned pgds and pin
 903 * them (unpinned pgds are not currently in use, probably because the
 904 * process is under construction or destruction).
 905 *
 906 * Expected to be called in stop_machine() ("equivalent to taking
 907 * every spinlock in the system"), so the locking doesn't really
 908 * matter all that much.
 909 */
 910void xen_mm_pin_all(void)
 911{
 912        unsigned long flags;
 913        struct page *page;
 914
 915        spin_lock_irqsave(&pgd_lock, flags);
 916
 917        list_for_each_entry(page, &pgd_list, lru) {
 918                if (!PagePinned(page)) {
 919                        __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
 920                        SetPageSavePinned(page);
 921                }
 922        }
 923
 924        spin_unlock_irqrestore(&pgd_lock, flags);
 925}
 926
 927/*
 928 * The init_mm pagetable is really pinned as soon as its created, but
 929 * that's before we have page structures to store the bits.  So do all
 930 * the book-keeping now.
 931 */
 932static __init int xen_mark_pinned(struct mm_struct *mm, struct page *page,
 933                                  enum pt_level level)
 934{
 935        SetPagePinned(page);
 936        return 0;
 937}
 938
 939void __init xen_mark_init_mm_pinned(void)
 940{
 941        xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
 942}
 943
 944static int xen_unpin_page(struct mm_struct *mm, struct page *page,
 945                          enum pt_level level)
 946{
 947        unsigned pgfl = TestClearPagePinned(page);
 948
 949        if (pgfl && !PageHighMem(page)) {
 950                void *pt = lowmem_page_address(page);
 951                unsigned long pfn = page_to_pfn(page);
 952                spinlock_t *ptl = NULL;
 953                struct multicall_space mcs;
 954
 955                /*
 956                 * Do the converse to pin_page.  If we're using split
 957                 * pte locks, we must be holding the lock for while
 958                 * the pte page is unpinned but still RO to prevent
 959                 * concurrent updates from seeing it in this
 960                 * partially-pinned state.
 961                 */
 962                if (level == PT_PTE) {
 963                        ptl = xen_pte_lock(page, mm);
 964
 965                        if (ptl)
 966                                xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
 967                }
 968
 969                mcs = __xen_mc_entry(0);
 970
 971                MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
 972                                        pfn_pte(pfn, PAGE_KERNEL),
 973                                        level == PT_PGD ? UVMF_TLB_FLUSH : 0);
 974
 975                if (ptl) {
 976                        /* unlock when batch completed */
 977                        xen_mc_callback(xen_pte_unlock, ptl);
 978                }
 979        }
 980
 981        return 0;               /* never need to flush on unpin */
 982}
 983
 984/* Release a pagetables pages back as normal RW */
 985static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
 986{
 987        xen_mc_batch();
 988
 989        xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
 990
 991#ifdef CONFIG_X86_64
 992        {
 993                pgd_t *user_pgd = xen_get_user_pgd(pgd);
 994
 995                if (user_pgd) {
 996                        xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(user_pgd)));
 997                        xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
 998                }
 999        }
1000#endif
1001
1002#ifdef CONFIG_X86_PAE
1003        /* Need to make sure unshared kernel PMD is unpinned */
1004        xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
1005                       PT_PMD);
1006#endif
1007
1008        __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
1009
1010        xen_mc_issue(0);
1011}
1012
1013static void xen_pgd_unpin(struct mm_struct *mm)
1014{
1015        __xen_pgd_unpin(mm, mm->pgd);
1016}
1017
1018/*
1019 * On resume, undo any pinning done at save, so that the rest of the
1020 * kernel doesn't see any unexpected pinned pagetables.
1021 */
1022void xen_mm_unpin_all(void)
1023{
1024        unsigned long flags;
1025        struct page *page;
1026
1027        spin_lock_irqsave(&pgd_lock, flags);
1028
1029        list_for_each_entry(page, &pgd_list, lru) {
1030                if (PageSavePinned(page)) {
1031                        BUG_ON(!PagePinned(page));
1032                        __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
1033                        ClearPageSavePinned(page);
1034                }
1035        }
1036
1037        spin_unlock_irqrestore(&pgd_lock, flags);
1038}
1039
1040void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
1041{
1042        spin_lock(&next->page_table_lock);
1043        xen_pgd_pin(next);
1044        spin_unlock(&next->page_table_lock);
1045}
1046
1047void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
1048{
1049        spin_lock(&mm->page_table_lock);
1050        xen_pgd_pin(mm);
1051        spin_unlock(&mm->page_table_lock);
1052}
1053
1054
1055#ifdef CONFIG_SMP
1056/* Another cpu may still have their %cr3 pointing at the pagetable, so
1057   we need to repoint it somewhere else before we can unpin it. */
1058static void drop_other_mm_ref(void *info)
1059{
1060        struct mm_struct *mm = info;
1061        struct mm_struct *active_mm;
1062
1063#ifdef CONFIG_X86_64
1064        active_mm = read_pda(active_mm);
1065#else
1066        active_mm = __get_cpu_var(cpu_tlbstate).active_mm;
1067#endif
1068
1069        if (active_mm == mm)
1070                leave_mm(smp_processor_id());
1071
1072        /* If this cpu still has a stale cr3 reference, then make sure
1073           it has been flushed. */
1074        if (x86_read_percpu(xen_current_cr3) == __pa(mm->pgd)) {
1075                load_cr3(swapper_pg_dir);
1076                arch_flush_lazy_cpu_mode();
1077        }
1078}
1079
1080static void xen_drop_mm_ref(struct mm_struct *mm)
1081{
1082        cpumask_t mask;
1083        unsigned cpu;
1084
1085        if (current->active_mm == mm) {
1086                if (current->mm == mm)
1087                        load_cr3(swapper_pg_dir);
1088                else
1089                        leave_mm(smp_processor_id());
1090                arch_flush_lazy_cpu_mode();
1091        }
1092
1093        /* Get the "official" set of cpus referring to our pagetable. */
1094        mask = mm->cpu_vm_mask;
1095
1096        /* It's possible that a vcpu may have a stale reference to our
1097           cr3, because its in lazy mode, and it hasn't yet flushed
1098           its set of pending hypercalls yet.  In this case, we can
1099           look at its actual current cr3 value, and force it to flush
1100           if needed. */
1101        for_each_online_cpu(cpu) {
1102                if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1103                        cpu_set(cpu, mask);
1104        }
1105
1106        if (!cpus_empty(mask))
1107                smp_call_function_mask(mask, drop_other_mm_ref, mm, 1);
1108}
1109#else
1110static void xen_drop_mm_ref(struct mm_struct *mm)
1111{
1112        if (current->active_mm == mm)
1113                load_cr3(swapper_pg_dir);
1114}
1115#endif
1116
1117/*
1118 * While a process runs, Xen pins its pagetables, which means that the
1119 * hypervisor forces it to be read-only, and it controls all updates
1120 * to it.  This means that all pagetable updates have to go via the
1121 * hypervisor, which is moderately expensive.
1122 *
1123 * Since we're pulling the pagetable down, we switch to use init_mm,
1124 * unpin old process pagetable and mark it all read-write, which
1125 * allows further operations on it to be simple memory accesses.
1126 *
1127 * The only subtle point is that another CPU may be still using the
1128 * pagetable because of lazy tlb flushing.  This means we need need to
1129 * switch all CPUs off this pagetable before we can unpin it.
1130 */
1131void xen_exit_mmap(struct mm_struct *mm)
1132{
1133        get_cpu();              /* make sure we don't move around */
1134        xen_drop_mm_ref(mm);
1135        put_cpu();
1136
1137        spin_lock(&mm->page_table_lock);
1138
1139        /* pgd may not be pinned in the error exit path of execve */
1140        if (xen_page_pinned(mm->pgd))
1141                xen_pgd_unpin(mm);
1142
1143        spin_unlock(&mm->page_table_lock);
1144}
1145
1146#ifdef CONFIG_XEN_DEBUG_FS
1147
1148static struct dentry *d_mmu_debug;
1149
1150static int __init xen_mmu_debugfs(void)
1151{
1152        struct dentry *d_xen = xen_init_debugfs();
1153
1154        if (d_xen == NULL)
1155                return -ENOMEM;
1156
1157        d_mmu_debug = debugfs_create_dir("mmu", d_xen);
1158
1159        debugfs_create_u8("zero_stats", 0644, d_mmu_debug, &zero_stats);
1160
1161        debugfs_create_u32("pgd_update", 0444, d_mmu_debug, &mmu_stats.pgd_update);
1162        debugfs_create_u32("pgd_update_pinned", 0444, d_mmu_debug,
1163                           &mmu_stats.pgd_update_pinned);
1164        debugfs_create_u32("pgd_update_batched", 0444, d_mmu_debug,
1165                           &mmu_stats.pgd_update_pinned);
1166
1167        debugfs_create_u32("pud_update", 0444, d_mmu_debug, &mmu_stats.pud_update);
1168        debugfs_create_u32("pud_update_pinned", 0444, d_mmu_debug,
1169                           &mmu_stats.pud_update_pinned);
1170        debugfs_create_u32("pud_update_batched", 0444, d_mmu_debug,
1171                           &mmu_stats.pud_update_pinned);
1172
1173        debugfs_create_u32("pmd_update", 0444, d_mmu_debug, &mmu_stats.pmd_update);
1174        debugfs_create_u32("pmd_update_pinned", 0444, d_mmu_debug,
1175                           &mmu_stats.pmd_update_pinned);
1176        debugfs_create_u32("pmd_update_batched", 0444, d_mmu_debug,
1177                           &mmu_stats.pmd_update_pinned);
1178
1179        debugfs_create_u32("pte_update", 0444, d_mmu_debug, &mmu_stats.pte_update);
1180//      debugfs_create_u32("pte_update_pinned", 0444, d_mmu_debug,
1181//                         &mmu_stats.pte_update_pinned);
1182        debugfs_create_u32("pte_update_batched", 0444, d_mmu_debug,
1183                           &mmu_stats.pte_update_pinned);
1184
1185        debugfs_create_u32("mmu_update", 0444, d_mmu_debug, &mmu_stats.mmu_update);
1186        debugfs_create_u32("mmu_update_extended", 0444, d_mmu_debug,
1187                           &mmu_stats.mmu_update_extended);
1188        xen_debugfs_create_u32_array("mmu_update_histo", 0444, d_mmu_debug,
1189                                     mmu_stats.mmu_update_histo, 20);
1190
1191        debugfs_create_u32("set_pte_at", 0444, d_mmu_debug, &mmu_stats.set_pte_at);
1192        debugfs_create_u32("set_pte_at_batched", 0444, d_mmu_debug,
1193                           &mmu_stats.set_pte_at_batched);
1194        debugfs_create_u32("set_pte_at_current", 0444, d_mmu_debug,
1195                           &mmu_stats.set_pte_at_current);
1196        debugfs_create_u32("set_pte_at_kernel", 0444, d_mmu_debug,
1197                           &mmu_stats.set_pte_at_kernel);
1198
1199        debugfs_create_u32("prot_commit", 0444, d_mmu_debug, &mmu_stats.prot_commit);
1200        debugfs_create_u32("prot_commit_batched", 0444, d_mmu_debug,
1201                           &mmu_stats.prot_commit_batched);
1202
1203        return 0;
1204}
1205fs_initcall(xen_mmu_debugfs);
1206
1207#endif  /* CONFIG_XEN_DEBUG_FS */
1208