LXR linux-bk/arch/i386/mm/pageattr.c

   1/* 
   2 * Copyright 2002 Andi Kleen, SuSE Labs. 
   3 * Thanks to Ben LaHaise for precious feedback.
   4 */ 
   5
   6#include <linux/config.h>
   7#include <linux/mm.h>
   8#include <linux/sched.h>
   9#include <linux/highmem.h>
  10#include <linux/module.h>
  11#include <linux/slab.h>
  12#include <asm/uaccess.h>
  13#include <asm/processor.h>
  14#include <asm/tlbflush.h>
  15
  16static inline pte_t *lookup_address(unsigned long address) 
  17{ 
  18        pgd_t *pgd = pgd_offset_k(address); 
  19        pmd_t *pmd = pmd_offset(pgd, address);         
  20        if (pmd_large(*pmd))
  21                return (pte_t *)pmd;
  22        return pte_offset_kernel(pmd, address);
  23} 
  24
  25static struct page *split_large_page(unsigned long address, pgprot_t prot)
  26{ 
  27        int i; 
  28        unsigned long addr;
  29        struct page *base = alloc_pages(GFP_KERNEL, 0);
  30        pte_t *pbase;
  31        if (!base) 
  32                return NULL;
  33        address = __pa(address);
  34        addr = address & LARGE_PAGE_MASK; 
  35        pbase = (pte_t *)page_address(base);
  36        for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) {
  37                pbase[i] = pfn_pte(addr >> PAGE_SHIFT, 
  38                                   addr == address ? prot : PAGE_KERNEL);
  39        }
  40        return base;
  41} 
  42
  43static void flush_kernel_map(void *dummy) 
  44{ 
  45        /* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */
  46        if (boot_cpu_data.x86_model >= 4) 
  47                asm volatile("wbinvd":::"memory"); 
  48        /* Flush all to work around Errata in early athlons regarding 
  49         * large page flushing. 
  50         */
  51        __flush_tlb_all();      
  52}
  53
  54static void set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) 
  55{ 
  56        set_pte_atomic(kpte, pte);      /* change init_mm */
  57#ifndef CONFIG_X86_PAE
  58        {
  59                struct list_head *l;
  60                spin_lock(&mmlist_lock);
  61                list_for_each(l, &init_mm.mmlist) { 
  62                        struct mm_struct *mm = list_entry(l, struct mm_struct, mmlist);
  63                        pmd_t *pmd = pmd_offset(pgd_offset(mm, address), address);
  64                        set_pte_atomic((pte_t *)pmd, pte);
  65                } 
  66                spin_unlock(&mmlist_lock);
  67        }
  68#endif
  69}
  70
  71/* 
  72 * No more special protections in this 2/4MB area - revert to a
  73 * large page again. 
  74 */
  75static inline void revert_page(struct page *kpte_page, unsigned long address)
  76{
  77        pte_t *linear = (pte_t *) 
  78                pmd_offset(pgd_offset(&init_mm, address), address);
  79        set_pmd_pte(linear,  address,
  80                    pfn_pte((__pa(address) & LARGE_PAGE_MASK) >> PAGE_SHIFT,
  81                            PAGE_KERNEL_LARGE));
  82}
  83
  84static int
  85__change_page_attr(struct page *page, pgprot_t prot, struct page **oldpage) 
  86{ 
  87        pte_t *kpte; 
  88        unsigned long address;
  89        struct page *kpte_page;
  90
  91#ifdef CONFIG_HIGHMEM
  92        if (page >= highmem_start_page) 
  93                BUG(); 
  94#endif
  95        address = (unsigned long)page_address(page);
  96
  97        kpte = lookup_address(address);
  98        kpte_page = virt_to_page(((unsigned long)kpte) & PAGE_MASK);
  99        if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) { 
 100                if ((pte_val(*kpte) & _PAGE_PSE) == 0) { 
 101                        pte_t old = *kpte;
 102                        pte_t standard = mk_pte(page, PAGE_KERNEL); 
 103
 104                        set_pte_atomic(kpte, mk_pte(page, prot)); 
 105                        if (pte_same(old,standard))
 106                                atomic_inc(&kpte_page->count);
 107                } else {
 108                        struct page *split = split_large_page(address, prot); 
 109                        if (!split)
 110                                return -ENOMEM;
 111                        set_pmd_pte(kpte,address,mk_pte(split, PAGE_KERNEL));
 112                }       
 113        } else if ((pte_val(*kpte) & _PAGE_PSE) == 0) { 
 114                set_pte_atomic(kpte, mk_pte(page, PAGE_KERNEL));
 115                atomic_dec(&kpte_page->count); 
 116        }
 117
 118        if (cpu_has_pse && (atomic_read(&kpte_page->count) == 1)) { 
 119                *oldpage = kpte_page;
 120                revert_page(kpte_page, address);
 121        } 
 122        return 0;
 123} 
 124
 125static inline void flush_map(void)
 126{       
 127#ifdef CONFIG_SMP 
 128        smp_call_function(flush_kernel_map, NULL, 1, 1);
 129#endif  
 130        flush_kernel_map(NULL);
 131}
 132
 133struct deferred_page { 
 134        struct deferred_page *next; 
 135        struct page *fpage;
 136}; 
 137static struct deferred_page *df_list; /* protected by init_mm.mmap_sem */
 138
 139/*
 140 * Change the page attributes of an page in the linear mapping.
 141 *
 142 * This should be used when a page is mapped with a different caching policy
 143 * than write-back somewhere - some CPUs do not like it when mappings with
 144 * different caching policies exist. This changes the page attributes of the
 145 * in kernel linear mapping too.
 146 * 
 147 * The caller needs to ensure that there are no conflicting mappings elsewhere.
 148 * This function only deals with the kernel linear map.
 149 * 
 150 * Caller must call global_flush_tlb() after this.
 151 */
 152int change_page_attr(struct page *page, int numpages, pgprot_t prot)
 153{
 154        int err = 0; 
 155        struct page *fpage; 
 156        int i; 
 157
 158        down_write(&init_mm.mmap_sem);
 159        for (i = 0; i < numpages; i++, page++) { 
 160                fpage = NULL;
 161                err = __change_page_attr(page, prot, &fpage); 
 162                if (err) 
 163                        break; 
 164                if (fpage) { 
 165                        struct deferred_page *df;
 166                        df = kmalloc(sizeof(struct deferred_page), GFP_KERNEL); 
 167                        if (!df) {
 168                                flush_map();
 169                                __free_page(fpage);
 170                        } else { 
 171                                df->next = df_list;
 172                                df->fpage = fpage;                              
 173                                df_list = df;
 174                        }                       
 175                } 
 176        }       
 177        up_write(&init_mm.mmap_sem); 
 178        return err;
 179}
 180
 181void global_flush_tlb(void)
 182{ 
 183        struct deferred_page *df, *next_df;
 184
 185        down_read(&init_mm.mmap_sem);
 186        df = xchg(&df_list, NULL);
 187        up_read(&init_mm.mmap_sem);
 188        flush_map();
 189        for (; df; df = next_df) { 
 190                next_df = df->next;
 191                if (df->fpage) 
 192                        __free_page(df->fpage);
 193                kfree(df);
 194        } 
 195} 
 196
 197EXPORT_SYMBOL(change_page_attr);
 198EXPORT_SYMBOL(global_flush_tlb);
 199