LXR linux/arch/ppc64/mm/tlb.c

   1/*
   2 * This file contains the routines for flushing entries from the
   3 * TLB and MMU hash table.
   4 *
   5 *  Derived from arch/ppc64/mm/init.c:
   6 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   7 *
   8 *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
   9 *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
  10 *    Copyright (C) 1996 Paul Mackerras
  11 *  Amiga/APUS changes by Jesper Skov (jskov@cygnus.co.uk).
  12 *
  13 *  Derived from "arch/i386/mm/init.c"
  14 *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  15 *
  16 *  Dave Engebretsen <engebret@us.ibm.com>
  17 *      Rework for PPC64 port.
  18 *
  19 *  This program is free software; you can redistribute it and/or
  20 *  modify it under the terms of the GNU General Public License
  21 *  as published by the Free Software Foundation; either version
  22 *  2 of the License, or (at your option) any later version.
  23 */
  24#include <linux/config.h>
  25#include <linux/kernel.h>
  26#include <linux/mm.h>
  27#include <linux/init.h>
  28#include <linux/percpu.h>
  29#include <linux/hardirq.h>
  30#include <asm/pgalloc.h>
  31#include <asm/tlbflush.h>
  32#include <asm/tlb.h>
  33#include <linux/highmem.h>
  34
  35DEFINE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
  36
  37/* This is declared as we are using the more or less generic
  38 * include/asm-ppc64/tlb.h file -- tgall
  39 */
  40DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
  41DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);
  42unsigned long pte_freelist_forced_free;
  43
  44void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage)
  45{
  46        /* This is safe as we are holding page_table_lock */
  47        cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id());
  48        struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur);
  49
  50        if (atomic_read(&tlb->mm->mm_users) < 2 ||
  51            cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) {
  52                pte_free(ptepage);
  53                return;
  54        }
  55
  56        if (*batchp == NULL) {
  57                *batchp = (struct pte_freelist_batch *)__get_free_page(GFP_ATOMIC);
  58                if (*batchp == NULL) {
  59                        pte_free_now(ptepage);
  60                        return;
  61                }
  62                (*batchp)->index = 0;
  63        }
  64        (*batchp)->pages[(*batchp)->index++] = ptepage;
  65        if ((*batchp)->index == PTE_FREELIST_SIZE) {
  66                pte_free_submit(*batchp);
  67                *batchp = NULL;
  68        }
  69}
  70
  71/*
  72 * Update the MMU hash table to correspond with a change to
  73 * a Linux PTE.  If wrprot is true, it is permissible to
  74 * change the existing HPTE to read-only rather than removing it
  75 * (if we remove it we should clear the _PTE_HPTEFLAGS bits).
  76 */
  77void hpte_update(pte_t *ptep, unsigned long pte, int wrprot)
  78{
  79        struct page *ptepage;
  80        struct mm_struct *mm;
  81        unsigned long addr;
  82        int i;
  83        unsigned long context = 0;
  84        struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
  85
  86        ptepage = virt_to_page(ptep);
  87        mm = (struct mm_struct *) ptepage->mapping;
  88        addr = ptepage->index;
  89        if (pte_huge(pte))
  90                addr +=  ((unsigned long)ptep & ~PAGE_MASK)
  91                        / sizeof(*ptep) * HPAGE_SIZE;
  92        else
  93                addr += ((unsigned long)ptep & ~PAGE_MASK) * PTRS_PER_PTE;
  94
  95        if (REGION_ID(addr) == USER_REGION_ID)
  96                context = mm->context.id;
  97        i = batch->index;
  98
  99        /*
 100         * This can happen when we are in the middle of a TLB batch and
 101         * we encounter memory pressure (eg copy_page_range when it tries
 102         * to allocate a new pte). If we have to reclaim memory and end
 103         * up scanning and resetting referenced bits then our batch context
 104         * will change mid stream.
 105         */
 106        if (unlikely(i != 0 && context != batch->context)) {
 107                flush_tlb_pending();
 108                i = 0;
 109        }
 110
 111        if (i == 0) {
 112                batch->context = context;
 113                batch->mm = mm;
 114        }
 115        batch->pte[i] = __pte(pte);
 116        batch->addr[i] = addr;
 117        batch->index = ++i;
 118        if (i >= PPC64_TLB_BATCH_NR)
 119                flush_tlb_pending();
 120}
 121
 122void __flush_tlb_pending(struct ppc64_tlb_batch *batch)
 123{
 124        int i;
 125        int cpu;
 126        cpumask_t tmp;
 127        int local = 0;
 128
 129        BUG_ON(in_interrupt());
 130
 131        cpu = get_cpu();
 132        i = batch->index;
 133        tmp = cpumask_of_cpu(cpu);
 134        if (cpus_equal(batch->mm->cpu_vm_mask, tmp))
 135                local = 1;
 136
 137        if (i == 1)
 138                flush_hash_page(batch->context, batch->addr[0], batch->pte[0],
 139                                local);
 140        else
 141                flush_hash_range(batch->context, i, local);
 142        batch->index = 0;
 143        put_cpu();
 144}
 145
 146#ifdef CONFIG_SMP
 147static void pte_free_smp_sync(void *arg)
 148{
 149        /* Do nothing, just ensure we sync with all CPUs */
 150}
 151#endif
 152
 153/* This is only called when we are critically out of memory
 154 * (and fail to get a page in pte_free_tlb).
 155 */
 156void pte_free_now(struct page *ptepage)
 157{
 158        pte_freelist_forced_free++;
 159
 160        smp_call_function(pte_free_smp_sync, NULL, 0, 1);
 161
 162        pte_free(ptepage);
 163}
 164
 165static void pte_free_rcu_callback(struct rcu_head *head)
 166{
 167        struct pte_freelist_batch *batch =
 168                container_of(head, struct pte_freelist_batch, rcu);
 169        unsigned int i;
 170
 171        for (i = 0; i < batch->index; i++)
 172                pte_free(batch->pages[i]);
 173        free_page((unsigned long)batch);
 174}
 175
 176void pte_free_submit(struct pte_freelist_batch *batch)
 177{
 178        INIT_RCU_HEAD(&batch->rcu);
 179        call_rcu(&batch->rcu, pte_free_rcu_callback);
 180}
 181
 182void pte_free_finish(void)
 183{
 184        /* This is safe as we are holding page_table_lock */
 185        struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur);
 186
 187        if (*batchp == NULL)
 188                return;
 189        pte_free_submit(*batchp);
 190        *batchp = NULL;
 191}
 192