linux/mm/vmalloc.c
<<
>>
Prefs
   1/*
   2 *  linux/mm/vmalloc.c
   3 *
   4 *  Copyright (C) 1993  Linus Torvalds
   5 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
   6 *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
   7 *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
   8 *  Numa awareness, Christoph Lameter, SGI, June 2005
   9 */
  10
  11#include <linux/mm.h>
  12#include <linux/module.h>
  13#include <linux/highmem.h>
  14#include <linux/slab.h>
  15#include <linux/spinlock.h>
  16#include <linux/interrupt.h>
  17#include <linux/seq_file.h>
  18#include <linux/debugobjects.h>
  19#include <linux/vmalloc.h>
  20#include <linux/kallsyms.h>
  21
  22#include <asm/uaccess.h>
  23#include <asm/tlbflush.h>
  24
  25
  26DEFINE_RWLOCK(vmlist_lock);
  27struct vm_struct *vmlist;
  28
  29static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
  30                            int node, void *caller);
  31
  32static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
  33{
  34        pte_t *pte;
  35
  36        pte = pte_offset_kernel(pmd, addr);
  37        do {
  38                pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
  39                WARN_ON(!pte_none(ptent) && !pte_present(ptent));
  40        } while (pte++, addr += PAGE_SIZE, addr != end);
  41}
  42
  43static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
  44                                                unsigned long end)
  45{
  46        pmd_t *pmd;
  47        unsigned long next;
  48
  49        pmd = pmd_offset(pud, addr);
  50        do {
  51                next = pmd_addr_end(addr, end);
  52                if (pmd_none_or_clear_bad(pmd))
  53                        continue;
  54                vunmap_pte_range(pmd, addr, next);
  55        } while (pmd++, addr = next, addr != end);
  56}
  57
  58static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
  59                                                unsigned long end)
  60{
  61        pud_t *pud;
  62        unsigned long next;
  63
  64        pud = pud_offset(pgd, addr);
  65        do {
  66                next = pud_addr_end(addr, end);
  67                if (pud_none_or_clear_bad(pud))
  68                        continue;
  69                vunmap_pmd_range(pud, addr, next);
  70        } while (pud++, addr = next, addr != end);
  71}
  72
  73void unmap_kernel_range(unsigned long addr, unsigned long size)
  74{
  75        pgd_t *pgd;
  76        unsigned long next;
  77        unsigned long start = addr;
  78        unsigned long end = addr + size;
  79
  80        BUG_ON(addr >= end);
  81        pgd = pgd_offset_k(addr);
  82        flush_cache_vunmap(addr, end);
  83        do {
  84                next = pgd_addr_end(addr, end);
  85                if (pgd_none_or_clear_bad(pgd))
  86                        continue;
  87                vunmap_pud_range(pgd, addr, next);
  88        } while (pgd++, addr = next, addr != end);
  89        flush_tlb_kernel_range(start, end);
  90}
  91
  92static void unmap_vm_area(struct vm_struct *area)
  93{
  94        unmap_kernel_range((unsigned long)area->addr, area->size);
  95}
  96
  97static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
  98                        unsigned long end, pgprot_t prot, struct page ***pages)
  99{
 100        pte_t *pte;
 101
 102        pte = pte_alloc_kernel(pmd, addr);
 103        if (!pte)
 104                return -ENOMEM;
 105        do {
 106                struct page *page = **pages;
 107                WARN_ON(!pte_none(*pte));
 108                if (!page)
 109                        return -ENOMEM;
 110                set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
 111                (*pages)++;
 112        } while (pte++, addr += PAGE_SIZE, addr != end);
 113        return 0;
 114}
 115
 116static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
 117                        unsigned long end, pgprot_t prot, struct page ***pages)
 118{
 119        pmd_t *pmd;
 120        unsigned long next;
 121
 122        pmd = pmd_alloc(&init_mm, pud, addr);
 123        if (!pmd)
 124                return -ENOMEM;
 125        do {
 126                next = pmd_addr_end(addr, end);
 127                if (vmap_pte_range(pmd, addr, next, prot, pages))
 128                        return -ENOMEM;
 129        } while (pmd++, addr = next, addr != end);
 130        return 0;
 131}
 132
 133static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
 134                        unsigned long end, pgprot_t prot, struct page ***pages)
 135{
 136        pud_t *pud;
 137        unsigned long next;
 138
 139        pud = pud_alloc(&init_mm, pgd, addr);
 140        if (!pud)
 141                return -ENOMEM;
 142        do {
 143                next = pud_addr_end(addr, end);
 144                if (vmap_pmd_range(pud, addr, next, prot, pages))
 145                        return -ENOMEM;
 146        } while (pud++, addr = next, addr != end);
 147        return 0;
 148}
 149
 150int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
 151{
 152        pgd_t *pgd;
 153        unsigned long next;
 154        unsigned long addr = (unsigned long) area->addr;
 155        unsigned long end = addr + area->size - PAGE_SIZE;
 156        int err;
 157
 158        BUG_ON(addr >= end);
 159        pgd = pgd_offset_k(addr);
 160        do {
 161                next = pgd_addr_end(addr, end);
 162                err = vmap_pud_range(pgd, addr, next, prot, pages);
 163                if (err)
 164                        break;
 165        } while (pgd++, addr = next, addr != end);
 166        flush_cache_vmap((unsigned long) area->addr, end);
 167        return err;
 168}
 169EXPORT_SYMBOL_GPL(map_vm_area);
 170
 171/*
 172 * Map a vmalloc()-space virtual address to the physical page.
 173 */
 174struct page *vmalloc_to_page(const void *vmalloc_addr)
 175{
 176        unsigned long addr = (unsigned long) vmalloc_addr;
 177        struct page *page = NULL;
 178        pgd_t *pgd = pgd_offset_k(addr);
 179        pud_t *pud;
 180        pmd_t *pmd;
 181        pte_t *ptep, pte;
 182
 183        if (!pgd_none(*pgd)) {
 184                pud = pud_offset(pgd, addr);
 185                if (!pud_none(*pud)) {
 186                        pmd = pmd_offset(pud, addr);
 187                        if (!pmd_none(*pmd)) {
 188                                ptep = pte_offset_map(pmd, addr);
 189                                pte = *ptep;
 190                                if (pte_present(pte))
 191                                        page = pte_page(pte);
 192                                pte_unmap(ptep);
 193                        }
 194                }
 195        }
 196        return page;
 197}
 198EXPORT_SYMBOL(vmalloc_to_page);
 199
 200/*
 201 * Map a vmalloc()-space virtual address to the physical page frame number.
 202 */
 203unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
 204{
 205        return page_to_pfn(vmalloc_to_page(vmalloc_addr));
 206}
 207EXPORT_SYMBOL(vmalloc_to_pfn);
 208
 209static struct vm_struct *
 210__get_vm_area_node(unsigned long size, unsigned long flags, unsigned long start,
 211                unsigned long end, int node, gfp_t gfp_mask, void *caller)
 212{
 213        struct vm_struct **p, *tmp, *area;
 214        unsigned long align = 1;
 215        unsigned long addr;
 216
 217        BUG_ON(in_interrupt());
 218        if (flags & VM_IOREMAP) {
 219                int bit = fls(size);
 220
 221                if (bit > IOREMAP_MAX_ORDER)
 222                        bit = IOREMAP_MAX_ORDER;
 223                else if (bit < PAGE_SHIFT)
 224                        bit = PAGE_SHIFT;
 225
 226                align = 1ul << bit;
 227        }
 228        addr = ALIGN(start, align);
 229        size = PAGE_ALIGN(size);
 230        if (unlikely(!size))
 231                return NULL;
 232
 233        area = kmalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
 234
 235        if (unlikely(!area))
 236                return NULL;
 237
 238        /*
 239         * We always allocate a guard page.
 240         */
 241        size += PAGE_SIZE;
 242
 243        write_lock(&vmlist_lock);
 244        for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
 245                if ((unsigned long)tmp->addr < addr) {
 246                        if((unsigned long)tmp->addr + tmp->size >= addr)
 247                                addr = ALIGN(tmp->size + 
 248                                             (unsigned long)tmp->addr, align);
 249                        continue;
 250                }
 251                if ((size + addr) < addr)
 252                        goto out;
 253                if (size + addr <= (unsigned long)tmp->addr)
 254                        goto found;
 255                addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
 256                if (addr > end - size)
 257                        goto out;
 258        }
 259        if ((size + addr) < addr)
 260                goto out;
 261        if (addr > end - size)
 262                goto out;
 263
 264found:
 265        area->next = *p;
 266        *p = area;
 267
 268        area->flags = flags;
 269        area->addr = (void *)addr;
 270        area->size = size;
 271        area->pages = NULL;
 272        area->nr_pages = 0;
 273        area->phys_addr = 0;
 274        area->caller = caller;
 275        write_unlock(&vmlist_lock);
 276
 277        return area;
 278
 279out:
 280        write_unlock(&vmlist_lock);
 281        kfree(area);
 282        if (printk_ratelimit())
 283                printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
 284        return NULL;
 285}
 286
 287struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
 288                                unsigned long start, unsigned long end)
 289{
 290        return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL,
 291                                                __builtin_return_address(0));
 292}
 293EXPORT_SYMBOL_GPL(__get_vm_area);
 294
 295/**
 296 *      get_vm_area  -  reserve a contiguous kernel virtual area
 297 *      @size:          size of the area
 298 *      @flags:         %VM_IOREMAP for I/O mappings or VM_ALLOC
 299 *
 300 *      Search an area of @size in the kernel virtual mapping area,
 301 *      and reserved it for out purposes.  Returns the area descriptor
 302 *      on success or %NULL on failure.
 303 */
 304struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
 305{
 306        return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
 307                                -1, GFP_KERNEL, __builtin_return_address(0));
 308}
 309
 310struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
 311                                void *caller)
 312{
 313        return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
 314                                                -1, GFP_KERNEL, caller);
 315}
 316
 317struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
 318                                   int node, gfp_t gfp_mask)
 319{
 320        return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
 321                                  gfp_mask, __builtin_return_address(0));
 322}
 323
 324/* Caller must hold vmlist_lock */
 325static struct vm_struct *__find_vm_area(const void *addr)
 326{
 327        struct vm_struct *tmp;
 328
 329        for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
 330                 if (tmp->addr == addr)
 331                        break;
 332        }
 333
 334        return tmp;
 335}
 336
 337/* Caller must hold vmlist_lock */
 338static struct vm_struct *__remove_vm_area(const void *addr)
 339{
 340        struct vm_struct **p, *tmp;
 341
 342        for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
 343                 if (tmp->addr == addr)
 344                         goto found;
 345        }
 346        return NULL;
 347
 348found:
 349        unmap_vm_area(tmp);
 350        *p = tmp->next;
 351
 352        /*
 353         * Remove the guard page.
 354         */
 355        tmp->size -= PAGE_SIZE;
 356        return tmp;
 357}
 358
 359/**
 360 *      remove_vm_area  -  find and remove a continuous kernel virtual area
 361 *      @addr:          base address
 362 *
 363 *      Search for the kernel VM area starting at @addr, and remove it.
 364 *      This function returns the found VM area, but using it is NOT safe
 365 *      on SMP machines, except for its size or flags.
 366 */
 367struct vm_struct *remove_vm_area(const void *addr)
 368{
 369        struct vm_struct *v;
 370        write_lock(&vmlist_lock);
 371        v = __remove_vm_area(addr);
 372        write_unlock(&vmlist_lock);
 373        return v;
 374}
 375
 376static void __vunmap(const void *addr, int deallocate_pages)
 377{
 378        struct vm_struct *area;
 379
 380        if (!addr)
 381                return;
 382
 383        if ((PAGE_SIZE-1) & (unsigned long)addr) {
 384                printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
 385                WARN_ON(1);
 386                return;
 387        }
 388
 389        area = remove_vm_area(addr);
 390        if (unlikely(!area)) {
 391                printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
 392                                addr);
 393                WARN_ON(1);
 394                return;
 395        }
 396
 397        debug_check_no_locks_freed(addr, area->size);
 398        debug_check_no_obj_freed(addr, area->size);
 399
 400        if (deallocate_pages) {
 401                int i;
 402
 403                for (i = 0; i < area->nr_pages; i++) {
 404                        struct page *page = area->pages[i];
 405
 406                        BUG_ON(!page);
 407                        __free_page(page);
 408                }
 409
 410                if (area->flags & VM_VPAGES)
 411                        vfree(area->pages);
 412                else
 413                        kfree(area->pages);
 414        }
 415
 416        kfree(area);
 417        return;
 418}
 419
 420/**
 421 *      vfree  -  release memory allocated by vmalloc()
 422 *      @addr:          memory base address
 423 *
 424 *      Free the virtually continuous memory area starting at @addr, as
 425 *      obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
 426 *      NULL, no operation is performed.
 427 *
 428 *      Must not be called in interrupt context.
 429 */
 430void vfree(const void *addr)
 431{
 432        BUG_ON(in_interrupt());
 433        __vunmap(addr, 1);
 434}
 435EXPORT_SYMBOL(vfree);
 436
 437/**
 438 *      vunmap  -  release virtual mapping obtained by vmap()
 439 *      @addr:          memory base address
 440 *
 441 *      Free the virtually contiguous memory area starting at @addr,
 442 *      which was created from the page array passed to vmap().
 443 *
 444 *      Must not be called in interrupt context.
 445 */
 446void vunmap(const void *addr)
 447{
 448        BUG_ON(in_interrupt());
 449        __vunmap(addr, 0);
 450}
 451EXPORT_SYMBOL(vunmap);
 452
 453/**
 454 *      vmap  -  map an array of pages into virtually contiguous space
 455 *      @pages:         array of page pointers
 456 *      @count:         number of pages to map
 457 *      @flags:         vm_area->flags
 458 *      @prot:          page protection for the mapping
 459 *
 460 *      Maps @count pages from @pages into contiguous kernel virtual
 461 *      space.
 462 */
 463void *vmap(struct page **pages, unsigned int count,
 464                unsigned long flags, pgprot_t prot)
 465{
 466        struct vm_struct *area;
 467
 468        if (count > num_physpages)
 469                return NULL;
 470
 471        area = get_vm_area_caller((count << PAGE_SHIFT), flags,
 472                                        __builtin_return_address(0));
 473        if (!area)
 474                return NULL;
 475
 476        if (map_vm_area(area, prot, &pages)) {
 477                vunmap(area->addr);
 478                return NULL;
 479        }
 480
 481        return area->addr;
 482}
 483EXPORT_SYMBOL(vmap);
 484
 485static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
 486                                 pgprot_t prot, int node, void *caller)
 487{
 488        struct page **pages;
 489        unsigned int nr_pages, array_size, i;
 490
 491        nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
 492        array_size = (nr_pages * sizeof(struct page *));
 493
 494        area->nr_pages = nr_pages;
 495        /* Please note that the recursion is strictly bounded. */
 496        if (array_size > PAGE_SIZE) {
 497                pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO,
 498                                PAGE_KERNEL, node, caller);
 499                area->flags |= VM_VPAGES;
 500        } else {
 501                pages = kmalloc_node(array_size,
 502                                (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO,
 503                                node);
 504        }
 505        area->pages = pages;
 506        area->caller = caller;
 507        if (!area->pages) {
 508                remove_vm_area(area->addr);
 509                kfree(area);
 510                return NULL;
 511        }
 512
 513        for (i = 0; i < area->nr_pages; i++) {
 514                struct page *page;
 515
 516                if (node < 0)
 517                        page = alloc_page(gfp_mask);
 518                else
 519                        page = alloc_pages_node(node, gfp_mask, 0);
 520
 521                if (unlikely(!page)) {
 522                        /* Successfully allocated i pages, free them in __vunmap() */
 523                        area->nr_pages = i;
 524                        goto fail;
 525                }
 526                area->pages[i] = page;
 527        }
 528
 529        if (map_vm_area(area, prot, &pages))
 530                goto fail;
 531        return area->addr;
 532
 533fail:
 534        vfree(area->addr);
 535        return NULL;
 536}
 537
 538void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
 539{
 540        return __vmalloc_area_node(area, gfp_mask, prot, -1,
 541                                        __builtin_return_address(0));
 542}
 543
 544/**
 545 *      __vmalloc_node  -  allocate virtually contiguous memory
 546 *      @size:          allocation size
 547 *      @gfp_mask:      flags for the page level allocator
 548 *      @prot:          protection mask for the allocated pages
 549 *      @node:          node to use for allocation or -1
 550 *      @caller:        caller's return address
 551 *
 552 *      Allocate enough pages to cover @size from the page level
 553 *      allocator with @gfp_mask flags.  Map them into contiguous
 554 *      kernel virtual space, using a pagetable protection of @prot.
 555 */
 556static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
 557                                                int node, void *caller)
 558{
 559        struct vm_struct *area;
 560
 561        size = PAGE_ALIGN(size);
 562        if (!size || (size >> PAGE_SHIFT) > num_physpages)
 563                return NULL;
 564
 565        area = __get_vm_area_node(size, VM_ALLOC, VMALLOC_START, VMALLOC_END,
 566                                                node, gfp_mask, caller);
 567
 568        if (!area)
 569                return NULL;
 570
 571        return __vmalloc_area_node(area, gfp_mask, prot, node, caller);
 572}
 573
 574void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
 575{
 576        return __vmalloc_node(size, gfp_mask, prot, -1,
 577                                __builtin_return_address(0));
 578}
 579EXPORT_SYMBOL(__vmalloc);
 580
 581/**
 582 *      vmalloc  -  allocate virtually contiguous memory
 583 *      @size:          allocation size
 584 *      Allocate enough pages to cover @size from the page level
 585 *      allocator and map them into contiguous kernel virtual space.
 586 *
 587 *      For tight control over page level allocator and protection flags
 588 *      use __vmalloc() instead.
 589 */
 590void *vmalloc(unsigned long size)
 591{
 592        return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
 593                                        -1, __builtin_return_address(0));
 594}
 595EXPORT_SYMBOL(vmalloc);
 596
 597/**
 598 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
 599 * @size: allocation size
 600 *
 601 * The resulting memory area is zeroed so it can be mapped to userspace
 602 * without leaking data.
 603 */
 604void *vmalloc_user(unsigned long size)
 605{
 606        struct vm_struct *area;
 607        void *ret;
 608
 609        ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
 610        if (ret) {
 611                write_lock(&vmlist_lock);
 612                area = __find_vm_area(ret);
 613                area->flags |= VM_USERMAP;
 614                write_unlock(&vmlist_lock);
 615        }
 616        return ret;
 617}
 618EXPORT_SYMBOL(vmalloc_user);
 619
 620/**
 621 *      vmalloc_node  -  allocate memory on a specific node
 622 *      @size:          allocation size
 623 *      @node:          numa node
 624 *
 625 *      Allocate enough pages to cover @size from the page level
 626 *      allocator and map them into contiguous kernel virtual space.
 627 *
 628 *      For tight control over page level allocator and protection flags
 629 *      use __vmalloc() instead.
 630 */
 631void *vmalloc_node(unsigned long size, int node)
 632{
 633        return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
 634                                        node, __builtin_return_address(0));
 635}
 636EXPORT_SYMBOL(vmalloc_node);
 637
 638#ifndef PAGE_KERNEL_EXEC
 639# define PAGE_KERNEL_EXEC PAGE_KERNEL
 640#endif
 641
 642/**
 643 *      vmalloc_exec  -  allocate virtually contiguous, executable memory
 644 *      @size:          allocation size
 645 *
 646 *      Kernel-internal function to allocate enough pages to cover @size
 647 *      the page level allocator and map them into contiguous and
 648 *      executable kernel virtual space.
 649 *
 650 *      For tight control over page level allocator and protection flags
 651 *      use __vmalloc() instead.
 652 */
 653
 654void *vmalloc_exec(unsigned long size)
 655{
 656        return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
 657}
 658
 659#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
 660#define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
 661#elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
 662#define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
 663#else
 664#define GFP_VMALLOC32 GFP_KERNEL
 665#endif
 666
 667/**
 668 *      vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
 669 *      @size:          allocation size
 670 *
 671 *      Allocate enough 32bit PA addressable pages to cover @size from the
 672 *      page level allocator and map them into contiguous kernel virtual space.
 673 */
 674void *vmalloc_32(unsigned long size)
 675{
 676        return __vmalloc(size, GFP_VMALLOC32, PAGE_KERNEL);
 677}
 678EXPORT_SYMBOL(vmalloc_32);
 679
 680/**
 681 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
 682 *      @size:          allocation size
 683 *
 684 * The resulting memory area is 32bit addressable and zeroed so it can be
 685 * mapped to userspace without leaking data.
 686 */
 687void *vmalloc_32_user(unsigned long size)
 688{
 689        struct vm_struct *area;
 690        void *ret;
 691
 692        ret = __vmalloc(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL);
 693        if (ret) {
 694                write_lock(&vmlist_lock);
 695                area = __find_vm_area(ret);
 696                area->flags |= VM_USERMAP;
 697                write_unlock(&vmlist_lock);
 698        }
 699        return ret;
 700}
 701EXPORT_SYMBOL(vmalloc_32_user);
 702
 703long vread(char *buf, char *addr, unsigned long count)
 704{
 705        struct vm_struct *tmp;
 706        char *vaddr, *buf_start = buf;
 707        unsigned long n;
 708
 709        /* Don't allow overflow */
 710        if ((unsigned long) addr + count < count)
 711                count = -(unsigned long) addr;
 712
 713        read_lock(&vmlist_lock);
 714        for (tmp = vmlist; tmp; tmp = tmp->next) {
 715                vaddr = (char *) tmp->addr;
 716                if (addr >= vaddr + tmp->size - PAGE_SIZE)
 717                        continue;
 718                while (addr < vaddr) {
 719                        if (count == 0)
 720                                goto finished;
 721                        *buf = '\0';
 722                        buf++;
 723                        addr++;
 724                        count--;
 725                }
 726                n = vaddr + tmp->size - PAGE_SIZE - addr;
 727                do {
 728                        if (count == 0)
 729                                goto finished;
 730                        *buf = *addr;
 731                        buf++;
 732                        addr++;
 733                        count--;
 734                } while (--n > 0);
 735        }
 736finished:
 737        read_unlock(&vmlist_lock);
 738        return buf - buf_start;
 739}
 740
 741long vwrite(char *buf, char *addr, unsigned long count)
 742{
 743        struct vm_struct *tmp;
 744        char *vaddr, *buf_start = buf;
 745        unsigned long n;
 746
 747        /* Don't allow overflow */
 748        if ((unsigned long) addr + count < count)
 749                count = -(unsigned long) addr;
 750
 751        read_lock(&vmlist_lock);
 752        for (tmp = vmlist; tmp; tmp = tmp->next) {
 753                vaddr = (char *) tmp->addr;
 754                if (addr >= vaddr + tmp->size - PAGE_SIZE)
 755                        continue;
 756                while (addr < vaddr) {
 757                        if (count == 0)
 758                                goto finished;
 759                        buf++;
 760                        addr++;
 761                        count--;
 762                }
 763                n = vaddr + tmp->size - PAGE_SIZE - addr;
 764                do {
 765                        if (count == 0)
 766                                goto finished;
 767                        *addr = *buf;
 768                        buf++;
 769                        addr++;
 770                        count--;
 771                } while (--n > 0);
 772        }
 773finished:
 774        read_unlock(&vmlist_lock);
 775        return buf - buf_start;
 776}
 777
 778/**
 779 *      remap_vmalloc_range  -  map vmalloc pages to userspace
 780 *      @vma:           vma to cover (map full range of vma)
 781 *      @addr:          vmalloc memory
 782 *      @pgoff:         number of pages into addr before first page to map
 783 *
 784 *      Returns:        0 for success, -Exxx on failure
 785 *
 786 *      This function checks that addr is a valid vmalloc'ed area, and
 787 *      that it is big enough to cover the vma. Will return failure if
 788 *      that criteria isn't met.
 789 *
 790 *      Similar to remap_pfn_range() (see mm/memory.c)
 791 */
 792int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
 793                                                unsigned long pgoff)
 794{
 795        struct vm_struct *area;
 796        unsigned long uaddr = vma->vm_start;
 797        unsigned long usize = vma->vm_end - vma->vm_start;
 798        int ret;
 799
 800        if ((PAGE_SIZE-1) & (unsigned long)addr)
 801                return -EINVAL;
 802
 803        read_lock(&vmlist_lock);
 804        area = __find_vm_area(addr);
 805        if (!area)
 806                goto out_einval_locked;
 807
 808        if (!(area->flags & VM_USERMAP))
 809                goto out_einval_locked;
 810
 811        if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
 812                goto out_einval_locked;
 813        read_unlock(&vmlist_lock);
 814
 815        addr += pgoff << PAGE_SHIFT;
 816        do {
 817                struct page *page = vmalloc_to_page(addr);
 818                ret = vm_insert_page(vma, uaddr, page);
 819                if (ret)
 820                        return ret;
 821
 822                uaddr += PAGE_SIZE;
 823                addr += PAGE_SIZE;
 824                usize -= PAGE_SIZE;
 825        } while (usize > 0);
 826
 827        /* Prevent "things" like memory migration? VM_flags need a cleanup... */
 828        vma->vm_flags |= VM_RESERVED;
 829
 830        return ret;
 831
 832out_einval_locked:
 833        read_unlock(&vmlist_lock);
 834        return -EINVAL;
 835}
 836EXPORT_SYMBOL(remap_vmalloc_range);
 837
 838/*
 839 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
 840 * have one.
 841 */
 842void  __attribute__((weak)) vmalloc_sync_all(void)
 843{
 844}
 845
 846
 847static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
 848{
 849        /* apply_to_page_range() does all the hard work. */
 850        return 0;
 851}
 852
 853/**
 854 *      alloc_vm_area - allocate a range of kernel address space
 855 *      @size:          size of the area
 856 *
 857 *      Returns:        NULL on failure, vm_struct on success
 858 *
 859 *      This function reserves a range of kernel address space, and
 860 *      allocates pagetables to map that range.  No actual mappings
 861 *      are created.  If the kernel address space is not shared
 862 *      between processes, it syncs the pagetable across all
 863 *      processes.
 864 */
 865struct vm_struct *alloc_vm_area(size_t size)
 866{
 867        struct vm_struct *area;
 868
 869        area = get_vm_area_caller(size, VM_IOREMAP,
 870                                __builtin_return_address(0));
 871        if (area == NULL)
 872                return NULL;
 873
 874        /*
 875         * This ensures that page tables are constructed for this region
 876         * of kernel virtual address space and mapped into init_mm.
 877         */
 878        if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
 879                                area->size, f, NULL)) {
 880                free_vm_area(area);
 881                return NULL;
 882        }
 883
 884        /* Make sure the pagetables are constructed in process kernel
 885           mappings */
 886        vmalloc_sync_all();
 887
 888        return area;
 889}
 890EXPORT_SYMBOL_GPL(alloc_vm_area);
 891
 892void free_vm_area(struct vm_struct *area)
 893{
 894        struct vm_struct *ret;
 895        ret = remove_vm_area(area->addr);
 896        BUG_ON(ret != area);
 897        kfree(area);
 898}
 899EXPORT_SYMBOL_GPL(free_vm_area);
 900
 901
 902#ifdef CONFIG_PROC_FS
 903static void *s_start(struct seq_file *m, loff_t *pos)
 904{
 905        loff_t n = *pos;
 906        struct vm_struct *v;
 907
 908        read_lock(&vmlist_lock);
 909        v = vmlist;
 910        while (n > 0 && v) {
 911                n--;
 912                v = v->next;
 913        }
 914        if (!n)
 915                return v;
 916
 917        return NULL;
 918
 919}
 920
 921static void *s_next(struct seq_file *m, void *p, loff_t *pos)
 922{
 923        struct vm_struct *v = p;
 924
 925        ++*pos;
 926        return v->next;
 927}
 928
 929static void s_stop(struct seq_file *m, void *p)
 930{
 931        read_unlock(&vmlist_lock);
 932}
 933
 934static int s_show(struct seq_file *m, void *p)
 935{
 936        struct vm_struct *v = p;
 937
 938        seq_printf(m, "0x%p-0x%p %7ld",
 939                v->addr, v->addr + v->size, v->size);
 940
 941        if (v->caller) {
 942                char buff[2 * KSYM_NAME_LEN];
 943
 944                seq_putc(m, ' ');
 945                sprint_symbol(buff, (unsigned long)v->caller);
 946                seq_puts(m, buff);
 947        }
 948
 949        if (v->nr_pages)
 950                seq_printf(m, " pages=%d", v->nr_pages);
 951
 952        if (v->phys_addr)
 953                seq_printf(m, " phys=%lx", v->phys_addr);
 954
 955        if (v->flags & VM_IOREMAP)
 956                seq_printf(m, " ioremap");
 957
 958        if (v->flags & VM_ALLOC)
 959                seq_printf(m, " vmalloc");
 960
 961        if (v->flags & VM_MAP)
 962                seq_printf(m, " vmap");
 963
 964        if (v->flags & VM_USERMAP)
 965                seq_printf(m, " user");
 966
 967        if (v->flags & VM_VPAGES)
 968                seq_printf(m, " vpages");
 969
 970        seq_putc(m, '\n');
 971        return 0;
 972}
 973
 974const struct seq_operations vmalloc_op = {
 975        .start = s_start,
 976        .next = s_next,
 977        .stop = s_stop,
 978        .show = s_show,
 979};
 980#endif
 981
 982