linux/mm/percpu-vm.c
<<
>>
Prefs
   1/*
   2 * mm/percpu-vm.c - vmalloc area based chunk allocation
   3 *
   4 * Copyright (C) 2010           SUSE Linux Products GmbH
   5 * Copyright (C) 2010           Tejun Heo <tj@kernel.org>
   6 *
   7 * This file is released under the GPLv2.
   8 *
   9 * Chunks are mapped into vmalloc areas and populated page by page.
  10 * This is the default chunk allocator.
  11 */
  12
  13static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
  14                                    unsigned int cpu, int page_idx)
  15{
  16        /* must not be used on pre-mapped chunk */
  17        WARN_ON(chunk->immutable);
  18
  19        return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
  20}
  21
  22/**
  23 * pcpu_get_pages_and_bitmap - get temp pages array and bitmap
  24 * @chunk: chunk of interest
  25 * @bitmapp: output parameter for bitmap
  26 * @may_alloc: may allocate the array
  27 *
  28 * Returns pointer to array of pointers to struct page and bitmap,
  29 * both of which can be indexed with pcpu_page_idx().  The returned
  30 * array is cleared to zero and *@bitmapp is copied from
  31 * @chunk->populated.  Note that there is only one array and bitmap
  32 * and access exclusion is the caller's responsibility.
  33 *
  34 * CONTEXT:
  35 * pcpu_alloc_mutex and does GFP_KERNEL allocation if @may_alloc.
  36 * Otherwise, don't care.
  37 *
  38 * RETURNS:
  39 * Pointer to temp pages array on success, NULL on failure.
  40 */
  41static struct page **pcpu_get_pages_and_bitmap(struct pcpu_chunk *chunk,
  42                                               unsigned long **bitmapp,
  43                                               bool may_alloc)
  44{
  45        static struct page **pages;
  46        static unsigned long *bitmap;
  47        size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
  48        size_t bitmap_size = BITS_TO_LONGS(pcpu_unit_pages) *
  49                             sizeof(unsigned long);
  50
  51        if (!pages || !bitmap) {
  52                if (may_alloc && !pages)
  53                        pages = pcpu_mem_zalloc(pages_size);
  54                if (may_alloc && !bitmap)
  55                        bitmap = pcpu_mem_zalloc(bitmap_size);
  56                if (!pages || !bitmap)
  57                        return NULL;
  58        }
  59
  60        bitmap_copy(bitmap, chunk->populated, pcpu_unit_pages);
  61
  62        *bitmapp = bitmap;
  63        return pages;
  64}
  65
  66/**
  67 * pcpu_free_pages - free pages which were allocated for @chunk
  68 * @chunk: chunk pages were allocated for
  69 * @pages: array of pages to be freed, indexed by pcpu_page_idx()
  70 * @populated: populated bitmap
  71 * @page_start: page index of the first page to be freed
  72 * @page_end: page index of the last page to be freed + 1
  73 *
  74 * Free pages [@page_start and @page_end) in @pages for all units.
  75 * The pages were allocated for @chunk.
  76 */
  77static void pcpu_free_pages(struct pcpu_chunk *chunk,
  78                            struct page **pages, unsigned long *populated,
  79                            int page_start, int page_end)
  80{
  81        unsigned int cpu;
  82        int i;
  83
  84        for_each_possible_cpu(cpu) {
  85                for (i = page_start; i < page_end; i++) {
  86                        struct page *page = pages[pcpu_page_idx(cpu, i)];
  87
  88                        if (page)
  89                                __free_page(page);
  90                }
  91        }
  92}
  93
  94/**
  95 * pcpu_alloc_pages - allocates pages for @chunk
  96 * @chunk: target chunk
  97 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
  98 * @populated: populated bitmap
  99 * @page_start: page index of the first page to be allocated
 100 * @page_end: page index of the last page to be allocated + 1
 101 *
 102 * Allocate pages [@page_start,@page_end) into @pages for all units.
 103 * The allocation is for @chunk.  Percpu core doesn't care about the
 104 * content of @pages and will pass it verbatim to pcpu_map_pages().
 105 */
 106static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
 107                            struct page **pages, unsigned long *populated,
 108                            int page_start, int page_end)
 109{
 110        const gfp_t gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD;
 111        unsigned int cpu;
 112        int i;
 113
 114        for_each_possible_cpu(cpu) {
 115                for (i = page_start; i < page_end; i++) {
 116                        struct page **pagep = &pages[pcpu_page_idx(cpu, i)];
 117
 118                        *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
 119                        if (!*pagep) {
 120                                pcpu_free_pages(chunk, pages, populated,
 121                                                page_start, page_end);
 122                                return -ENOMEM;
 123                        }
 124                }
 125        }
 126        return 0;
 127}
 128
 129/**
 130 * pcpu_pre_unmap_flush - flush cache prior to unmapping
 131 * @chunk: chunk the regions to be flushed belongs to
 132 * @page_start: page index of the first page to be flushed
 133 * @page_end: page index of the last page to be flushed + 1
 134 *
 135 * Pages in [@page_start,@page_end) of @chunk are about to be
 136 * unmapped.  Flush cache.  As each flushing trial can be very
 137 * expensive, issue flush on the whole region at once rather than
 138 * doing it for each cpu.  This could be an overkill but is more
 139 * scalable.
 140 */
 141static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
 142                                 int page_start, int page_end)
 143{
 144        flush_cache_vunmap(
 145                pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
 146                pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
 147}
 148
 149static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
 150{
 151        unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT);
 152}
 153
 154/**
 155 * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
 156 * @chunk: chunk of interest
 157 * @pages: pages array which can be used to pass information to free
 158 * @populated: populated bitmap
 159 * @page_start: page index of the first page to unmap
 160 * @page_end: page index of the last page to unmap + 1
 161 *
 162 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
 163 * Corresponding elements in @pages were cleared by the caller and can
 164 * be used to carry information to pcpu_free_pages() which will be
 165 * called after all unmaps are finished.  The caller should call
 166 * proper pre/post flush functions.
 167 */
 168static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
 169                             struct page **pages, unsigned long *populated,
 170                             int page_start, int page_end)
 171{
 172        unsigned int cpu;
 173        int i;
 174
 175        for_each_possible_cpu(cpu) {
 176                for (i = page_start; i < page_end; i++) {
 177                        struct page *page;
 178
 179                        page = pcpu_chunk_page(chunk, cpu, i);
 180                        WARN_ON(!page);
 181                        pages[pcpu_page_idx(cpu, i)] = page;
 182                }
 183                __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
 184                                   page_end - page_start);
 185        }
 186
 187        bitmap_clear(populated, page_start, page_end - page_start);
 188}
 189
 190/**
 191 * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
 192 * @chunk: pcpu_chunk the regions to be flushed belong to
 193 * @page_start: page index of the first page to be flushed
 194 * @page_end: page index of the last page to be flushed + 1
 195 *
 196 * Pages [@page_start,@page_end) of @chunk have been unmapped.  Flush
 197 * TLB for the regions.  This can be skipped if the area is to be
 198 * returned to vmalloc as vmalloc will handle TLB flushing lazily.
 199 *
 200 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
 201 * for the whole region.
 202 */
 203static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
 204                                      int page_start, int page_end)
 205{
 206        flush_tlb_kernel_range(
 207                pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
 208                pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
 209}
 210
 211static int __pcpu_map_pages(unsigned long addr, struct page **pages,
 212                            int nr_pages)
 213{
 214        return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT,
 215                                        PAGE_KERNEL, pages);
 216}
 217
 218/**
 219 * pcpu_map_pages - map pages into a pcpu_chunk
 220 * @chunk: chunk of interest
 221 * @pages: pages array containing pages to be mapped
 222 * @populated: populated bitmap
 223 * @page_start: page index of the first page to map
 224 * @page_end: page index of the last page to map + 1
 225 *
 226 * For each cpu, map pages [@page_start,@page_end) into @chunk.  The
 227 * caller is responsible for calling pcpu_post_map_flush() after all
 228 * mappings are complete.
 229 *
 230 * This function is responsible for setting corresponding bits in
 231 * @chunk->populated bitmap and whatever is necessary for reverse
 232 * lookup (addr -> chunk).
 233 */
 234static int pcpu_map_pages(struct pcpu_chunk *chunk,
 235                          struct page **pages, unsigned long *populated,
 236                          int page_start, int page_end)
 237{
 238        unsigned int cpu, tcpu;
 239        int i, err;
 240
 241        for_each_possible_cpu(cpu) {
 242                err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
 243                                       &pages[pcpu_page_idx(cpu, page_start)],
 244                                       page_end - page_start);
 245                if (err < 0)
 246                        goto err;
 247        }
 248
 249        /* mapping successful, link chunk and mark populated */
 250        for (i = page_start; i < page_end; i++) {
 251                for_each_possible_cpu(cpu)
 252                        pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
 253                                            chunk);
 254                __set_bit(i, populated);
 255        }
 256
 257        return 0;
 258
 259err:
 260        for_each_possible_cpu(tcpu) {
 261                if (tcpu == cpu)
 262                        break;
 263                __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
 264                                   page_end - page_start);
 265        }
 266        return err;
 267}
 268
 269/**
 270 * pcpu_post_map_flush - flush cache after mapping
 271 * @chunk: pcpu_chunk the regions to be flushed belong to
 272 * @page_start: page index of the first page to be flushed
 273 * @page_end: page index of the last page to be flushed + 1
 274 *
 275 * Pages [@page_start,@page_end) of @chunk have been mapped.  Flush
 276 * cache.
 277 *
 278 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
 279 * for the whole region.
 280 */
 281static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
 282                                int page_start, int page_end)
 283{
 284        flush_cache_vmap(
 285                pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
 286                pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
 287}
 288
 289/**
 290 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
 291 * @chunk: chunk of interest
 292 * @off: offset to the area to populate
 293 * @size: size of the area to populate in bytes
 294 *
 295 * For each cpu, populate and map pages [@page_start,@page_end) into
 296 * @chunk.  The area is cleared on return.
 297 *
 298 * CONTEXT:
 299 * pcpu_alloc_mutex, does GFP_KERNEL allocation.
 300 */
 301static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
 302{
 303        int page_start = PFN_DOWN(off);
 304        int page_end = PFN_UP(off + size);
 305        int free_end = page_start, unmap_end = page_start;
 306        struct page **pages;
 307        unsigned long *populated;
 308        unsigned int cpu;
 309        int rs, re, rc;
 310
 311        /* quick path, check whether all pages are already there */
 312        rs = page_start;
 313        pcpu_next_pop(chunk, &rs, &re, page_end);
 314        if (rs == page_start && re == page_end)
 315                goto clear;
 316
 317        /* need to allocate and map pages, this chunk can't be immutable */
 318        WARN_ON(chunk->immutable);
 319
 320        pages = pcpu_get_pages_and_bitmap(chunk, &populated, true);
 321        if (!pages)
 322                return -ENOMEM;
 323
 324        /* alloc and map */
 325        pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
 326                rc = pcpu_alloc_pages(chunk, pages, populated, rs, re);
 327                if (rc)
 328                        goto err_free;
 329                free_end = re;
 330        }
 331
 332        pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
 333                rc = pcpu_map_pages(chunk, pages, populated, rs, re);
 334                if (rc)
 335                        goto err_unmap;
 336                unmap_end = re;
 337        }
 338        pcpu_post_map_flush(chunk, page_start, page_end);
 339
 340        /* commit new bitmap */
 341        bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
 342clear:
 343        for_each_possible_cpu(cpu)
 344                memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
 345        return 0;
 346
 347err_unmap:
 348        pcpu_pre_unmap_flush(chunk, page_start, unmap_end);
 349        pcpu_for_each_unpop_region(chunk, rs, re, page_start, unmap_end)
 350                pcpu_unmap_pages(chunk, pages, populated, rs, re);
 351        pcpu_post_unmap_tlb_flush(chunk, page_start, unmap_end);
 352err_free:
 353        pcpu_for_each_unpop_region(chunk, rs, re, page_start, free_end)
 354                pcpu_free_pages(chunk, pages, populated, rs, re);
 355        return rc;
 356}
 357
 358/**
 359 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
 360 * @chunk: chunk to depopulate
 361 * @off: offset to the area to depopulate
 362 * @size: size of the area to depopulate in bytes
 363 *
 364 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
 365 * from @chunk.  If @flush is true, vcache is flushed before unmapping
 366 * and tlb after.
 367 *
 368 * CONTEXT:
 369 * pcpu_alloc_mutex.
 370 */
 371static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
 372{
 373        int page_start = PFN_DOWN(off);
 374        int page_end = PFN_UP(off + size);
 375        struct page **pages;
 376        unsigned long *populated;
 377        int rs, re;
 378
 379        /* quick path, check whether it's empty already */
 380        rs = page_start;
 381        pcpu_next_unpop(chunk, &rs, &re, page_end);
 382        if (rs == page_start && re == page_end)
 383                return;
 384
 385        /* immutable chunks can't be depopulated */
 386        WARN_ON(chunk->immutable);
 387
 388        /*
 389         * If control reaches here, there must have been at least one
 390         * successful population attempt so the temp pages array must
 391         * be available now.
 392         */
 393        pages = pcpu_get_pages_and_bitmap(chunk, &populated, false);
 394        BUG_ON(!pages);
 395
 396        /* unmap and free */
 397        pcpu_pre_unmap_flush(chunk, page_start, page_end);
 398
 399        pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
 400                pcpu_unmap_pages(chunk, pages, populated, rs, re);
 401
 402        /* no need to flush tlb, vmalloc will handle it lazily */
 403
 404        pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
 405                pcpu_free_pages(chunk, pages, populated, rs, re);
 406
 407        /* commit new bitmap */
 408        bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
 409}
 410
 411static struct pcpu_chunk *pcpu_create_chunk(void)
 412{
 413        struct pcpu_chunk *chunk;
 414        struct vm_struct **vms;
 415
 416        chunk = pcpu_alloc_chunk();
 417        if (!chunk)
 418                return NULL;
 419
 420        vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
 421                                pcpu_nr_groups, pcpu_atom_size);
 422        if (!vms) {
 423                pcpu_free_chunk(chunk);
 424                return NULL;
 425        }
 426
 427        chunk->data = vms;
 428        chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];
 429        return chunk;
 430}
 431
 432static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
 433{
 434        if (chunk && chunk->data)
 435                pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);
 436        pcpu_free_chunk(chunk);
 437}
 438
 439static struct page *pcpu_addr_to_page(void *addr)
 440{
 441        return vmalloc_to_page(addr);
 442}
 443
 444static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
 445{
 446        /* no extra restriction */
 447        return 0;
 448}
 449
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.