linux/mm/balloon_compaction.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * mm/balloon_compaction.c
   4 *
   5 * Common interface for making balloon pages movable by compaction.
   6 *
   7 * Copyright (C) 2012, Red Hat, Inc.  Rafael Aquini <aquini@redhat.com>
   8 */
   9#include <linux/mm.h>
  10#include <linux/slab.h>
  11#include <linux/export.h>
  12#include <linux/balloon_compaction.h>
  13
  14static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
  15                                     struct page *page)
  16{
  17        /*
  18         * Block others from accessing the 'page' when we get around to
  19         * establishing additional references. We should be the only one
  20         * holding a reference to the 'page' at this point. If we are not, then
  21         * memory corruption is possible and we should stop execution.
  22         */
  23        BUG_ON(!trylock_page(page));
  24        balloon_page_insert(b_dev_info, page);
  25        unlock_page(page);
  26        __count_vm_event(BALLOON_INFLATE);
  27}
  28
  29/**
  30 * balloon_page_list_enqueue() - inserts a list of pages into the balloon page
  31 *                               list.
  32 * @b_dev_info: balloon device descriptor where we will insert a new page to
  33 * @pages: pages to enqueue - allocated using balloon_page_alloc.
  34 *
  35 * Driver must call this function to properly enqueue balloon pages before
  36 * definitively removing them from the guest system.
  37 *
  38 * Return: number of pages that were enqueued.
  39 */
  40size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
  41                                 struct list_head *pages)
  42{
  43        struct page *page, *tmp;
  44        unsigned long flags;
  45        size_t n_pages = 0;
  46
  47        spin_lock_irqsave(&b_dev_info->pages_lock, flags);
  48        list_for_each_entry_safe(page, tmp, pages, lru) {
  49                list_del(&page->lru);
  50                balloon_page_enqueue_one(b_dev_info, page);
  51                n_pages++;
  52        }
  53        spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
  54        return n_pages;
  55}
  56EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);
  57
  58/**
  59 * balloon_page_list_dequeue() - removes pages from balloon's page list and
  60 *                               returns a list of the pages.
  61 * @b_dev_info: balloon device descriptor where we will grab a page from.
  62 * @pages: pointer to the list of pages that would be returned to the caller.
  63 * @n_req_pages: number of requested pages.
  64 *
  65 * Driver must call this function to properly de-allocate a previous enlisted
  66 * balloon pages before definitively releasing it back to the guest system.
  67 * This function tries to remove @n_req_pages from the ballooned pages and
  68 * return them to the caller in the @pages list.
  69 *
  70 * Note that this function may fail to dequeue some pages even if the balloon
  71 * isn't empty - since the page list can be temporarily empty due to compaction
  72 * of isolated pages.
  73 *
  74 * Return: number of pages that were added to the @pages list.
  75 */
  76size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
  77                                 struct list_head *pages, size_t n_req_pages)
  78{
  79        struct page *page, *tmp;
  80        unsigned long flags;
  81        size_t n_pages = 0;
  82
  83        spin_lock_irqsave(&b_dev_info->pages_lock, flags);
  84        list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
  85                if (n_pages == n_req_pages)
  86                        break;
  87
  88                /*
  89                 * Block others from accessing the 'page' while we get around to
  90                 * establishing additional references and preparing the 'page'
  91                 * to be released by the balloon driver.
  92                 */
  93                if (!trylock_page(page))
  94                        continue;
  95
  96                if (IS_ENABLED(CONFIG_BALLOON_COMPACTION) &&
  97                    PageIsolated(page)) {
  98                        /* raced with isolation */
  99                        unlock_page(page);
 100                        continue;
 101                }
 102                balloon_page_delete(page);
 103                __count_vm_event(BALLOON_DEFLATE);
 104                list_add(&page->lru, pages);
 105                unlock_page(page);
 106                n_pages++;
 107        }
 108        spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 109
 110        return n_pages;
 111}
 112EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);
 113
 114/*
 115 * balloon_page_alloc - allocates a new page for insertion into the balloon
 116 *                      page list.
 117 *
 118 * Driver must call this function to properly allocate a new balloon page.
 119 * Driver must call balloon_page_enqueue before definitively removing the page
 120 * from the guest system.
 121 *
 122 * Return: struct page for the allocated page or NULL on allocation failure.
 123 */
 124struct page *balloon_page_alloc(void)
 125{
 126        struct page *page = alloc_page(balloon_mapping_gfp_mask() |
 127                                       __GFP_NOMEMALLOC | __GFP_NORETRY |
 128                                       __GFP_NOWARN);
 129        return page;
 130}
 131EXPORT_SYMBOL_GPL(balloon_page_alloc);
 132
 133/*
 134 * balloon_page_enqueue - inserts a new page into the balloon page list.
 135 *
 136 * @b_dev_info: balloon device descriptor where we will insert a new page
 137 * @page: new page to enqueue - allocated using balloon_page_alloc.
 138 *
 139 * Drivers must call this function to properly enqueue a new allocated balloon
 140 * page before definitively removing the page from the guest system.
 141 *
 142 * Drivers must not call balloon_page_enqueue on pages that have been pushed to
 143 * a list with balloon_page_push before removing them with balloon_page_pop. To
 144 * enqueue a list of pages, use balloon_page_list_enqueue instead.
 145 */
 146void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
 147                          struct page *page)
 148{
 149        unsigned long flags;
 150
 151        spin_lock_irqsave(&b_dev_info->pages_lock, flags);
 152        balloon_page_enqueue_one(b_dev_info, page);
 153        spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 154}
 155EXPORT_SYMBOL_GPL(balloon_page_enqueue);
 156
 157/*
 158 * balloon_page_dequeue - removes a page from balloon's page list and returns
 159 *                        its address to allow the driver to release the page.
 160 * @b_dev_info: balloon device descriptor where we will grab a page from.
 161 *
 162 * Driver must call this function to properly dequeue a previously enqueued page
 163 * before definitively releasing it back to the guest system.
 164 *
 165 * Caller must perform its own accounting to ensure that this
 166 * function is called only if some pages are actually enqueued.
 167 *
 168 * Note that this function may fail to dequeue some pages even if there are
 169 * some enqueued pages - since the page list can be temporarily empty due to
 170 * the compaction of isolated pages.
 171 *
 172 * TODO: remove the caller accounting requirements, and allow caller to wait
 173 * until all pages can be dequeued.
 174 *
 175 * Return: struct page for the dequeued page, or NULL if no page was dequeued.
 176 */
 177struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
 178{
 179        unsigned long flags;
 180        LIST_HEAD(pages);
 181        int n_pages;
 182
 183        n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);
 184
 185        if (n_pages != 1) {
 186                /*
 187                 * If we are unable to dequeue a balloon page because the page
 188                 * list is empty and there are no isolated pages, then something
 189                 * went out of track and some balloon pages are lost.
 190                 * BUG() here, otherwise the balloon driver may get stuck in
 191                 * an infinite loop while attempting to release all its pages.
 192                 */
 193                spin_lock_irqsave(&b_dev_info->pages_lock, flags);
 194                if (unlikely(list_empty(&b_dev_info->pages) &&
 195                             !b_dev_info->isolated_pages))
 196                        BUG();
 197                spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 198                return NULL;
 199        }
 200        return list_first_entry(&pages, struct page, lru);
 201}
 202EXPORT_SYMBOL_GPL(balloon_page_dequeue);
 203
 204#ifdef CONFIG_BALLOON_COMPACTION
 205
 206bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
 207
 208{
 209        struct balloon_dev_info *b_dev_info = balloon_page_device(page);
 210        unsigned long flags;
 211
 212        spin_lock_irqsave(&b_dev_info->pages_lock, flags);
 213        list_del(&page->lru);
 214        b_dev_info->isolated_pages++;
 215        spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 216
 217        return true;
 218}
 219
 220void balloon_page_putback(struct page *page)
 221{
 222        struct balloon_dev_info *b_dev_info = balloon_page_device(page);
 223        unsigned long flags;
 224
 225        spin_lock_irqsave(&b_dev_info->pages_lock, flags);
 226        list_add(&page->lru, &b_dev_info->pages);
 227        b_dev_info->isolated_pages--;
 228        spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
 229}
 230
 231
 232/* move_to_new_page() counterpart for a ballooned page */
 233int balloon_page_migrate(struct address_space *mapping,
 234                struct page *newpage, struct page *page,
 235                enum migrate_mode mode)
 236{
 237        struct balloon_dev_info *balloon = balloon_page_device(page);
 238
 239        /*
 240         * We can not easily support the no copy case here so ignore it as it
 241         * is unlikely to be used with balloon pages. See include/linux/hmm.h
 242         * for a user of the MIGRATE_SYNC_NO_COPY mode.
 243         */
 244        if (mode == MIGRATE_SYNC_NO_COPY)
 245                return -EINVAL;
 246
 247        VM_BUG_ON_PAGE(!PageLocked(page), page);
 248        VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
 249
 250        return balloon->migratepage(balloon, newpage, page, mode);
 251}
 252
 253const struct address_space_operations balloon_aops = {
 254        .migratepage = balloon_page_migrate,
 255        .isolate_page = balloon_page_isolate,
 256        .putback_page = balloon_page_putback,
 257};
 258EXPORT_SYMBOL_GPL(balloon_aops);
 259
 260#endif /* CONFIG_BALLOON_COMPACTION */
 261
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.