linux/fs/fscache/page.c
<<
>>
Prefs
   1/* Cache page management and data I/O routines
   2 *
   3 * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
   4 * Written by David Howells (dhowells@redhat.com)
   5 *
   6 * This program is free software; you can redistribute it and/or
   7 * modify it under the terms of the GNU General Public License
   8 * as published by the Free Software Foundation; either version
   9 * 2 of the License, or (at your option) any later version.
  10 */
  11
  12#define FSCACHE_DEBUG_LEVEL PAGE
  13#include <linux/module.h>
  14#include <linux/fscache-cache.h>
  15#include <linux/buffer_head.h>
  16#include <linux/pagevec.h>
  17#include <linux/slab.h>
  18#include "internal.h"
  19
  20/*
  21 * check to see if a page is being written to the cache
  22 */
  23bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
  24{
  25        void *val;
  26
  27        rcu_read_lock();
  28        val = radix_tree_lookup(&cookie->stores, page->index);
  29        rcu_read_unlock();
  30
  31        return val != NULL;
  32}
  33EXPORT_SYMBOL(__fscache_check_page_write);
  34
  35/*
  36 * wait for a page to finish being written to the cache
  37 */
  38void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
  39{
  40        wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
  41
  42        wait_event(*wq, !__fscache_check_page_write(cookie, page));
  43}
  44EXPORT_SYMBOL(__fscache_wait_on_page_write);
  45
  46/*
  47 * decide whether a page can be released, possibly by cancelling a store to it
  48 * - we're allowed to sleep if __GFP_WAIT is flagged
  49 */
  50bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
  51                                  struct page *page,
  52                                  gfp_t gfp)
  53{
  54        struct page *xpage;
  55        void *val;
  56
  57        _enter("%p,%p,%x", cookie, page, gfp);
  58
  59try_again:
  60        rcu_read_lock();
  61        val = radix_tree_lookup(&cookie->stores, page->index);
  62        if (!val) {
  63                rcu_read_unlock();
  64                fscache_stat(&fscache_n_store_vmscan_not_storing);
  65                __fscache_uncache_page(cookie, page);
  66                return true;
  67        }
  68
  69        /* see if the page is actually undergoing storage - if so we can't get
  70         * rid of it till the cache has finished with it */
  71        if (radix_tree_tag_get(&cookie->stores, page->index,
  72                               FSCACHE_COOKIE_STORING_TAG)) {
  73                rcu_read_unlock();
  74                goto page_busy;
  75        }
  76
  77        /* the page is pending storage, so we attempt to cancel the store and
  78         * discard the store request so that the page can be reclaimed */
  79        spin_lock(&cookie->stores_lock);
  80        rcu_read_unlock();
  81
  82        if (radix_tree_tag_get(&cookie->stores, page->index,
  83                               FSCACHE_COOKIE_STORING_TAG)) {
  84                /* the page started to undergo storage whilst we were looking,
  85                 * so now we can only wait or return */
  86                spin_unlock(&cookie->stores_lock);
  87                goto page_busy;
  88        }
  89
  90        xpage = radix_tree_delete(&cookie->stores, page->index);
  91        spin_unlock(&cookie->stores_lock);
  92
  93        if (xpage) {
  94                fscache_stat(&fscache_n_store_vmscan_cancelled);
  95                fscache_stat(&fscache_n_store_radix_deletes);
  96                ASSERTCMP(xpage, ==, page);
  97        } else {
  98                fscache_stat(&fscache_n_store_vmscan_gone);
  99        }
 100
 101        wake_up_bit(&cookie->flags, 0);
 102        if (xpage)
 103                page_cache_release(xpage);
 104        __fscache_uncache_page(cookie, page);
 105        return true;
 106
 107page_busy:
 108        /* We will wait here if we're allowed to, but that could deadlock the
 109         * allocator as the work threads writing to the cache may all end up
 110         * sleeping on memory allocation, so we may need to impose a timeout
 111         * too. */
 112        if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) {
 113                fscache_stat(&fscache_n_store_vmscan_busy);
 114                return false;
 115        }
 116
 117        fscache_stat(&fscache_n_store_vmscan_wait);
 118        __fscache_wait_on_page_write(cookie, page);
 119        gfp &= ~__GFP_WAIT;
 120        goto try_again;
 121}
 122EXPORT_SYMBOL(__fscache_maybe_release_page);
 123
 124/*
 125 * note that a page has finished being written to the cache
 126 */
 127static void fscache_end_page_write(struct fscache_object *object,
 128                                   struct page *page)
 129{
 130        struct fscache_cookie *cookie;
 131        struct page *xpage = NULL;
 132
 133        spin_lock(&object->lock);
 134        cookie = object->cookie;
 135        if (cookie) {
 136                /* delete the page from the tree if it is now no longer
 137                 * pending */
 138                spin_lock(&cookie->stores_lock);
 139                radix_tree_tag_clear(&cookie->stores, page->index,
 140                                     FSCACHE_COOKIE_STORING_TAG);
 141                if (!radix_tree_tag_get(&cookie->stores, page->index,
 142                                        FSCACHE_COOKIE_PENDING_TAG)) {
 143                        fscache_stat(&fscache_n_store_radix_deletes);
 144                        xpage = radix_tree_delete(&cookie->stores, page->index);
 145                }
 146                spin_unlock(&cookie->stores_lock);
 147                wake_up_bit(&cookie->flags, 0);
 148        }
 149        spin_unlock(&object->lock);
 150        if (xpage)
 151                page_cache_release(xpage);
 152}
 153
 154/*
 155 * actually apply the changed attributes to a cache object
 156 */
 157static void fscache_attr_changed_op(struct fscache_operation *op)
 158{
 159        struct fscache_object *object = op->object;
 160        int ret;
 161
 162        _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
 163
 164        fscache_stat(&fscache_n_attr_changed_calls);
 165
 166        if (fscache_object_is_active(object)) {
 167                fscache_stat(&fscache_n_cop_attr_changed);
 168                ret = object->cache->ops->attr_changed(object);
 169                fscache_stat_d(&fscache_n_cop_attr_changed);
 170                if (ret < 0)
 171                        fscache_abort_object(object);
 172        }
 173
 174        fscache_op_complete(op, true);
 175        _leave("");
 176}
 177
 178/*
 179 * notification that the attributes on an object have changed
 180 */
 181int __fscache_attr_changed(struct fscache_cookie *cookie)
 182{
 183        struct fscache_operation *op;
 184        struct fscache_object *object;
 185        bool wake_cookie;
 186
 187        _enter("%p", cookie);
 188
 189        ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 190
 191        fscache_stat(&fscache_n_attr_changed);
 192
 193        op = kzalloc(sizeof(*op), GFP_KERNEL);
 194        if (!op) {
 195                fscache_stat(&fscache_n_attr_changed_nomem);
 196                _leave(" = -ENOMEM");
 197                return -ENOMEM;
 198        }
 199
 200        fscache_operation_init(op, fscache_attr_changed_op, NULL);
 201        op->flags = FSCACHE_OP_ASYNC |
 202                (1 << FSCACHE_OP_EXCLUSIVE) |
 203                (1 << FSCACHE_OP_UNUSE_COOKIE);
 204
 205        spin_lock(&cookie->lock);
 206
 207        if (!fscache_cookie_enabled(cookie) ||
 208            hlist_empty(&cookie->backing_objects))
 209                goto nobufs;
 210        object = hlist_entry(cookie->backing_objects.first,
 211                             struct fscache_object, cookie_link);
 212
 213        __fscache_use_cookie(cookie);
 214        if (fscache_submit_exclusive_op(object, op) < 0)
 215                goto nobufs;
 216        spin_unlock(&cookie->lock);
 217        fscache_stat(&fscache_n_attr_changed_ok);
 218        fscache_put_operation(op);
 219        _leave(" = 0");
 220        return 0;
 221
 222nobufs:
 223        wake_cookie = __fscache_unuse_cookie(cookie);
 224        spin_unlock(&cookie->lock);
 225        kfree(op);
 226        if (wake_cookie)
 227                __fscache_wake_unused_cookie(cookie);
 228        fscache_stat(&fscache_n_attr_changed_nobufs);
 229        _leave(" = %d", -ENOBUFS);
 230        return -ENOBUFS;
 231}
 232EXPORT_SYMBOL(__fscache_attr_changed);
 233
 234/*
 235 * release a retrieval op reference
 236 */
 237static void fscache_release_retrieval_op(struct fscache_operation *_op)
 238{
 239        struct fscache_retrieval *op =
 240                container_of(_op, struct fscache_retrieval, op);
 241
 242        _enter("{OP%x}", op->op.debug_id);
 243
 244        ASSERTCMP(atomic_read(&op->n_pages), ==, 0);
 245
 246        fscache_hist(fscache_retrieval_histogram, op->start_time);
 247        if (op->context)
 248                fscache_put_context(op->op.object->cookie, op->context);
 249
 250        _leave("");
 251}
 252
 253/*
 254 * allocate a retrieval op
 255 */
 256static struct fscache_retrieval *fscache_alloc_retrieval(
 257        struct fscache_cookie *cookie,
 258        struct address_space *mapping,
 259        fscache_rw_complete_t end_io_func,
 260        void *context)
 261{
 262        struct fscache_retrieval *op;
 263
 264        /* allocate a retrieval operation and attempt to submit it */
 265        op = kzalloc(sizeof(*op), GFP_NOIO);
 266        if (!op) {
 267                fscache_stat(&fscache_n_retrievals_nomem);
 268                return NULL;
 269        }
 270
 271        fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op);
 272        op->op.flags    = FSCACHE_OP_MYTHREAD |
 273                (1UL << FSCACHE_OP_WAITING) |
 274                (1UL << FSCACHE_OP_UNUSE_COOKIE);
 275        op->mapping     = mapping;
 276        op->end_io_func = end_io_func;
 277        op->context     = context;
 278        op->start_time  = jiffies;
 279        INIT_LIST_HEAD(&op->to_do);
 280        return op;
 281}
 282
 283/*
 284 * wait for a deferred lookup to complete
 285 */
 286int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
 287{
 288        unsigned long jif;
 289
 290        _enter("");
 291
 292        if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
 293                _leave(" = 0 [imm]");
 294                return 0;
 295        }
 296
 297        fscache_stat(&fscache_n_retrievals_wait);
 298
 299        jif = jiffies;
 300        if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 301                        fscache_wait_bit_interruptible,
 302                        TASK_INTERRUPTIBLE) != 0) {
 303                fscache_stat(&fscache_n_retrievals_intr);
 304                _leave(" = -ERESTARTSYS");
 305                return -ERESTARTSYS;
 306        }
 307
 308        ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
 309
 310        smp_rmb();
 311        fscache_hist(fscache_retrieval_delay_histogram, jif);
 312        _leave(" = 0 [dly]");
 313        return 0;
 314}
 315
 316/*
 317 * Handle cancellation of a pending retrieval op
 318 */
 319static void fscache_do_cancel_retrieval(struct fscache_operation *_op)
 320{
 321        struct fscache_retrieval *op =
 322                container_of(_op, struct fscache_retrieval, op);
 323
 324        atomic_set(&op->n_pages, 0);
 325}
 326
 327/*
 328 * wait for an object to become active (or dead)
 329 */
 330int fscache_wait_for_operation_activation(struct fscache_object *object,
 331                                          struct fscache_operation *op,
 332                                          atomic_t *stat_op_waits,
 333                                          atomic_t *stat_object_dead,
 334                                          void (*do_cancel)(struct fscache_operation *))
 335{
 336        int ret;
 337
 338        if (!test_bit(FSCACHE_OP_WAITING, &op->flags))
 339                goto check_if_dead;
 340
 341        _debug(">>> WT");
 342        if (stat_op_waits)
 343                fscache_stat(stat_op_waits);
 344        if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 345                        fscache_wait_bit_interruptible,
 346                        TASK_INTERRUPTIBLE) != 0) {
 347                ret = fscache_cancel_op(op, do_cancel);
 348                if (ret == 0)
 349                        return -ERESTARTSYS;
 350
 351                /* it's been removed from the pending queue by another party,
 352                 * so we should get to run shortly */
 353                wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
 354                            fscache_wait_bit, TASK_UNINTERRUPTIBLE);
 355        }
 356        _debug("<<< GO");
 357
 358check_if_dead:
 359        if (op->state == FSCACHE_OP_ST_CANCELLED) {
 360                if (stat_object_dead)
 361                        fscache_stat(stat_object_dead);
 362                _leave(" = -ENOBUFS [cancelled]");
 363                return -ENOBUFS;
 364        }
 365        if (unlikely(fscache_object_is_dead(object))) {
 366                pr_err("%s() = -ENOBUFS [obj dead %d]\n", __func__, op->state);
 367                fscache_cancel_op(op, do_cancel);
 368                if (stat_object_dead)
 369                        fscache_stat(stat_object_dead);
 370                return -ENOBUFS;
 371        }
 372        return 0;
 373}
 374
 375/*
 376 * read a page from the cache or allocate a block in which to store it
 377 * - we return:
 378 *   -ENOMEM    - out of memory, nothing done
 379 *   -ERESTARTSYS - interrupted
 380 *   -ENOBUFS   - no backing object available in which to cache the block
 381 *   -ENODATA   - no data available in the backing object for this block
 382 *   0          - dispatched a read - it'll call end_io_func() when finished
 383 */
 384int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
 385                                 struct page *page,
 386                                 fscache_rw_complete_t end_io_func,
 387                                 void *context,
 388                                 gfp_t gfp)
 389{
 390        struct fscache_retrieval *op;
 391        struct fscache_object *object;
 392        bool wake_cookie = false;
 393        int ret;
 394
 395        _enter("%p,%p,,,", cookie, page);
 396
 397        fscache_stat(&fscache_n_retrievals);
 398
 399        if (hlist_empty(&cookie->backing_objects))
 400                goto nobufs;
 401
 402        if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 403                _leave(" = -ENOBUFS [invalidating]");
 404                return -ENOBUFS;
 405        }
 406
 407        ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 408        ASSERTCMP(page, !=, NULL);
 409
 410        if (fscache_wait_for_deferred_lookup(cookie) < 0)
 411                return -ERESTARTSYS;
 412
 413        op = fscache_alloc_retrieval(cookie, page->mapping,
 414                                     end_io_func, context);
 415        if (!op) {
 416                _leave(" = -ENOMEM");
 417                return -ENOMEM;
 418        }
 419        atomic_set(&op->n_pages, 1);
 420
 421        spin_lock(&cookie->lock);
 422
 423        if (!fscache_cookie_enabled(cookie) ||
 424            hlist_empty(&cookie->backing_objects))
 425                goto nobufs_unlock;
 426        object = hlist_entry(cookie->backing_objects.first,
 427                             struct fscache_object, cookie_link);
 428
 429        ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags));
 430
 431        __fscache_use_cookie(cookie);
 432        atomic_inc(&object->n_reads);
 433        __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 434
 435        if (fscache_submit_op(object, &op->op) < 0)
 436                goto nobufs_unlock_dec;
 437        spin_unlock(&cookie->lock);
 438
 439        fscache_stat(&fscache_n_retrieval_ops);
 440
 441        /* pin the netfs read context in case we need to do the actual netfs
 442         * read because we've encountered a cache read failure */
 443        fscache_get_context(object->cookie, op->context);
 444
 445        /* we wait for the operation to become active, and then process it
 446         * *here*, in this thread, and not in the thread pool */
 447        ret = fscache_wait_for_operation_activation(
 448                object, &op->op,
 449                __fscache_stat(&fscache_n_retrieval_op_waits),
 450                __fscache_stat(&fscache_n_retrievals_object_dead),
 451                fscache_do_cancel_retrieval);
 452        if (ret < 0)
 453                goto error;
 454
 455        /* ask the cache to honour the operation */
 456        if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 457                fscache_stat(&fscache_n_cop_allocate_page);
 458                ret = object->cache->ops->allocate_page(op, page, gfp);
 459                fscache_stat_d(&fscache_n_cop_allocate_page);
 460                if (ret == 0)
 461                        ret = -ENODATA;
 462        } else {
 463                fscache_stat(&fscache_n_cop_read_or_alloc_page);
 464                ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
 465                fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
 466        }
 467
 468error:
 469        if (ret == -ENOMEM)
 470                fscache_stat(&fscache_n_retrievals_nomem);
 471        else if (ret == -ERESTARTSYS)
 472                fscache_stat(&fscache_n_retrievals_intr);
 473        else if (ret == -ENODATA)
 474                fscache_stat(&fscache_n_retrievals_nodata);
 475        else if (ret < 0)
 476                fscache_stat(&fscache_n_retrievals_nobufs);
 477        else
 478                fscache_stat(&fscache_n_retrievals_ok);
 479
 480        fscache_put_retrieval(op);
 481        _leave(" = %d", ret);
 482        return ret;
 483
 484nobufs_unlock_dec:
 485        atomic_dec(&object->n_reads);
 486        wake_cookie = __fscache_unuse_cookie(cookie);
 487nobufs_unlock:
 488        spin_unlock(&cookie->lock);
 489        if (wake_cookie)
 490                __fscache_wake_unused_cookie(cookie);
 491        kfree(op);
 492nobufs:
 493        fscache_stat(&fscache_n_retrievals_nobufs);
 494        _leave(" = -ENOBUFS");
 495        return -ENOBUFS;
 496}
 497EXPORT_SYMBOL(__fscache_read_or_alloc_page);
 498
 499/*
 500 * read a list of page from the cache or allocate a block in which to store
 501 * them
 502 * - we return:
 503 *   -ENOMEM    - out of memory, some pages may be being read
 504 *   -ERESTARTSYS - interrupted, some pages may be being read
 505 *   -ENOBUFS   - no backing object or space available in which to cache any
 506 *                pages not being read
 507 *   -ENODATA   - no data available in the backing object for some or all of
 508 *                the pages
 509 *   0          - dispatched a read on all pages
 510 *
 511 * end_io_func() will be called for each page read from the cache as it is
 512 * finishes being read
 513 *
 514 * any pages for which a read is dispatched will be removed from pages and
 515 * nr_pages
 516 */
 517int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
 518                                  struct address_space *mapping,
 519                                  struct list_head *pages,
 520                                  unsigned *nr_pages,
 521                                  fscache_rw_complete_t end_io_func,
 522                                  void *context,
 523                                  gfp_t gfp)
 524{
 525        struct fscache_retrieval *op;
 526        struct fscache_object *object;
 527        bool wake_cookie = false;
 528        int ret;
 529
 530        _enter("%p,,%d,,,", cookie, *nr_pages);
 531
 532        fscache_stat(&fscache_n_retrievals);
 533
 534        if (hlist_empty(&cookie->backing_objects))
 535                goto nobufs;
 536
 537        if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 538                _leave(" = -ENOBUFS [invalidating]");
 539                return -ENOBUFS;
 540        }
 541
 542        ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 543        ASSERTCMP(*nr_pages, >, 0);
 544        ASSERT(!list_empty(pages));
 545
 546        if (fscache_wait_for_deferred_lookup(cookie) < 0)
 547                return -ERESTARTSYS;
 548
 549        op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);
 550        if (!op)
 551                return -ENOMEM;
 552        atomic_set(&op->n_pages, *nr_pages);
 553
 554        spin_lock(&cookie->lock);
 555
 556        if (!fscache_cookie_enabled(cookie) ||
 557            hlist_empty(&cookie->backing_objects))
 558                goto nobufs_unlock;
 559        object = hlist_entry(cookie->backing_objects.first,
 560                             struct fscache_object, cookie_link);
 561
 562        __fscache_use_cookie(cookie);
 563        atomic_inc(&object->n_reads);
 564        __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
 565
 566        if (fscache_submit_op(object, &op->op) < 0)
 567                goto nobufs_unlock_dec;
 568        spin_unlock(&cookie->lock);
 569
 570        fscache_stat(&fscache_n_retrieval_ops);
 571
 572        /* pin the netfs read context in case we need to do the actual netfs
 573         * read because we've encountered a cache read failure */
 574        fscache_get_context(object->cookie, op->context);
 575
 576        /* we wait for the operation to become active, and then process it
 577         * *here*, in this thread, and not in the thread pool */
 578        ret = fscache_wait_for_operation_activation(
 579                object, &op->op,
 580                __fscache_stat(&fscache_n_retrieval_op_waits),
 581                __fscache_stat(&fscache_n_retrievals_object_dead),
 582                fscache_do_cancel_retrieval);
 583        if (ret < 0)
 584                goto error;
 585
 586        /* ask the cache to honour the operation */
 587        if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
 588                fscache_stat(&fscache_n_cop_allocate_pages);
 589                ret = object->cache->ops->allocate_pages(
 590                        op, pages, nr_pages, gfp);
 591                fscache_stat_d(&fscache_n_cop_allocate_pages);
 592        } else {
 593                fscache_stat(&fscache_n_cop_read_or_alloc_pages);
 594                ret = object->cache->ops->read_or_alloc_pages(
 595                        op, pages, nr_pages, gfp);
 596                fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
 597        }
 598
 599error:
 600        if (ret == -ENOMEM)
 601                fscache_stat(&fscache_n_retrievals_nomem);
 602        else if (ret == -ERESTARTSYS)
 603                fscache_stat(&fscache_n_retrievals_intr);
 604        else if (ret == -ENODATA)
 605                fscache_stat(&fscache_n_retrievals_nodata);
 606        else if (ret < 0)
 607                fscache_stat(&fscache_n_retrievals_nobufs);
 608        else
 609                fscache_stat(&fscache_n_retrievals_ok);
 610
 611        fscache_put_retrieval(op);
 612        _leave(" = %d", ret);
 613        return ret;
 614
 615nobufs_unlock_dec:
 616        atomic_dec(&object->n_reads);
 617        wake_cookie = __fscache_unuse_cookie(cookie);
 618nobufs_unlock:
 619        spin_unlock(&cookie->lock);
 620        kfree(op);
 621        if (wake_cookie)
 622                __fscache_wake_unused_cookie(cookie);
 623nobufs:
 624        fscache_stat(&fscache_n_retrievals_nobufs);
 625        _leave(" = -ENOBUFS");
 626        return -ENOBUFS;
 627}
 628EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
 629
 630/*
 631 * allocate a block in the cache on which to store a page
 632 * - we return:
 633 *   -ENOMEM    - out of memory, nothing done
 634 *   -ERESTARTSYS - interrupted
 635 *   -ENOBUFS   - no backing object available in which to cache the block
 636 *   0          - block allocated
 637 */
 638int __fscache_alloc_page(struct fscache_cookie *cookie,
 639                         struct page *page,
 640                         gfp_t gfp)
 641{
 642        struct fscache_retrieval *op;
 643        struct fscache_object *object;
 644        bool wake_cookie = false;
 645        int ret;
 646
 647        _enter("%p,%p,,,", cookie, page);
 648
 649        fscache_stat(&fscache_n_allocs);
 650
 651        if (hlist_empty(&cookie->backing_objects))
 652                goto nobufs;
 653
 654        ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 655        ASSERTCMP(page, !=, NULL);
 656
 657        if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 658                _leave(" = -ENOBUFS [invalidating]");
 659                return -ENOBUFS;
 660        }
 661
 662        if (fscache_wait_for_deferred_lookup(cookie) < 0)
 663                return -ERESTARTSYS;
 664
 665        op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);
 666        if (!op)
 667                return -ENOMEM;
 668        atomic_set(&op->n_pages, 1);
 669
 670        spin_lock(&cookie->lock);
 671
 672        if (!fscache_cookie_enabled(cookie) ||
 673            hlist_empty(&cookie->backing_objects))
 674                goto nobufs_unlock;
 675        object = hlist_entry(cookie->backing_objects.first,
 676                             struct fscache_object, cookie_link);
 677
 678        __fscache_use_cookie(cookie);
 679        if (fscache_submit_op(object, &op->op) < 0)
 680                goto nobufs_unlock_dec;
 681        spin_unlock(&cookie->lock);
 682
 683        fscache_stat(&fscache_n_alloc_ops);
 684
 685        ret = fscache_wait_for_operation_activation(
 686                object, &op->op,
 687                __fscache_stat(&fscache_n_alloc_op_waits),
 688                __fscache_stat(&fscache_n_allocs_object_dead),
 689                fscache_do_cancel_retrieval);
 690        if (ret < 0)
 691                goto error;
 692
 693        /* ask the cache to honour the operation */
 694        fscache_stat(&fscache_n_cop_allocate_page);
 695        ret = object->cache->ops->allocate_page(op, page, gfp);
 696        fscache_stat_d(&fscache_n_cop_allocate_page);
 697
 698error:
 699        if (ret == -ERESTARTSYS)
 700                fscache_stat(&fscache_n_allocs_intr);
 701        else if (ret < 0)
 702                fscache_stat(&fscache_n_allocs_nobufs);
 703        else
 704                fscache_stat(&fscache_n_allocs_ok);
 705
 706        fscache_put_retrieval(op);
 707        _leave(" = %d", ret);
 708        return ret;
 709
 710nobufs_unlock_dec:
 711        wake_cookie = __fscache_unuse_cookie(cookie);
 712nobufs_unlock:
 713        spin_unlock(&cookie->lock);
 714        kfree(op);
 715        if (wake_cookie)
 716                __fscache_wake_unused_cookie(cookie);
 717nobufs:
 718        fscache_stat(&fscache_n_allocs_nobufs);
 719        _leave(" = -ENOBUFS");
 720        return -ENOBUFS;
 721}
 722EXPORT_SYMBOL(__fscache_alloc_page);
 723
 724/*
 725 * Unmark pages allocate in the readahead code path (via:
 726 * fscache_readpages_or_alloc) after delegating to the base filesystem
 727 */
 728void __fscache_readpages_cancel(struct fscache_cookie *cookie,
 729                                struct list_head *pages)
 730{
 731        struct page *page;
 732
 733        list_for_each_entry(page, pages, lru) {
 734                if (PageFsCache(page))
 735                        __fscache_uncache_page(cookie, page);
 736        }
 737}
 738EXPORT_SYMBOL(__fscache_readpages_cancel);
 739
 740/*
 741 * release a write op reference
 742 */
 743static void fscache_release_write_op(struct fscache_operation *_op)
 744{
 745        _enter("{OP%x}", _op->debug_id);
 746}
 747
 748/*
 749 * perform the background storage of a page into the cache
 750 */
 751static void fscache_write_op(struct fscache_operation *_op)
 752{
 753        struct fscache_storage *op =
 754                container_of(_op, struct fscache_storage, op);
 755        struct fscache_object *object = op->op.object;
 756        struct fscache_cookie *cookie;
 757        struct page *page;
 758        unsigned n;
 759        void *results[1];
 760        int ret;
 761
 762        _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
 763
 764        spin_lock(&object->lock);
 765        cookie = object->cookie;
 766
 767        if (!fscache_object_is_active(object)) {
 768                /* If we get here, then the on-disk cache object likely longer
 769                 * exists, so we should just cancel this write operation.
 770                 */
 771                spin_unlock(&object->lock);
 772                fscache_op_complete(&op->op, false);
 773                _leave(" [inactive]");
 774                return;
 775        }
 776
 777        if (!cookie) {
 778                /* If we get here, then the cookie belonging to the object was
 779                 * detached, probably by the cookie being withdrawn due to
 780                 * memory pressure, which means that the pages we might write
 781                 * to the cache from no longer exist - therefore, we can just
 782                 * cancel this write operation.
 783                 */
 784                spin_unlock(&object->lock);
 785                fscache_op_complete(&op->op, false);
 786                _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}",
 787                       _op->flags, _op->state, object->state->short_name,
 788                       object->flags);
 789                return;
 790        }
 791
 792        spin_lock(&cookie->stores_lock);
 793
 794        fscache_stat(&fscache_n_store_calls);
 795
 796        /* find a page to store */
 797        page = NULL;
 798        n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
 799                                       FSCACHE_COOKIE_PENDING_TAG);
 800        if (n != 1)
 801                goto superseded;
 802        page = results[0];
 803        _debug("gang %d [%lx]", n, page->index);
 804        if (page->index > op->store_limit) {
 805                fscache_stat(&fscache_n_store_pages_over_limit);
 806                goto superseded;
 807        }
 808
 809        radix_tree_tag_set(&cookie->stores, page->index,
 810                           FSCACHE_COOKIE_STORING_TAG);
 811        radix_tree_tag_clear(&cookie->stores, page->index,
 812                             FSCACHE_COOKIE_PENDING_TAG);
 813
 814        spin_unlock(&cookie->stores_lock);
 815        spin_unlock(&object->lock);
 816
 817        fscache_stat(&fscache_n_store_pages);
 818        fscache_stat(&fscache_n_cop_write_page);
 819        ret = object->cache->ops->write_page(op, page);
 820        fscache_stat_d(&fscache_n_cop_write_page);
 821        fscache_end_page_write(object, page);
 822        if (ret < 0) {
 823                fscache_abort_object(object);
 824                fscache_op_complete(&op->op, true);
 825        } else {
 826                fscache_enqueue_operation(&op->op);
 827        }
 828
 829        _leave("");
 830        return;
 831
 832superseded:
 833        /* this writer is going away and there aren't any more things to
 834         * write */
 835        _debug("cease");
 836        spin_unlock(&cookie->stores_lock);
 837        clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 838        spin_unlock(&object->lock);
 839        fscache_op_complete(&op->op, true);
 840        _leave("");
 841}
 842
 843/*
 844 * Clear the pages pending writing for invalidation
 845 */
 846void fscache_invalidate_writes(struct fscache_cookie *cookie)
 847{
 848        struct page *page;
 849        void *results[16];
 850        int n, i;
 851
 852        _enter("");
 853
 854        for (;;) {
 855                spin_lock(&cookie->stores_lock);
 856                n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
 857                                               ARRAY_SIZE(results),
 858                                               FSCACHE_COOKIE_PENDING_TAG);
 859                if (n == 0) {
 860                        spin_unlock(&cookie->stores_lock);
 861                        break;
 862                }
 863
 864                for (i = n - 1; i >= 0; i--) {
 865                        page = results[i];
 866                        radix_tree_delete(&cookie->stores, page->index);
 867                }
 868
 869                spin_unlock(&cookie->stores_lock);
 870
 871                for (i = n - 1; i >= 0; i--)
 872                        page_cache_release(results[i]);
 873        }
 874
 875        _leave("");
 876}
 877
 878/*
 879 * request a page be stored in the cache
 880 * - returns:
 881 *   -ENOMEM    - out of memory, nothing done
 882 *   -ENOBUFS   - no backing object available in which to cache the page
 883 *   0          - dispatched a write - it'll call end_io_func() when finished
 884 *
 885 * if the cookie still has a backing object at this point, that object can be
 886 * in one of a few states with respect to storage processing:
 887 *
 888 *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
 889 *      set)
 890 *
 891 *      (a) no writes yet
 892 *
 893 *      (b) writes deferred till post-creation (mark page for writing and
 894 *          return immediately)
 895 *
 896 *  (2) negative lookup, object created, initial fill being made from netfs
 897 *
 898 *      (a) fill point not yet reached this page (mark page for writing and
 899 *          return)
 900 *
 901 *      (b) fill point passed this page (queue op to store this page)
 902 *
 903 *  (3) object extant (queue op to store this page)
 904 *
 905 * any other state is invalid
 906 */
 907int __fscache_write_page(struct fscache_cookie *cookie,
 908                         struct page *page,
 909                         gfp_t gfp)
 910{
 911        struct fscache_storage *op;
 912        struct fscache_object *object;
 913        bool wake_cookie = false;
 914        int ret;
 915
 916        _enter("%p,%x,", cookie, (u32) page->flags);
 917
 918        ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
 919        ASSERT(PageFsCache(page));
 920
 921        fscache_stat(&fscache_n_stores);
 922
 923        if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
 924                _leave(" = -ENOBUFS [invalidating]");
 925                return -ENOBUFS;
 926        }
 927
 928        op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 929        if (!op)
 930                goto nomem;
 931
 932        fscache_operation_init(&op->op, fscache_write_op,
 933                               fscache_release_write_op);
 934        op->op.flags = FSCACHE_OP_ASYNC |
 935                (1 << FSCACHE_OP_WAITING) |
 936                (1 << FSCACHE_OP_UNUSE_COOKIE);
 937
 938        ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM);
 939        if (ret < 0)
 940                goto nomem_free;
 941
 942        ret = -ENOBUFS;
 943        spin_lock(&cookie->lock);
 944
 945        if (!fscache_cookie_enabled(cookie) ||
 946            hlist_empty(&cookie->backing_objects))
 947                goto nobufs;
 948        object = hlist_entry(cookie->backing_objects.first,
 949                             struct fscache_object, cookie_link);
 950        if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 951                goto nobufs;
 952
 953        /* add the page to the pending-storage radix tree on the backing
 954         * object */
 955        spin_lock(&object->lock);
 956        spin_lock(&cookie->stores_lock);
 957
 958        _debug("store limit %llx", (unsigned long long) object->store_limit);
 959
 960        ret = radix_tree_insert(&cookie->stores, page->index, page);
 961        if (ret < 0) {
 962                if (ret == -EEXIST)
 963                        goto already_queued;
 964                _debug("insert failed %d", ret);
 965                goto nobufs_unlock_obj;
 966        }
 967
 968        radix_tree_tag_set(&cookie->stores, page->index,
 969                           FSCACHE_COOKIE_PENDING_TAG);
 970        page_cache_get(page);
 971
 972        /* we only want one writer at a time, but we do need to queue new
 973         * writers after exclusive ops */
 974        if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
 975                goto already_pending;
 976
 977        spin_unlock(&cookie->stores_lock);
 978        spin_unlock(&object->lock);
 979
 980        op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
 981        op->store_limit = object->store_limit;
 982
 983        __fscache_use_cookie(cookie);
 984        if (fscache_submit_op(object, &op->op) < 0)
 985                goto submit_failed;
 986
 987        spin_unlock(&cookie->lock);
 988        radix_tree_preload_end();
 989        fscache_stat(&fscache_n_store_ops);
 990        fscache_stat(&fscache_n_stores_ok);
 991
 992        /* the work queue now carries its own ref on the object */
 993        fscache_put_operation(&op->op);
 994        _leave(" = 0");
 995        return 0;
 996
 997already_queued:
 998        fscache_stat(&fscache_n_stores_again);
 999already_pending:
1000        spin_unlock(&cookie->stores_lock);
1001        spin_unlock(&object->lock);
1002        spin_unlock(&cookie->lock);
1003        radix_tree_preload_end();
1004        kfree(op);
1005        fscache_stat(&fscache_n_stores_ok);
1006        _leave(" = 0");
1007        return 0;
1008
1009submit_failed:
1010        spin_lock(&cookie->stores_lock);
1011        radix_tree_delete(&cookie->stores, page->index);
1012        spin_unlock(&cookie->stores_lock);
1013        wake_cookie = __fscache_unuse_cookie(cookie);
1014        page_cache_release(page);
1015        ret = -ENOBUFS;
1016        goto nobufs;
1017
1018nobufs_unlock_obj:
1019        spin_unlock(&cookie->stores_lock);
1020        spin_unlock(&object->lock);
1021nobufs:
1022        spin_unlock(&cookie->lock);
1023        radix_tree_preload_end();
1024        kfree(op);
1025        if (wake_cookie)
1026                __fscache_wake_unused_cookie(cookie);
1027        fscache_stat(&fscache_n_stores_nobufs);
1028        _leave(" = -ENOBUFS");
1029        return -ENOBUFS;
1030
1031nomem_free:
1032        kfree(op);
1033nomem:
1034        fscache_stat(&fscache_n_stores_oom);
1035        _leave(" = -ENOMEM");
1036        return -ENOMEM;
1037}
1038EXPORT_SYMBOL(__fscache_write_page);
1039
1040/*
1041 * remove a page from the cache
1042 */
1043void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
1044{
1045        struct fscache_object *object;
1046
1047        _enter(",%p", page);
1048
1049        ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
1050        ASSERTCMP(page, !=, NULL);
1051
1052        fscache_stat(&fscache_n_uncaches);
1053
1054        /* cache withdrawal may beat us to it */
1055        if (!PageFsCache(page))
1056                goto done;
1057
1058        /* get the object */
1059        spin_lock(&cookie->lock);
1060
1061        if (hlist_empty(&cookie->backing_objects)) {
1062                ClearPageFsCache(page);
1063                goto done_unlock;
1064        }
1065
1066        object = hlist_entry(cookie->backing_objects.first,
1067                             struct fscache_object, cookie_link);
1068
1069        /* there might now be stuff on disk we could read */
1070        clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
1071
1072        /* only invoke the cache backend if we managed to mark the page
1073         * uncached here; this deals with synchronisation vs withdrawal */
1074        if (TestClearPageFsCache(page) &&
1075            object->cache->ops->uncache_page) {
1076                /* the cache backend releases the cookie lock */
1077                fscache_stat(&fscache_n_cop_uncache_page);
1078                object->cache->ops->uncache_page(object, page);
1079                fscache_stat_d(&fscache_n_cop_uncache_page);
1080                goto done;
1081        }
1082
1083done_unlock:
1084        spin_unlock(&cookie->lock);
1085done:
1086        _leave("");
1087}
1088EXPORT_SYMBOL(__fscache_uncache_page);
1089
1090/**
1091 * fscache_mark_page_cached - Mark a page as being cached
1092 * @op: The retrieval op pages are being marked for
1093 * @page: The page to be marked
1094 *
1095 * Mark a netfs page as being cached.  After this is called, the netfs
1096 * must call fscache_uncache_page() to remove the mark.
1097 */
1098void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
1099{
1100        struct fscache_cookie *cookie = op->op.object->cookie;
1101
1102#ifdef CONFIG_FSCACHE_STATS
1103        atomic_inc(&fscache_n_marks);
1104#endif
1105
1106        _debug("- mark %p{%lx}", page, page->index);
1107        if (TestSetPageFsCache(page)) {
1108                static bool once_only;
1109                if (!once_only) {
1110                        once_only = true;
1111                        printk(KERN_WARNING "FS-Cache:"
1112                               " Cookie type %s marked page %lx"
1113                               " multiple times\n",
1114                               cookie->def->name, page->index);
1115                }
1116        }
1117
1118        if (cookie->def->mark_page_cached)
1119                cookie->def->mark_page_cached(cookie->netfs_data,
1120                                              op->mapping, page);
1121}
1122EXPORT_SYMBOL(fscache_mark_page_cached);
1123
1124/**
1125 * fscache_mark_pages_cached - Mark pages as being cached
1126 * @op: The retrieval op pages are being marked for
1127 * @pagevec: The pages to be marked
1128 *
1129 * Mark a bunch of netfs pages as being cached.  After this is called,
1130 * the netfs must call fscache_uncache_page() to remove the mark.
1131 */
1132void fscache_mark_pages_cached(struct fscache_retrieval *op,
1133                               struct pagevec *pagevec)
1134{
1135        unsigned long loop;
1136
1137        for (loop = 0; loop < pagevec->nr; loop++)
1138                fscache_mark_page_cached(op, pagevec->pages[loop]);
1139
1140        pagevec_reinit(pagevec);
1141}
1142EXPORT_SYMBOL(fscache_mark_pages_cached);
1143
1144/*
1145 * Uncache all the pages in an inode that are marked PG_fscache, assuming them
1146 * to be associated with the given cookie.
1147 */
1148void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
1149                                       struct inode *inode)
1150{
1151        struct address_space *mapping = inode->i_mapping;
1152        struct pagevec pvec;
1153        pgoff_t next;
1154        int i;
1155
1156        _enter("%p,%p", cookie, inode);
1157
1158        if (!mapping || mapping->nrpages == 0) {
1159                _leave(" [no pages]");
1160                return;
1161        }
1162
1163        pagevec_init(&pvec, 0);
1164        next = 0;
1165        do {
1166                if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE))
1167                        break;
1168                for (i = 0; i < pagevec_count(&pvec); i++) {
1169                        struct page *page = pvec.pages[i];
1170                        next = page->index;
1171                        if (PageFsCache(page)) {
1172                                __fscache_wait_on_page_write(cookie, page);
1173                                __fscache_uncache_page(cookie, page);
1174                        }
1175                }
1176                pagevec_release(&pvec);
1177                cond_resched();
1178        } while (++next);
1179
1180        _leave("");
1181}
1182EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);
1183