linux/fs/nfs/write.c
<<
>>
Prefs
   1/*
   2 * linux/fs/nfs/write.c
   3 *
   4 * Write file data over NFS.
   5 *
   6 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
   7 */
   8
   9#include <linux/types.h>
  10#include <linux/slab.h>
  11#include <linux/mm.h>
  12#include <linux/pagemap.h>
  13#include <linux/file.h>
  14#include <linux/writeback.h>
  15#include <linux/swap.h>
  16#include <linux/migrate.h>
  17
  18#include <linux/sunrpc/clnt.h>
  19#include <linux/nfs_fs.h>
  20#include <linux/nfs_mount.h>
  21#include <linux/nfs_page.h>
  22#include <linux/backing-dev.h>
  23#include <linux/export.h>
  24
  25#include <asm/uaccess.h>
  26
  27#include "delegation.h"
  28#include "internal.h"
  29#include "iostat.h"
  30#include "nfs4_fs.h"
  31#include "fscache.h"
  32#include "pnfs.h"
  33
  34#define NFSDBG_FACILITY         NFSDBG_PAGECACHE
  35
  36#define MIN_POOL_WRITE          (32)
  37#define MIN_POOL_COMMIT         (4)
  38
  39/*
  40 * Local function declarations
  41 */
  42static void nfs_redirty_request(struct nfs_page *req);
  43static const struct rpc_call_ops nfs_write_common_ops;
  44static const struct rpc_call_ops nfs_commit_ops;
  45static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
  46static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
  47
  48static struct kmem_cache *nfs_wdata_cachep;
  49static mempool_t *nfs_wdata_mempool;
  50static struct kmem_cache *nfs_cdata_cachep;
  51static mempool_t *nfs_commit_mempool;
  52
  53struct nfs_commit_data *nfs_commitdata_alloc(void)
  54{
  55        struct nfs_commit_data *p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
  56
  57        if (p) {
  58                memset(p, 0, sizeof(*p));
  59                INIT_LIST_HEAD(&p->pages);
  60        }
  61        return p;
  62}
  63EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
  64
  65void nfs_commit_free(struct nfs_commit_data *p)
  66{
  67        mempool_free(p, nfs_commit_mempool);
  68}
  69EXPORT_SYMBOL_GPL(nfs_commit_free);
  70
  71struct nfs_write_header *nfs_writehdr_alloc(void)
  72{
  73        struct nfs_write_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
  74
  75        if (p) {
  76                struct nfs_pgio_header *hdr = &p->header;
  77
  78                memset(p, 0, sizeof(*p));
  79                INIT_LIST_HEAD(&hdr->pages);
  80                INIT_LIST_HEAD(&hdr->rpc_list);
  81                spin_lock_init(&hdr->lock);
  82                atomic_set(&hdr->refcnt, 0);
  83                hdr->verf = &p->verf;
  84        }
  85        return p;
  86}
  87EXPORT_SYMBOL_GPL(nfs_writehdr_alloc);
  88
  89static struct nfs_write_data *nfs_writedata_alloc(struct nfs_pgio_header *hdr,
  90                                                  unsigned int pagecount)
  91{
  92        struct nfs_write_data *data, *prealloc;
  93
  94        prealloc = &container_of(hdr, struct nfs_write_header, header)->rpc_data;
  95        if (prealloc->header == NULL)
  96                data = prealloc;
  97        else
  98                data = kzalloc(sizeof(*data), GFP_KERNEL);
  99        if (!data)
 100                goto out;
 101
 102        if (nfs_pgarray_set(&data->pages, pagecount)) {
 103                data->header = hdr;
 104                atomic_inc(&hdr->refcnt);
 105        } else {
 106                if (data != prealloc)
 107                        kfree(data);
 108                data = NULL;
 109        }
 110out:
 111        return data;
 112}
 113
 114void nfs_writehdr_free(struct nfs_pgio_header *hdr)
 115{
 116        struct nfs_write_header *whdr = container_of(hdr, struct nfs_write_header, header);
 117        mempool_free(whdr, nfs_wdata_mempool);
 118}
 119EXPORT_SYMBOL_GPL(nfs_writehdr_free);
 120
 121void nfs_writedata_release(struct nfs_write_data *wdata)
 122{
 123        struct nfs_pgio_header *hdr = wdata->header;
 124        struct nfs_write_header *write_header = container_of(hdr, struct nfs_write_header, header);
 125
 126        put_nfs_open_context(wdata->args.context);
 127        if (wdata->pages.pagevec != wdata->pages.page_array)
 128                kfree(wdata->pages.pagevec);
 129        if (wdata == &write_header->rpc_data) {
 130                wdata->header = NULL;
 131                wdata = NULL;
 132        }
 133        if (atomic_dec_and_test(&hdr->refcnt))
 134                hdr->completion_ops->completion(hdr);
 135        /* Note: we only free the rpc_task after callbacks are done.
 136         * See the comment in rpc_free_task() for why
 137         */
 138        kfree(wdata);
 139}
 140EXPORT_SYMBOL_GPL(nfs_writedata_release);
 141
 142static void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
 143{
 144        ctx->error = error;
 145        smp_wmb();
 146        set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
 147}
 148
 149static struct nfs_page *
 150nfs_page_find_request_locked(struct nfs_inode *nfsi, struct page *page)
 151{
 152        struct nfs_page *req = NULL;
 153
 154        if (PagePrivate(page))
 155                req = (struct nfs_page *)page_private(page);
 156        else if (unlikely(PageSwapCache(page))) {
 157                struct nfs_page *freq, *t;
 158
 159                /* Linearly search the commit list for the correct req */
 160                list_for_each_entry_safe(freq, t, &nfsi->commit_info.list, wb_list) {
 161                        if (freq->wb_page == page) {
 162                                req = freq;
 163                                break;
 164                        }
 165                }
 166        }
 167
 168        if (req)
 169                kref_get(&req->wb_kref);
 170
 171        return req;
 172}
 173
 174static struct nfs_page *nfs_page_find_request(struct page *page)
 175{
 176        struct inode *inode = page_file_mapping(page)->host;
 177        struct nfs_page *req = NULL;
 178
 179        spin_lock(&inode->i_lock);
 180        req = nfs_page_find_request_locked(NFS_I(inode), page);
 181        spin_unlock(&inode->i_lock);
 182        return req;
 183}
 184
 185/* Adjust the file length if we're writing beyond the end */
 186static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
 187{
 188        struct inode *inode = page_file_mapping(page)->host;
 189        loff_t end, i_size;
 190        pgoff_t end_index;
 191
 192        spin_lock(&inode->i_lock);
 193        i_size = i_size_read(inode);
 194        end_index = (i_size - 1) >> PAGE_CACHE_SHIFT;
 195        if (i_size > 0 && page_file_index(page) < end_index)
 196                goto out;
 197        end = page_file_offset(page) + ((loff_t)offset+count);
 198        if (i_size >= end)
 199                goto out;
 200        i_size_write(inode, end);
 201        nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
 202out:
 203        spin_unlock(&inode->i_lock);
 204}
 205
 206/* A writeback failed: mark the page as bad, and invalidate the page cache */
 207static void nfs_set_pageerror(struct page *page)
 208{
 209        nfs_zap_mapping(page_file_mapping(page)->host, page_file_mapping(page));
 210}
 211
 212/* We can set the PG_uptodate flag if we see that a write request
 213 * covers the full page.
 214 */
 215static void nfs_mark_uptodate(struct page *page, unsigned int base, unsigned int count)
 216{
 217        if (PageUptodate(page))
 218                return;
 219        if (base != 0)
 220                return;
 221        if (count != nfs_page_length(page))
 222                return;
 223        SetPageUptodate(page);
 224}
 225
 226static int wb_priority(struct writeback_control *wbc)
 227{
 228        if (wbc->for_reclaim)
 229                return FLUSH_HIGHPRI | FLUSH_STABLE;
 230        if (wbc->for_kupdate || wbc->for_background)
 231                return FLUSH_LOWPRI | FLUSH_COND_STABLE;
 232        return FLUSH_COND_STABLE;
 233}
 234
 235/*
 236 * NFS congestion control
 237 */
 238
 239int nfs_congestion_kb;
 240
 241#define NFS_CONGESTION_ON_THRESH        (nfs_congestion_kb >> (PAGE_SHIFT-10))
 242#define NFS_CONGESTION_OFF_THRESH       \
 243        (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
 244
 245static int nfs_set_page_writeback(struct page *page)
 246{
 247        int ret = test_set_page_writeback(page);
 248
 249        if (!ret) {
 250                struct inode *inode = page_file_mapping(page)->host;
 251                struct nfs_server *nfss = NFS_SERVER(inode);
 252
 253                if (atomic_long_inc_return(&nfss->writeback) >
 254                                NFS_CONGESTION_ON_THRESH) {
 255                        set_bdi_congested(&nfss->backing_dev_info,
 256                                                BLK_RW_ASYNC);
 257                }
 258        }
 259        return ret;
 260}
 261
 262static void nfs_end_page_writeback(struct page *page)
 263{
 264        struct inode *inode = page_file_mapping(page)->host;
 265        struct nfs_server *nfss = NFS_SERVER(inode);
 266
 267        end_page_writeback(page);
 268        if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
 269                clear_bdi_congested(&nfss->backing_dev_info, BLK_RW_ASYNC);
 270}
 271
 272static struct nfs_page *nfs_find_and_lock_request(struct page *page, bool nonblock)
 273{
 274        struct inode *inode = page_file_mapping(page)->host;
 275        struct nfs_page *req;
 276        int ret;
 277
 278        spin_lock(&inode->i_lock);
 279        for (;;) {
 280                req = nfs_page_find_request_locked(NFS_I(inode), page);
 281                if (req == NULL)
 282                        break;
 283                if (nfs_lock_request(req))
 284                        break;
 285                /* Note: If we hold the page lock, as is the case in nfs_writepage,
 286                 *       then the call to nfs_lock_request() will always
 287                 *       succeed provided that someone hasn't already marked the
 288                 *       request as dirty (in which case we don't care).
 289                 */
 290                spin_unlock(&inode->i_lock);
 291                if (!nonblock)
 292                        ret = nfs_wait_on_request(req);
 293                else
 294                        ret = -EAGAIN;
 295                nfs_release_request(req);
 296                if (ret != 0)
 297                        return ERR_PTR(ret);
 298                spin_lock(&inode->i_lock);
 299        }
 300        spin_unlock(&inode->i_lock);
 301        return req;
 302}
 303
 304/*
 305 * Find an associated nfs write request, and prepare to flush it out
 306 * May return an error if the user signalled nfs_wait_on_request().
 307 */
 308static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
 309                                struct page *page, bool nonblock)
 310{
 311        struct nfs_page *req;
 312        int ret = 0;
 313
 314        req = nfs_find_and_lock_request(page, nonblock);
 315        if (!req)
 316                goto out;
 317        ret = PTR_ERR(req);
 318        if (IS_ERR(req))
 319                goto out;
 320
 321        ret = nfs_set_page_writeback(page);
 322        BUG_ON(ret != 0);
 323        BUG_ON(test_bit(PG_CLEAN, &req->wb_flags));
 324
 325        if (!nfs_pageio_add_request(pgio, req)) {
 326                nfs_redirty_request(req);
 327                ret = pgio->pg_error;
 328        }
 329out:
 330        return ret;
 331}
 332
 333static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
 334{
 335        struct inode *inode = page_file_mapping(page)->host;
 336        int ret;
 337
 338        nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
 339        nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
 340
 341        nfs_pageio_cond_complete(pgio, page_file_index(page));
 342        ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE);
 343        if (ret == -EAGAIN) {
 344                redirty_page_for_writepage(wbc, page);
 345                ret = 0;
 346        }
 347        return ret;
 348}
 349
 350/*
 351 * Write an mmapped page to the server.
 352 */
 353static int nfs_writepage_locked(struct page *page, struct writeback_control *wbc)
 354{
 355        struct nfs_pageio_descriptor pgio;
 356        int err;
 357
 358        NFS_PROTO(page_file_mapping(page)->host)->write_pageio_init(&pgio,
 359                                                          page->mapping->host,
 360                                                          wb_priority(wbc),
 361                                                          &nfs_async_write_completion_ops);
 362        err = nfs_do_writepage(page, wbc, &pgio);
 363        nfs_pageio_complete(&pgio);
 364        if (err < 0)
 365                return err;
 366        if (pgio.pg_error < 0)
 367                return pgio.pg_error;
 368        return 0;
 369}
 370
 371int nfs_writepage(struct page *page, struct writeback_control *wbc)
 372{
 373        int ret;
 374
 375        ret = nfs_writepage_locked(page, wbc);
 376        unlock_page(page);
 377        return ret;
 378}
 379
 380static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
 381{
 382        int ret;
 383
 384        ret = nfs_do_writepage(page, wbc, data);
 385        unlock_page(page);
 386        return ret;
 387}
 388
 389int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
 390{
 391        struct inode *inode = mapping->host;
 392        unsigned long *bitlock = &NFS_I(inode)->flags;
 393        struct nfs_pageio_descriptor pgio;
 394        int err;
 395
 396        /* Stop dirtying of new pages while we sync */
 397        err = wait_on_bit_lock(bitlock, NFS_INO_FLUSHING,
 398                        nfs_wait_bit_killable, TASK_KILLABLE);
 399        if (err)
 400                goto out_err;
 401
 402        nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
 403
 404        NFS_PROTO(inode)->write_pageio_init(&pgio, inode, wb_priority(wbc), &nfs_async_write_completion_ops);
 405        err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
 406        nfs_pageio_complete(&pgio);
 407
 408        clear_bit_unlock(NFS_INO_FLUSHING, bitlock);
 409        smp_mb__after_clear_bit();
 410        wake_up_bit(bitlock, NFS_INO_FLUSHING);
 411
 412        if (err < 0)
 413                goto out_err;
 414        err = pgio.pg_error;
 415        if (err < 0)
 416                goto out_err;
 417        return 0;
 418out_err:
 419        return err;
 420}
 421
 422/*
 423 * Insert a write request into an inode
 424 */
 425static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
 426{
 427        struct nfs_inode *nfsi = NFS_I(inode);
 428
 429        /* Lock the request! */
 430        nfs_lock_request(req);
 431
 432        spin_lock(&inode->i_lock);
 433        if (!nfsi->npages && NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
 434                inode->i_version++;
 435        /*
 436         * Swap-space should not get truncated. Hence no need to plug the race
 437         * with invalidate/truncate.
 438         */
 439        if (likely(!PageSwapCache(req->wb_page))) {
 440                set_bit(PG_MAPPED, &req->wb_flags);
 441                SetPagePrivate(req->wb_page);
 442                set_page_private(req->wb_page, (unsigned long)req);
 443        }
 444        nfsi->npages++;
 445        kref_get(&req->wb_kref);
 446        spin_unlock(&inode->i_lock);
 447}
 448
 449/*
 450 * Remove a write request from an inode
 451 */
 452static void nfs_inode_remove_request(struct nfs_page *req)
 453{
 454        struct inode *inode = req->wb_context->dentry->d_inode;
 455        struct nfs_inode *nfsi = NFS_I(inode);
 456
 457        BUG_ON (!NFS_WBACK_BUSY(req));
 458
 459        spin_lock(&inode->i_lock);
 460        if (likely(!PageSwapCache(req->wb_page))) {
 461                set_page_private(req->wb_page, 0);
 462                ClearPagePrivate(req->wb_page);
 463                clear_bit(PG_MAPPED, &req->wb_flags);
 464        }
 465        nfsi->npages--;
 466        spin_unlock(&inode->i_lock);
 467        nfs_release_request(req);
 468}
 469
 470static void
 471nfs_mark_request_dirty(struct nfs_page *req)
 472{
 473        __set_page_dirty_nobuffers(req->wb_page);
 474}
 475
 476#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
 477/**
 478 * nfs_request_add_commit_list - add request to a commit list
 479 * @req: pointer to a struct nfs_page
 480 * @dst: commit list head
 481 * @cinfo: holds list lock and accounting info
 482 *
 483 * This sets the PG_CLEAN bit, updates the cinfo count of
 484 * number of outstanding requests requiring a commit as well as
 485 * the MM page stats.
 486 *
 487 * The caller must _not_ hold the cinfo->lock, but must be
 488 * holding the nfs_page lock.
 489 */
 490void
 491nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst,
 492                            struct nfs_commit_info *cinfo)
 493{
 494        set_bit(PG_CLEAN, &(req)->wb_flags);
 495        spin_lock(cinfo->lock);
 496        nfs_list_add_request(req, dst);
 497        cinfo->mds->ncommit++;
 498        spin_unlock(cinfo->lock);
 499        if (!cinfo->dreq) {
 500                inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
 501                inc_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
 502                             BDI_RECLAIMABLE);
 503                __mark_inode_dirty(req->wb_context->dentry->d_inode,
 504                                   I_DIRTY_DATASYNC);
 505        }
 506}
 507EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);
 508
 509/**
 510 * nfs_request_remove_commit_list - Remove request from a commit list
 511 * @req: pointer to a nfs_page
 512 * @cinfo: holds list lock and accounting info
 513 *
 514 * This clears the PG_CLEAN bit, and updates the cinfo's count of
 515 * number of outstanding requests requiring a commit
 516 * It does not update the MM page stats.
 517 *
 518 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
 519 */
 520void
 521nfs_request_remove_commit_list(struct nfs_page *req,
 522                               struct nfs_commit_info *cinfo)
 523{
 524        if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
 525                return;
 526        nfs_list_remove_request(req);
 527        cinfo->mds->ncommit--;
 528}
 529EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);
 530
 531static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
 532                                      struct inode *inode)
 533{
 534        cinfo->lock = &inode->i_lock;
 535        cinfo->mds = &NFS_I(inode)->commit_info;
 536        cinfo->ds = pnfs_get_ds_info(inode);
 537        cinfo->dreq = NULL;
 538        cinfo->completion_ops = &nfs_commit_completion_ops;
 539}
 540
 541void nfs_init_cinfo(struct nfs_commit_info *cinfo,
 542                    struct inode *inode,
 543                    struct nfs_direct_req *dreq)
 544{
 545        if (dreq)
 546                nfs_init_cinfo_from_dreq(cinfo, dreq);
 547        else
 548                nfs_init_cinfo_from_inode(cinfo, inode);
 549}
 550EXPORT_SYMBOL_GPL(nfs_init_cinfo);
 551
 552/*
 553 * Add a request to the inode's commit list.
 554 */
 555void
 556nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
 557                        struct nfs_commit_info *cinfo)
 558{
 559        if (pnfs_mark_request_commit(req, lseg, cinfo))
 560                return;
 561        nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo);
 562}
 563
 564static void
 565nfs_clear_page_commit(struct page *page)
 566{
 567        dec_zone_page_state(page, NR_UNSTABLE_NFS);
 568        dec_bdi_stat(page_file_mapping(page)->backing_dev_info, BDI_RECLAIMABLE);
 569}
 570
 571static void
 572nfs_clear_request_commit(struct nfs_page *req)
 573{
 574        if (test_bit(PG_CLEAN, &req->wb_flags)) {
 575                struct inode *inode = req->wb_context->dentry->d_inode;
 576                struct nfs_commit_info cinfo;
 577
 578                nfs_init_cinfo_from_inode(&cinfo, inode);
 579                if (!pnfs_clear_request_commit(req, &cinfo)) {
 580                        spin_lock(cinfo.lock);
 581                        nfs_request_remove_commit_list(req, &cinfo);
 582                        spin_unlock(cinfo.lock);
 583                }
 584                nfs_clear_page_commit(req->wb_page);
 585        }
 586}
 587
 588static inline
 589int nfs_write_need_commit(struct nfs_write_data *data)
 590{
 591        if (data->verf.committed == NFS_DATA_SYNC)
 592                return data->header->lseg == NULL;
 593        return data->verf.committed != NFS_FILE_SYNC;
 594}
 595
 596#else
 597static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
 598                                      struct inode *inode)
 599{
 600}
 601
 602void nfs_init_cinfo(struct nfs_commit_info *cinfo,
 603                    struct inode *inode,
 604                    struct nfs_direct_req *dreq)
 605{
 606}
 607
 608void
 609nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
 610                        struct nfs_commit_info *cinfo)
 611{
 612}
 613
 614static void
 615nfs_clear_request_commit(struct nfs_page *req)
 616{
 617}
 618
 619static inline
 620int nfs_write_need_commit(struct nfs_write_data *data)
 621{
 622        return 0;
 623}
 624
 625#endif
 626
 627static void nfs_write_completion(struct nfs_pgio_header *hdr)
 628{
 629        struct nfs_commit_info cinfo;
 630        unsigned long bytes = 0;
 631
 632        if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
 633                goto out;
 634        nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
 635        while (!list_empty(&hdr->pages)) {
 636                struct nfs_page *req = nfs_list_entry(hdr->pages.next);
 637
 638                bytes += req->wb_bytes;
 639                nfs_list_remove_request(req);
 640                if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
 641                    (hdr->good_bytes < bytes)) {
 642                        nfs_set_pageerror(req->wb_page);
 643                        nfs_context_set_write_error(req->wb_context, hdr->error);
 644                        goto remove_req;
 645                }
 646                if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) {
 647                        nfs_mark_request_dirty(req);
 648                        goto next;
 649                }
 650                if (test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) {
 651                        memcpy(&req->wb_verf, &hdr->verf->verifier, sizeof(req->wb_verf));
 652                        nfs_mark_request_commit(req, hdr->lseg, &cinfo);
 653                        goto next;
 654                }
 655remove_req:
 656                nfs_inode_remove_request(req);
 657next:
 658                nfs_unlock_request(req);
 659                nfs_end_page_writeback(req->wb_page);
 660                nfs_release_request(req);
 661        }
 662out:
 663        hdr->release(hdr);
 664}
 665
 666#if  IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
 667static unsigned long
 668nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
 669{
 670        return cinfo->mds->ncommit;
 671}
 672
 673/* cinfo->lock held by caller */
 674int
 675nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
 676                     struct nfs_commit_info *cinfo, int max)
 677{
 678        struct nfs_page *req, *tmp;
 679        int ret = 0;
 680
 681        list_for_each_entry_safe(req, tmp, src, wb_list) {
 682                if (!nfs_lock_request(req))
 683                        continue;
 684                kref_get(&req->wb_kref);
 685                if (cond_resched_lock(cinfo->lock))
 686                        list_safe_reset_next(req, tmp, wb_list);
 687                nfs_request_remove_commit_list(req, cinfo);
 688                nfs_list_add_request(req, dst);
 689                ret++;
 690                if ((ret == max) && !cinfo->dreq)
 691                        break;
 692        }
 693        return ret;
 694}
 695
 696/*
 697 * nfs_scan_commit - Scan an inode for commit requests
 698 * @inode: NFS inode to scan
 699 * @dst: mds destination list
 700 * @cinfo: mds and ds lists of reqs ready to commit
 701 *
 702 * Moves requests from the inode's 'commit' request list.
 703 * The requests are *not* checked to ensure that they form a contiguous set.
 704 */
 705int
 706nfs_scan_commit(struct inode *inode, struct list_head *dst,
 707                struct nfs_commit_info *cinfo)
 708{
 709        int ret = 0;
 710
 711        spin_lock(cinfo->lock);
 712        if (cinfo->mds->ncommit > 0) {
 713                const int max = INT_MAX;
 714
 715                ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
 716                                           cinfo, max);
 717                ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
 718        }
 719        spin_unlock(cinfo->lock);
 720        return ret;
 721}
 722
 723#else
 724static unsigned long nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
 725{
 726        return 0;
 727}
 728
 729int nfs_scan_commit(struct inode *inode, struct list_head *dst,
 730                    struct nfs_commit_info *cinfo)
 731{
 732        return 0;
 733}
 734#endif
 735
 736/*
 737 * Search for an existing write request, and attempt to update
 738 * it to reflect a new dirty region on a given page.
 739 *
 740 * If the attempt fails, then the existing request is flushed out
 741 * to disk.
 742 */
 743static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
 744                struct page *page,
 745                unsigned int offset,
 746                unsigned int bytes)
 747{
 748        struct nfs_page *req;
 749        unsigned int rqend;
 750        unsigned int end;
 751        int error;
 752
 753        if (!PagePrivate(page))
 754                return NULL;
 755
 756        end = offset + bytes;
 757        spin_lock(&inode->i_lock);
 758
 759        for (;;) {
 760                req = nfs_page_find_request_locked(NFS_I(inode), page);
 761                if (req == NULL)
 762                        goto out_unlock;
 763
 764                rqend = req->wb_offset + req->wb_bytes;
 765                /*
 766                 * Tell the caller to flush out the request if
 767                 * the offsets are non-contiguous.
 768                 * Note: nfs_flush_incompatible() will already
 769                 * have flushed out requests having wrong owners.
 770                 */
 771                if (offset > rqend
 772                    || end < req->wb_offset)
 773                        goto out_flushme;
 774
 775                if (nfs_lock_request(req))
 776                        break;
 777
 778                /* The request is locked, so wait and then retry */
 779                spin_unlock(&inode->i_lock);
 780                error = nfs_wait_on_request(req);
 781                nfs_release_request(req);
 782                if (error != 0)
 783                        goto out_err;
 784                spin_lock(&inode->i_lock);
 785        }
 786
 787        /* Okay, the request matches. Update the region */
 788        if (offset < req->wb_offset) {
 789                req->wb_offset = offset;
 790                req->wb_pgbase = offset;
 791        }
 792        if (end > rqend)
 793                req->wb_bytes = end - req->wb_offset;
 794        else
 795                req->wb_bytes = rqend - req->wb_offset;
 796out_unlock:
 797        spin_unlock(&inode->i_lock);
 798        if (req)
 799                nfs_clear_request_commit(req);
 800        return req;
 801out_flushme:
 802        spin_unlock(&inode->i_lock);
 803        nfs_release_request(req);
 804        error = nfs_wb_page(inode, page);
 805out_err:
 806        return ERR_PTR(error);
 807}
 808
 809/*
 810 * Try to update an existing write request, or create one if there is none.
 811 *
 812 * Note: Should always be called with the Page Lock held to prevent races
 813 * if we have to add a new request. Also assumes that the caller has
 814 * already called nfs_flush_incompatible() if necessary.
 815 */
 816static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
 817                struct page *page, unsigned int offset, unsigned int bytes)
 818{
 819        struct inode *inode = page_file_mapping(page)->host;
 820        struct nfs_page *req;
 821
 822        req = nfs_try_to_update_request(inode, page, offset, bytes);
 823        if (req != NULL)
 824                goto out;
 825        req = nfs_create_request(ctx, inode, page, offset, bytes);
 826        if (IS_ERR(req))
 827                goto out;
 828        nfs_inode_add_request(inode, req);
 829out:
 830        return req;
 831}
 832
 833static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
 834                unsigned int offset, unsigned int count)
 835{
 836        struct nfs_page *req;
 837
 838        req = nfs_setup_write_request(ctx, page, offset, count);
 839        if (IS_ERR(req))
 840                return PTR_ERR(req);
 841        /* Update file length */
 842        nfs_grow_file(page, offset, count);
 843        nfs_mark_uptodate(page, req->wb_pgbase, req->wb_bytes);
 844        nfs_mark_request_dirty(req);
 845        nfs_unlock_and_release_request(req);
 846        return 0;
 847}
 848
 849int nfs_flush_incompatible(struct file *file, struct page *page)
 850{
 851        struct nfs_open_context *ctx = nfs_file_open_context(file);
 852        struct nfs_lock_context *l_ctx;
 853        struct nfs_page *req;
 854        int do_flush, status;
 855        /*
 856         * Look for a request corresponding to this page. If there
 857         * is one, and it belongs to another file, we flush it out
 858         * before we try to copy anything into the page. Do this
 859         * due to the lack of an ACCESS-type call in NFSv2.
 860         * Also do the same if we find a request from an existing
 861         * dropped page.
 862         */
 863        do {
 864                req = nfs_page_find_request(page);
 865                if (req == NULL)
 866                        return 0;
 867                l_ctx = req->wb_lock_context;
 868                do_flush = req->wb_page != page || req->wb_context != ctx;
 869                if (l_ctx) {
 870                        do_flush |= l_ctx->lockowner.l_owner != current->files
 871                                || l_ctx->lockowner.l_pid != current->tgid;
 872                }
 873                nfs_release_request(req);
 874                if (!do_flush)
 875                        return 0;
 876                status = nfs_wb_page(page_file_mapping(page)->host, page);
 877        } while (status == 0);
 878        return status;
 879}
 880
 881/*
 882 * If the page cache is marked as unsafe or invalid, then we can't rely on
 883 * the PageUptodate() flag. In this case, we will need to turn off
 884 * write optimisations that depend on the page contents being correct.
 885 */
 886static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
 887{
 888        if (nfs_have_delegated_attributes(inode))
 889                goto out;
 890        if (NFS_I(inode)->cache_validity & (NFS_INO_INVALID_DATA|NFS_INO_REVAL_PAGECACHE))
 891                return false;
 892out:
 893        return PageUptodate(page) != 0;
 894}
 895
 896/*
 897 * Update and possibly write a cached page of an NFS file.
 898 *
 899 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
 900 * things with a page scheduled for an RPC call (e.g. invalidate it).
 901 */
 902int nfs_updatepage(struct file *file, struct page *page,
 903                unsigned int offset, unsigned int count)
 904{
 905        struct nfs_open_context *ctx = nfs_file_open_context(file);
 906        struct inode    *inode = page_file_mapping(page)->host;
 907        int             status = 0;
 908
 909        nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
 910
 911        dprintk("NFS:       nfs_updatepage(%s/%s %d@%lld)\n",
 912                file->f_path.dentry->d_parent->d_name.name,
 913                file->f_path.dentry->d_name.name, count,
 914                (long long)(page_file_offset(page) + offset));
 915
 916        /* If we're not using byte range locks, and we know the page
 917         * is up to date, it may be more efficient to extend the write
 918         * to cover the entire page in order to avoid fragmentation
 919         * inefficiencies.
 920         */
 921        if (nfs_write_pageuptodate(page, inode) &&
 922                        inode->i_flock == NULL &&
 923                        !(file->f_flags & O_DSYNC)) {
 924                count = max(count + offset, nfs_page_length(page));
 925                offset = 0;
 926        }
 927
 928        status = nfs_writepage_setup(ctx, page, offset, count);
 929        if (status < 0)
 930                nfs_set_pageerror(page);
 931        else
 932                __set_page_dirty_nobuffers(page);
 933
 934        dprintk("NFS:       nfs_updatepage returns %d (isize %lld)\n",
 935                        status, (long long)i_size_read(inode));
 936        return status;
 937}
 938
 939static int flush_task_priority(int how)
 940{
 941        switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
 942                case FLUSH_HIGHPRI:
 943                        return RPC_PRIORITY_HIGH;
 944                case FLUSH_LOWPRI:
 945                        return RPC_PRIORITY_LOW;
 946        }
 947        return RPC_PRIORITY_NORMAL;
 948}
 949
 950int nfs_initiate_write(struct rpc_clnt *clnt,
 951                       struct nfs_write_data *data,
 952                       const struct rpc_call_ops *call_ops,
 953                       int how, int flags)
 954{
 955        struct inode *inode = data->header->inode;
 956        int priority = flush_task_priority(how);
 957        struct rpc_task *task;
 958        struct rpc_message msg = {
 959                .rpc_argp = &data->args,
 960                .rpc_resp = &data->res,
 961                .rpc_cred = data->header->cred,
 962        };
 963        struct rpc_task_setup task_setup_data = {
 964                .rpc_client = clnt,
 965                .task = &data->task,
 966                .rpc_message = &msg,
 967                .callback_ops = call_ops,
 968                .callback_data = data,
 969                .workqueue = nfsiod_workqueue,
 970                .flags = RPC_TASK_ASYNC | flags,
 971                .priority = priority,
 972        };
 973        int ret = 0;
 974
 975        /* Set up the initial task struct.  */
 976        NFS_PROTO(inode)->write_setup(data, &msg);
 977
 978        dprintk("NFS: %5u initiated write call "
 979                "(req %s/%lld, %u bytes @ offset %llu)\n",
 980                data->task.tk_pid,
 981                inode->i_sb->s_id,
 982                (long long)NFS_FILEID(inode),
 983                data->args.count,
 984                (unsigned long long)data->args.offset);
 985
 986        task = rpc_run_task(&task_setup_data);
 987        if (IS_ERR(task)) {
 988                ret = PTR_ERR(task);
 989                goto out;
 990        }
 991        if (how & FLUSH_SYNC) {
 992                ret = rpc_wait_for_completion_task(task);
 993                if (ret == 0)
 994                        ret = task->tk_status;
 995        }
 996        rpc_put_task(task);
 997out:
 998        return ret;
 999}
1000EXPORT_SYMBOL_GPL(nfs_initiate_write);
1001
1002/*
1003 * Set up the argument/result storage required for the RPC call.
1004 */
1005static void nfs_write_rpcsetup(struct nfs_write_data *data,
1006                unsigned int count, unsigned int offset,
1007                int how, struct nfs_commit_info *cinfo)
1008{
1009        struct nfs_page *req = data->header->req;
1010
1011        /* Set up the RPC argument and reply structs
1012         * NB: take care not to mess about with data->commit et al. */
1013
1014        data->args.fh     = NFS_FH(data->header->inode);
1015        data->args.offset = req_offset(req) + offset;
1016        /* pnfs_set_layoutcommit needs this */
1017        data->mds_offset = data->args.offset;
1018        data->args.pgbase = req->wb_pgbase + offset;
1019        data->args.pages  = data->pages.pagevec;
1020        data->args.count  = count;
1021        data->args.context = get_nfs_open_context(req->wb_context);
1022        data->args.lock_context = req->wb_lock_context;
1023        data->args.stable  = NFS_UNSTABLE;
1024        switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
1025        case 0:
1026                break;
1027        case FLUSH_COND_STABLE:
1028                if (nfs_reqs_to_commit(cinfo))
1029                        break;
1030        default:
1031                data->args.stable = NFS_FILE_SYNC;
1032        }
1033
1034        data->res.fattr   = &data->fattr;
1035        data->res.count   = count;
1036        data->res.verf    = &data->verf;
1037        nfs_fattr_init(&data->fattr);
1038}
1039
1040static int nfs_do_write(struct nfs_write_data *data,
1041                const struct rpc_call_ops *call_ops,
1042                int how)
1043{
1044        struct inode *inode = data->header->inode;
1045
1046        return nfs_initiate_write(NFS_CLIENT(inode), data, call_ops, how, 0);
1047}
1048
1049static int nfs_do_multiple_writes(struct list_head *head,
1050                const struct rpc_call_ops *call_ops,
1051                int how)
1052{
1053        struct nfs_write_data *data;
1054        int ret = 0;
1055
1056        while (!list_empty(head)) {
1057                int ret2;
1058
1059                data = list_first_entry(head, struct nfs_write_data, list);
1060                list_del_init(&data->list);
1061                
1062                ret2 = nfs_do_write(data, call_ops, how);
1063                 if (ret == 0)
1064                         ret = ret2;
1065        }
1066        return ret;
1067}
1068
1069/* If a nfs_flush_* function fails, it should remove reqs from @head and
1070 * call this on each, which will prepare them to be retried on next
1071 * writeback using standard nfs.
1072 */
1073static void nfs_redirty_request(struct nfs_page *req)
1074{
1075        nfs_mark_request_dirty(req);
1076        nfs_unlock_request(req);
1077        nfs_end_page_writeback(req->wb_page);
1078        nfs_release_request(req);
1079}
1080
1081static void nfs_async_write_error(struct list_head *head)
1082{
1083        struct nfs_page *req;
1084
1085        while (!list_empty(head)) {
1086                req = nfs_list_entry(head->next);
1087                nfs_list_remove_request(req);
1088                nfs_redirty_request(req);
1089        }
1090}
1091
1092static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1093        .error_cleanup = nfs_async_write_error,
1094        .completion = nfs_write_completion,
1095};
1096
1097static void nfs_flush_error(struct nfs_pageio_descriptor *desc,
1098                struct nfs_pgio_header *hdr)
1099{
1100        set_bit(NFS_IOHDR_REDO, &hdr->flags);
1101        while (!list_empty(&hdr->rpc_list)) {
1102                struct nfs_write_data *data = list_first_entry(&hdr->rpc_list,
1103                                struct nfs_write_data, list);
1104                list_del(&data->list);
1105                nfs_writedata_release(data);
1106        }
1107        desc->pg_completion_ops->error_cleanup(&desc->pg_list);
1108}
1109
1110/*
1111 * Generate multiple small requests to write out a single
1112 * contiguous dirty area on one page.
1113 */
1114static int nfs_flush_multi(struct nfs_pageio_descriptor *desc,
1115                           struct nfs_pgio_header *hdr)
1116{
1117        struct nfs_page *req = hdr->req;
1118        struct page *page = req->wb_page;
1119        struct nfs_write_data *data;
1120        size_t wsize = desc->pg_bsize, nbytes;
1121        unsigned int offset;
1122        int requests = 0;
1123        struct nfs_commit_info cinfo;
1124
1125        nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);
1126
1127        if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
1128            (desc->pg_moreio || nfs_reqs_to_commit(&cinfo) ||
1129             desc->pg_count > wsize))
1130                desc->pg_ioflags &= ~FLUSH_COND_STABLE;
1131
1132
1133        offset = 0;
1134        nbytes = desc->pg_count;
1135        do {
1136                size_t len = min(nbytes, wsize);
1137
1138                data = nfs_writedata_alloc(hdr, 1);
1139                if (!data) {
1140                        nfs_flush_error(desc, hdr);
1141                        return -ENOMEM;
1142                }
1143                data->pages.pagevec[0] = page;
1144                nfs_write_rpcsetup(data, len, offset, desc->pg_ioflags, &cinfo);
1145                list_add(&data->list, &hdr->rpc_list);
1146                requests++;
1147                nbytes -= len;
1148                offset += len;
1149        } while (nbytes != 0);
1150        nfs_list_remove_request(req);
1151        nfs_list_add_request(req, &hdr->pages);
1152        desc->pg_rpc_callops = &nfs_write_common_ops;
1153        return 0;
1154}
1155
1156/*
1157 * Create an RPC task for the given write request and kick it.
1158 * The page must have been locked by the caller.
1159 *
1160 * It may happen that the page we're passed is not marked dirty.
1161 * This is the case if nfs_updatepage detects a conflicting request
1162 * that has been written but not committed.
1163 */
1164static int nfs_flush_one(struct nfs_pageio_descriptor *desc,
1165                         struct nfs_pgio_header *hdr)
1166{
1167        struct nfs_page         *req;
1168        struct page             **pages;
1169        struct nfs_write_data   *data;
1170        struct list_head *head = &desc->pg_list;
1171        struct nfs_commit_info cinfo;
1172
1173        data = nfs_writedata_alloc(hdr, nfs_page_array_len(desc->pg_base,
1174                                                           desc->pg_count));
1175        if (!data) {
1176                nfs_flush_error(desc, hdr);
1177                return -ENOMEM;
1178        }
1179
1180        nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);
1181        pages = data->pages.pagevec;
1182        while (!list_empty(head)) {
1183                req = nfs_list_entry(head->next);
1184                nfs_list_remove_request(req);
1185                nfs_list_add_request(req, &hdr->pages);
1186                *pages++ = req->wb_page;
1187        }
1188
1189        if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
1190            (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))
1191                desc->pg_ioflags &= ~FLUSH_COND_STABLE;
1192
1193        /* Set up the argument struct */
1194        nfs_write_rpcsetup(data, desc->pg_count, 0, desc->pg_ioflags, &cinfo);
1195        list_add(&data->list, &hdr->rpc_list);
1196        desc->pg_rpc_callops = &nfs_write_common_ops;
1197        return 0;
1198}
1199
1200int nfs_generic_flush(struct nfs_pageio_descriptor *desc,
1201                      struct nfs_pgio_header *hdr)
1202{
1203        if (desc->pg_bsize < PAGE_CACHE_SIZE)
1204                return nfs_flush_multi(desc, hdr);
1205        return nfs_flush_one(desc, hdr);
1206}
1207EXPORT_SYMBOL_GPL(nfs_generic_flush);
1208
1209static int nfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
1210{
1211        struct nfs_write_header *whdr;
1212        struct nfs_pgio_header *hdr;
1213        int ret;
1214
1215        whdr = nfs_writehdr_alloc();
1216        if (!whdr) {
1217                desc->pg_completion_ops->error_cleanup(&desc->pg_list);
1218                return -ENOMEM;
1219        }
1220        hdr = &whdr->header;
1221        nfs_pgheader_init(desc, hdr, nfs_writehdr_free);
1222        atomic_inc(&hdr->refcnt);
1223        ret = nfs_generic_flush(desc, hdr);
1224        if (ret == 0)
1225                ret = nfs_do_multiple_writes(&hdr->rpc_list,
1226                                             desc->pg_rpc_callops,
1227                                             desc->pg_ioflags);
1228        if (atomic_dec_and_test(&hdr->refcnt))
1229                hdr->completion_ops->completion(hdr);
1230        return ret;
1231}
1232
1233static const struct nfs_pageio_ops nfs_pageio_write_ops = {
1234        .pg_test = nfs_generic_pg_test,
1235        .pg_doio = nfs_generic_pg_writepages,
1236};
1237
1238void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1239                               struct inode *inode, int ioflags,
1240                               const struct nfs_pgio_completion_ops *compl_ops)
1241{
1242        nfs_pageio_init(pgio, inode, &nfs_pageio_write_ops, compl_ops,
1243                                NFS_SERVER(inode)->wsize, ioflags);
1244}
1245EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
1246
1247void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
1248{
1249        pgio->pg_ops = &nfs_pageio_write_ops;
1250        pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1251}
1252EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1253
1254
1255void nfs_write_prepare(struct rpc_task *task, void *calldata)
1256{
1257        struct nfs_write_data *data = calldata;
1258        NFS_PROTO(data->header->inode)->write_rpc_prepare(task, data);
1259}
1260
1261void nfs_commit_prepare(struct rpc_task *task, void *calldata)
1262{
1263        struct nfs_commit_data *data = calldata;
1264
1265        NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
1266}
1267
1268/*
1269 * Handle a write reply that flushes a whole page.
1270 *
1271 * FIXME: There is an inherent race with invalidate_inode_pages and
1272 *        writebacks since the page->count is kept > 1 for as long
1273 *        as the page has a write request pending.
1274 */
1275static void nfs_writeback_done_common(struct rpc_task *task, void *calldata)
1276{
1277        struct nfs_write_data   *data = calldata;
1278
1279        nfs_writeback_done(task, data);
1280}
1281
1282static void nfs_writeback_release_common(void *calldata)
1283{
1284        struct nfs_write_data   *data = calldata;
1285        struct nfs_pgio_header *hdr = data->header;
1286        int status = data->task.tk_status;
1287
1288        if ((status >= 0) && nfs_write_need_commit(data)) {
1289                spin_lock(&hdr->lock);
1290                if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags))
1291                        ; /* Do nothing */
1292                else if (!test_and_set_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags))
1293                        memcpy(hdr->verf, &data->verf, sizeof(*hdr->verf));
1294                else if (memcmp(hdr->verf, &data->verf, sizeof(*hdr->verf)))
1295                        set_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags);
1296                spin_unlock(&hdr->lock);
1297        }
1298        nfs_writedata_release(data);
1299}
1300
1301static const struct rpc_call_ops nfs_write_common_ops = {
1302        .rpc_call_prepare = nfs_write_prepare,
1303        .rpc_call_done = nfs_writeback_done_common,
1304        .rpc_release = nfs_writeback_release_common,
1305};
1306
1307
1308/*
1309 * This function is called when the WRITE call is complete.
1310 */
1311void nfs_writeback_done(struct rpc_task *task, struct nfs_write_data *data)
1312{
1313        struct nfs_writeargs    *argp = &data->args;
1314        struct nfs_writeres     *resp = &data->res;
1315        struct inode            *inode = data->header->inode;
1316        int status;
1317
1318        dprintk("NFS: %5u nfs_writeback_done (status %d)\n",
1319                task->tk_pid, task->tk_status);
1320
1321        /*
1322         * ->write_done will attempt to use post-op attributes to detect
1323         * conflicting writes by other clients.  A strict interpretation
1324         * of close-to-open would allow us to continue caching even if
1325         * another writer had changed the file, but some applications
1326         * depend on tighter cache coherency when writing.
1327         */
1328        status = NFS_PROTO(inode)->write_done(task, data);
1329        if (status != 0)
1330                return;
1331        nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, resp->count);
1332
1333#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
1334        if (resp->verf->committed < argp->stable && task->tk_status >= 0) {
1335                /* We tried a write call, but the server did not
1336                 * commit data to stable storage even though we
1337                 * requested it.
1338                 * Note: There is a known bug in Tru64 < 5.0 in which
1339                 *       the server reports NFS_DATA_SYNC, but performs
1340                 *       NFS_FILE_SYNC. We therefore implement this checking
1341                 *       as a dprintk() in order to avoid filling syslog.
1342                 */
1343                static unsigned long    complain;
1344
1345                /* Note this will print the MDS for a DS write */
1346                if (time_before(complain, jiffies)) {
1347                        dprintk("NFS:       faulty NFS server %s:"
1348                                " (committed = %d) != (stable = %d)\n",
1349                                NFS_SERVER(inode)->nfs_client->cl_hostname,
1350                                resp->verf->committed, argp->stable);
1351                        complain = jiffies + 300 * HZ;
1352                }
1353        }
1354#endif
1355        if (task->tk_status < 0)
1356                nfs_set_pgio_error(data->header, task->tk_status, argp->offset);
1357        else if (resp->count < argp->count) {
1358                static unsigned long    complain;
1359
1360                /* This a short write! */
1361                nfs_inc_stats(inode, NFSIOS_SHORTWRITE);
1362
1363                /* Has the server at least made some progress? */
1364                if (resp->count == 0) {
1365                        if (time_before(complain, jiffies)) {
1366                                printk(KERN_WARNING
1367                                       "NFS: Server wrote zero bytes, expected %u.\n",
1368                                       argp->count);
1369                                complain = jiffies + 300 * HZ;
1370                        }
1371                        nfs_set_pgio_error(data->header, -EIO, argp->offset);
1372                        task->tk_status = -EIO;
1373                        return;
1374                }
1375                /* Was this an NFSv2 write or an NFSv3 stable write? */
1376                if (resp->verf->committed != NFS_UNSTABLE) {
1377                        /* Resend from where the server left off */
1378                        data->mds_offset += resp->count;
1379                        argp->offset += resp->count;
1380                        argp->pgbase += resp->count;
1381                        argp->count -= resp->count;
1382                } else {
1383                        /* Resend as a stable write in order to avoid
1384                         * headaches in the case of a server crash.
1385                         */
1386                        argp->stable = NFS_FILE_SYNC;
1387                }
1388                rpc_restart_call_prepare(task);
1389        }
1390}
1391
1392
1393#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
1394static int nfs_commit_set_lock(struct nfs_inode *nfsi, int may_wait)
1395{
1396        int ret;
1397
1398        if (!test_and_set_bit(NFS_INO_COMMIT, &nfsi->flags))
1399                return 1;
1400        if (!may_wait)
1401                return 0;
1402        ret = out_of_line_wait_on_bit_lock(&nfsi->flags,
1403                                NFS_INO_COMMIT,
1404                                nfs_wait_bit_killable,
1405                                TASK_KILLABLE);
1406        return (ret < 0) ? ret : 1;
1407}
1408
1409static void nfs_commit_clear_lock(struct nfs_inode *nfsi)
1410{
1411        clear_bit(NFS_INO_COMMIT, &nfsi->flags);
1412        smp_mb__after_clear_bit();
1413        wake_up_bit(&nfsi->flags, NFS_INO_COMMIT);
1414}
1415
1416void nfs_commitdata_release(struct nfs_commit_data *data)
1417{
1418        put_nfs_open_context(data->context);
1419        nfs_commit_free(data);
1420}
1421EXPORT_SYMBOL_GPL(nfs_commitdata_release);
1422
1423int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1424                        const struct rpc_call_ops *call_ops,
1425                        int how, int flags)
1426{
1427        struct rpc_task *task;
1428        int priority = flush_task_priority(how);
1429        struct rpc_message msg = {
1430                .rpc_argp = &data->args,
1431                .rpc_resp = &data->res,
1432                .rpc_cred = data->cred,
1433        };
1434        struct rpc_task_setup task_setup_data = {
1435                .task = &data->task,
1436                .rpc_client = clnt,
1437                .rpc_message = &msg,
1438                .callback_ops = call_ops,
1439                .callback_data = data,
1440                .workqueue = nfsiod_workqueue,
1441                .flags = RPC_TASK_ASYNC | flags,
1442                .priority = priority,
1443        };
1444        /* Set up the initial task struct.  */
1445        NFS_PROTO(data->inode)->commit_setup(data, &msg);
1446
1447        dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
1448
1449        task = rpc_run_task(&task_setup_data);
1450        if (IS_ERR(task))
1451                return PTR_ERR(task);
1452        if (how & FLUSH_SYNC)
1453                rpc_wait_for_completion_task(task);
1454        rpc_put_task(task);
1455        return 0;
1456}
1457EXPORT_SYMBOL_GPL(nfs_initiate_commit);
1458
1459/*
1460 * Set up the argument/result storage required for the RPC call.
1461 */
1462void nfs_init_commit(struct nfs_commit_data *data,
1463                     struct list_head *head,
1464                     struct pnfs_layout_segment *lseg,
1465                     struct nfs_commit_info *cinfo)
1466{
1467        struct nfs_page *first = nfs_list_entry(head->next);
1468        struct inode *inode = first->wb_context->dentry->d_inode;
1469
1470        /* Set up the RPC argument and reply structs
1471         * NB: take care not to mess about with data->commit et al. */
1472
1473        list_splice_init(head, &data->pages);
1474
1475        data->inode       = inode;
1476        data->cred        = first->wb_context->cred;
1477        data->lseg        = lseg; /* reference transferred */
1478        data->mds_ops     = &nfs_commit_ops;
1479        data->completion_ops = cinfo->completion_ops;
1480        data->dreq        = cinfo->dreq;
1481
1482        data->args.fh     = NFS_FH(data->inode);
1483        /* Note: we always request a commit of the entire inode */
1484        data->args.offset = 0;
1485        data->args.count  = 0;
1486        data->context     = get_nfs_open_context(first->wb_context);
1487        data->res.fattr   = &data->fattr;
1488        data->res.verf    = &data->verf;
1489        nfs_fattr_init(&data->fattr);
1490}
1491EXPORT_SYMBOL_GPL(nfs_init_commit);
1492
1493void nfs_retry_commit(struct list_head *page_list,
1494                      struct pnfs_layout_segment *lseg,
1495                      struct nfs_commit_info *cinfo)
1496{
1497        struct nfs_page *req;
1498
1499        while (!list_empty(page_list)) {
1500                req = nfs_list_entry(page_list->next);
1501                nfs_list_remove_request(req);
1502                nfs_mark_request_commit(req, lseg, cinfo);
1503                if (!cinfo->dreq) {
1504                        dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
1505                        dec_bdi_stat(page_file_mapping(req->wb_page)->backing_dev_info,
1506                                     BDI_RECLAIMABLE);
1507                }
1508                nfs_unlock_and_release_request(req);
1509        }
1510}
1511EXPORT_SYMBOL_GPL(nfs_retry_commit);
1512
1513/*
1514 * Commit dirty pages
1515 */
1516static int
1517nfs_commit_list(struct inode *inode, struct list_head *head, int how,
1518                struct nfs_commit_info *cinfo)
1519{
1520        struct nfs_commit_data  *data;
1521
1522        data = nfs_commitdata_alloc();
1523
1524        if (!data)
1525                goto out_bad;
1526
1527        /* Set up the argument struct */
1528        nfs_init_commit(data, head, NULL, cinfo);
1529        atomic_inc(&cinfo->mds->rpcs_out);
1530        return nfs_initiate_commit(NFS_CLIENT(inode), data, data->mds_ops,
1531                                   how, 0);
1532 out_bad:
1533        nfs_retry_commit(head, NULL, cinfo);
1534        cinfo->completion_ops->error_cleanup(NFS_I(inode));
1535        return -ENOMEM;
1536}
1537
1538/*
1539 * COMMIT call returned
1540 */
1541static void nfs_commit_done(struct rpc_task *task, void *calldata)
1542{
1543        struct nfs_commit_data  *data = calldata;
1544
1545        dprintk("NFS: %5u nfs_commit_done (status %d)\n",
1546                                task->tk_pid, task->tk_status);
1547
1548        /* Call the NFS version-specific code */
1549        NFS_PROTO(data->inode)->commit_done(task, data);
1550}
1551
1552static void nfs_commit_release_pages(struct nfs_commit_data *data)
1553{
1554        struct nfs_page *req;
1555        int status = data->task.tk_status;
1556        struct nfs_commit_info cinfo;
1557
1558        while (!list_empty(&data->pages)) {
1559                req = nfs_list_entry(data->pages.next);
1560                nfs_list_remove_request(req);
1561                nfs_clear_page_commit(req->wb_page);
1562
1563                dprintk("NFS:       commit (%s/%lld %d@%lld)",
1564                        req->wb_context->dentry->d_sb->s_id,
1565                        (long long)NFS_FILEID(req->wb_context->dentry->d_inode),
1566                        req->wb_bytes,
1567                        (long long)req_offset(req));
1568                if (status < 0) {
1569                        nfs_context_set_write_error(req->wb_context, status);
1570                        nfs_inode_remove_request(req);
1571                        dprintk(", error = %d\n", status);
1572                        goto next;
1573                }
1574
1575                /* Okay, COMMIT succeeded, apparently. Check the verifier
1576                 * returned by the server against all stored verfs. */
1577                if (!memcmp(&req->wb_verf, &data->verf.verifier, sizeof(req->wb_verf))) {
1578                        /* We have a match */
1579                        nfs_inode_remove_request(req);
1580                        dprintk(" OK\n");
1581                        goto next;
1582                }
1583                /* We have a mismatch. Write the page again */
1584                dprintk(" mismatch\n");
1585                nfs_mark_request_dirty(req);
1586                set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1587        next:
1588                nfs_unlock_and_release_request(req);
1589        }
1590        nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1591        if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
1592                nfs_commit_clear_lock(NFS_I(data->inode));
1593}
1594
1595static void nfs_commit_release(void *calldata)
1596{
1597        struct nfs_commit_data *data = calldata;
1598
1599        data->completion_ops->completion(data);
1600        nfs_commitdata_release(calldata);
1601}
1602
1603static const struct rpc_call_ops nfs_commit_ops = {
1604        .rpc_call_prepare = nfs_commit_prepare,
1605        .rpc_call_done = nfs_commit_done,
1606        .rpc_release = nfs_commit_release,
1607};
1608
1609static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
1610        .completion = nfs_commit_release_pages,
1611        .error_cleanup = nfs_commit_clear_lock,
1612};
1613
1614int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
1615                            int how, struct nfs_commit_info *cinfo)
1616{
1617        int status;
1618
1619        status = pnfs_commit_list(inode, head, how, cinfo);
1620        if (status == PNFS_NOT_ATTEMPTED)
1621                status = nfs_commit_list(inode, head, how, cinfo);
1622        return status;
1623}
1624
1625int nfs_commit_inode(struct inode *inode, int how)
1626{
1627        LIST_HEAD(head);
1628        struct nfs_commit_info cinfo;
1629        int may_wait = how & FLUSH_SYNC;
1630        int res;
1631
1632        res = nfs_commit_set_lock(NFS_I(inode), may_wait);
1633        if (res <= 0)
1634                goto out_mark_dirty;
1635        nfs_init_cinfo_from_inode(&cinfo, inode);
1636        res = nfs_scan_commit(inode, &head, &cinfo);
1637        if (res) {
1638                int error;
1639
1640                error = nfs_generic_commit_list(inode, &head, how, &cinfo);
1641                if (error < 0)
1642                        return error;
1643                if (!may_wait)
1644                        goto out_mark_dirty;
1645                error = wait_on_bit(&NFS_I(inode)->flags,
1646                                NFS_INO_COMMIT,
1647                                nfs_wait_bit_killable,
1648                                TASK_KILLABLE);
1649                if (error < 0)
1650                        return error;
1651        } else
1652                nfs_commit_clear_lock(NFS_I(inode));
1653        return res;
1654        /* Note: If we exit without ensuring that the commit is complete,
1655         * we must mark the inode as dirty. Otherwise, future calls to
1656         * sync_inode() with the WB_SYNC_ALL flag set will fail to ensure
1657         * that the data is on the disk.
1658         */
1659out_mark_dirty:
1660        __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1661        return res;
1662}
1663
1664static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc)
1665{
1666        struct nfs_inode *nfsi = NFS_I(inode);
1667        int flags = FLUSH_SYNC;
1668        int ret = 0;
1669
1670        /* no commits means nothing needs to be done */
1671        if (!nfsi->commit_info.ncommit)
1672                return ret;
1673
1674        if (wbc->sync_mode == WB_SYNC_NONE) {
1675                /* Don't commit yet if this is a non-blocking flush and there
1676                 * are a lot of outstanding writes for this mapping.
1677                 */
1678                if (nfsi->commit_info.ncommit <= (nfsi->npages >> 1))
1679                        goto out_mark_dirty;
1680
1681                /* don't wait for the COMMIT response */
1682                flags = 0;
1683        }
1684
1685        ret = nfs_commit_inode(inode, flags);
1686        if (ret >= 0) {
1687                if (wbc->sync_mode == WB_SYNC_NONE) {
1688                        if (ret < wbc->nr_to_write)
1689                                wbc->nr_to_write -= ret;
1690                        else
1691                                wbc->nr_to_write = 0;
1692                }
1693                return 0;
1694        }
1695out_mark_dirty:
1696        __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1697        return ret;
1698}
1699#else
1700static int nfs_commit_unstable_pages(struct inode *inode, struct writeback_control *wbc)
1701{
1702        return 0;
1703}
1704#endif
1705
1706int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1707{
1708        return nfs_commit_unstable_pages(inode, wbc);
1709}
1710EXPORT_SYMBOL_GPL(nfs_write_inode);
1711
1712/*
1713 * flush the inode to disk.
1714 */
1715int nfs_wb_all(struct inode *inode)
1716{
1717        struct writeback_control wbc = {
1718                .sync_mode = WB_SYNC_ALL,
1719                .nr_to_write = LONG_MAX,
1720                .range_start = 0,
1721                .range_end = LLONG_MAX,
1722        };
1723
1724        return sync_inode(inode, &wbc);
1725}
1726EXPORT_SYMBOL_GPL(nfs_wb_all);
1727
1728int nfs_wb_page_cancel(struct inode *inode, struct page *page)
1729{
1730        struct nfs_page *req;
1731        int ret = 0;
1732
1733        BUG_ON(!PageLocked(page));
1734        for (;;) {
1735                wait_on_page_writeback(page);
1736                req = nfs_page_find_request(page);
1737                if (req == NULL)
1738                        break;
1739                if (nfs_lock_request(req)) {
1740                        nfs_clear_request_commit(req);
1741                        nfs_inode_remove_request(req);
1742                        /*
1743                         * In case nfs_inode_remove_request has marked the
1744                         * page as being dirty
1745                         */
1746                        cancel_dirty_page(page, PAGE_CACHE_SIZE);
1747                        nfs_unlock_and_release_request(req);
1748                        break;
1749                }
1750                ret = nfs_wait_on_request(req);
1751                nfs_release_request(req);
1752                if (ret < 0)
1753                        break;
1754        }
1755        return ret;
1756}
1757
1758/*
1759 * Write back all requests on one page - we do this before reading it.
1760 */
1761int nfs_wb_page(struct inode *inode, struct page *page)
1762{
1763        loff_t range_start = page_file_offset(page);
1764        loff_t range_end = range_start + (loff_t)(PAGE_CACHE_SIZE - 1);
1765        struct writeback_control wbc = {
1766                .sync_mode = WB_SYNC_ALL,
1767                .nr_to_write = 0,
1768                .range_start = range_start,
1769                .range_end = range_end,
1770        };
1771        int ret;
1772
1773        for (;;) {
1774                wait_on_page_writeback(page);
1775                if (clear_page_dirty_for_io(page)) {
1776                        ret = nfs_writepage_locked(page, &wbc);
1777                        if (ret < 0)
1778                                goto out_error;
1779                        continue;
1780                }
1781                if (!PagePrivate(page))
1782                        break;
1783                ret = nfs_commit_inode(inode, FLUSH_SYNC);
1784                if (ret < 0)
1785                        goto out_error;
1786        }
1787        return 0;
1788out_error:
1789        return ret;
1790}
1791
1792#ifdef CONFIG_MIGRATION
1793int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
1794                struct page *page, enum migrate_mode mode)
1795{
1796        /*
1797         * If PagePrivate is set, then the page is currently associated with
1798         * an in-progress read or write request. Don't try to migrate it.
1799         *
1800         * FIXME: we could do this in principle, but we'll need a way to ensure
1801         *        that we can safely release the inode reference while holding
1802         *        the page lock.
1803         */
1804        if (PagePrivate(page))
1805                return -EBUSY;
1806
1807        nfs_fscache_release_page(page, GFP_KERNEL);
1808
1809        return migrate_page(mapping, newpage, page, mode);
1810}
1811#endif
1812
1813int __init nfs_init_writepagecache(void)
1814{
1815        nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
1816                                             sizeof(struct nfs_write_header),
1817                                             0, SLAB_HWCACHE_ALIGN,
1818                                             NULL);
1819        if (nfs_wdata_cachep == NULL)
1820                return -ENOMEM;
1821
1822        nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
1823                                                     nfs_wdata_cachep);
1824        if (nfs_wdata_mempool == NULL)
1825                goto out_destroy_write_cache;
1826
1827        nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
1828                                             sizeof(struct nfs_commit_data),
1829                                             0, SLAB_HWCACHE_ALIGN,
1830                                             NULL);
1831        if (nfs_cdata_cachep == NULL)
1832                goto out_destroy_write_mempool;
1833
1834        nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
1835                                                      nfs_wdata_cachep);
1836        if (nfs_commit_mempool == NULL)
1837                goto out_destroy_commit_cache;
1838
1839        /*
1840         * NFS congestion size, scale with available memory.
1841         *
1842         *  64MB:    8192k
1843         * 128MB:   11585k
1844         * 256MB:   16384k
1845         * 512MB:   23170k
1846         *   1GB:   32768k
1847         *   2GB:   46340k
1848         *   4GB:   65536k
1849         *   8GB:   92681k
1850         *  16GB:  131072k
1851         *
1852         * This allows larger machines to have larger/more transfers.
1853         * Limit the default to 256M
1854         */
1855        nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
1856        if (nfs_congestion_kb > 256*1024)
1857                nfs_congestion_kb = 256*1024;
1858
1859        return 0;
1860
1861out_destroy_commit_cache:
1862        kmem_cache_destroy(nfs_cdata_cachep);
1863out_destroy_write_mempool:
1864        mempool_destroy(nfs_wdata_mempool);
1865out_destroy_write_cache:
1866        kmem_cache_destroy(nfs_wdata_cachep);
1867        return -ENOMEM;
1868}
1869
1870void nfs_destroy_writepagecache(void)
1871{
1872        mempool_destroy(nfs_commit_mempool);
1873        kmem_cache_destroy(nfs_cdata_cachep);
1874        mempool_destroy(nfs_wdata_mempool);
1875        kmem_cache_destroy(nfs_wdata_cachep);
1876}
1877
1878
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.