linux/fs/xfs/xfs_aops.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
   3 * All Rights Reserved.
   4 *
   5 * This program is free software; you can redistribute it and/or
   6 * modify it under the terms of the GNU General Public License as
   7 * published by the Free Software Foundation.
   8 *
   9 * This program is distributed in the hope that it would be useful,
  10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 * GNU General Public License for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write the Free Software Foundation,
  16 * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18#include "xfs.h"
  19#include "xfs_log.h"
  20#include "xfs_sb.h"
  21#include "xfs_ag.h"
  22#include "xfs_trans.h"
  23#include "xfs_mount.h"
  24#include "xfs_bmap_btree.h"
  25#include "xfs_dinode.h"
  26#include "xfs_inode.h"
  27#include "xfs_inode_item.h"
  28#include "xfs_alloc.h"
  29#include "xfs_error.h"
  30#include "xfs_iomap.h"
  31#include "xfs_vnodeops.h"
  32#include "xfs_trace.h"
  33#include "xfs_bmap.h"
  34#include <linux/gfp.h>
  35#include <linux/mpage.h>
  36#include <linux/pagevec.h>
  37#include <linux/writeback.h>
  38
  39void
  40xfs_count_page_state(
  41        struct page             *page,
  42        int                     *delalloc,
  43        int                     *unwritten)
  44{
  45        struct buffer_head      *bh, *head;
  46
  47        *delalloc = *unwritten = 0;
  48
  49        bh = head = page_buffers(page);
  50        do {
  51                if (buffer_unwritten(bh))
  52                        (*unwritten) = 1;
  53                else if (buffer_delay(bh))
  54                        (*delalloc) = 1;
  55        } while ((bh = bh->b_this_page) != head);
  56}
  57
  58STATIC struct block_device *
  59xfs_find_bdev_for_inode(
  60        struct inode            *inode)
  61{
  62        struct xfs_inode        *ip = XFS_I(inode);
  63        struct xfs_mount        *mp = ip->i_mount;
  64
  65        if (XFS_IS_REALTIME_INODE(ip))
  66                return mp->m_rtdev_targp->bt_bdev;
  67        else
  68                return mp->m_ddev_targp->bt_bdev;
  69}
  70
  71/*
  72 * We're now finished for good with this ioend structure.
  73 * Update the page state via the associated buffer_heads,
  74 * release holds on the inode and bio, and finally free
  75 * up memory.  Do not use the ioend after this.
  76 */
  77STATIC void
  78xfs_destroy_ioend(
  79        xfs_ioend_t             *ioend)
  80{
  81        struct buffer_head      *bh, *next;
  82
  83        for (bh = ioend->io_buffer_head; bh; bh = next) {
  84                next = bh->b_private;
  85                bh->b_end_io(bh, !ioend->io_error);
  86        }
  87
  88        if (ioend->io_iocb) {
  89                inode_dio_done(ioend->io_inode);
  90                if (ioend->io_isasync) {
  91                        aio_complete(ioend->io_iocb, ioend->io_error ?
  92                                        ioend->io_error : ioend->io_result, 0);
  93                }
  94        }
  95
  96        mempool_free(ioend, xfs_ioend_pool);
  97}
  98
  99/*
 100 * Fast and loose check if this write could update the on-disk inode size.
 101 */
 102static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend)
 103{
 104        return ioend->io_offset + ioend->io_size >
 105                XFS_I(ioend->io_inode)->i_d.di_size;
 106}
 107
 108STATIC int
 109xfs_setfilesize_trans_alloc(
 110        struct xfs_ioend        *ioend)
 111{
 112        struct xfs_mount        *mp = XFS_I(ioend->io_inode)->i_mount;
 113        struct xfs_trans        *tp;
 114        int                     error;
 115
 116        tp = xfs_trans_alloc(mp, XFS_TRANS_FSYNC_TS);
 117
 118        error = xfs_trans_reserve(tp, 0, XFS_FSYNC_TS_LOG_RES(mp), 0, 0, 0);
 119        if (error) {
 120                xfs_trans_cancel(tp, 0);
 121                return error;
 122        }
 123
 124        ioend->io_append_trans = tp;
 125
 126        /*
 127         * We may pass freeze protection with a transaction.  So tell lockdep
 128         * we released it.
 129         */
 130        rwsem_release(&ioend->io_inode->i_sb->s_writers.lock_map[SB_FREEZE_FS-1],
 131                      1, _THIS_IP_);
 132        /*
 133         * We hand off the transaction to the completion thread now, so
 134         * clear the flag here.
 135         */
 136        current_restore_flags_nested(&tp->t_pflags, PF_FSTRANS);
 137        return 0;
 138}
 139
 140/*
 141 * Update on-disk file size now that data has been written to disk.
 142 */
 143STATIC int
 144xfs_setfilesize(
 145        struct xfs_ioend        *ioend)
 146{
 147        struct xfs_inode        *ip = XFS_I(ioend->io_inode);
 148        struct xfs_trans        *tp = ioend->io_append_trans;
 149        xfs_fsize_t             isize;
 150
 151        /*
 152         * The transaction may have been allocated in the I/O submission thread,
 153         * thus we need to mark ourselves as beeing in a transaction manually.
 154         * Similarly for freeze protection.
 155         */
 156        current_set_flags_nested(&tp->t_pflags, PF_FSTRANS);
 157        rwsem_acquire_read(&VFS_I(ip)->i_sb->s_writers.lock_map[SB_FREEZE_FS-1],
 158                           0, 1, _THIS_IP_);
 159
 160        xfs_ilock(ip, XFS_ILOCK_EXCL);
 161        isize = xfs_new_eof(ip, ioend->io_offset + ioend->io_size);
 162        if (!isize) {
 163                xfs_iunlock(ip, XFS_ILOCK_EXCL);
 164                xfs_trans_cancel(tp, 0);
 165                return 0;
 166        }
 167
 168        trace_xfs_setfilesize(ip, ioend->io_offset, ioend->io_size);
 169
 170        ip->i_d.di_size = isize;
 171        xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
 172        xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 173
 174        return xfs_trans_commit(tp, 0);
 175}
 176
 177/*
 178 * Schedule IO completion handling on the final put of an ioend.
 179 *
 180 * If there is no work to do we might as well call it a day and free the
 181 * ioend right now.
 182 */
 183STATIC void
 184xfs_finish_ioend(
 185        struct xfs_ioend        *ioend)
 186{
 187        if (atomic_dec_and_test(&ioend->io_remaining)) {
 188                struct xfs_mount        *mp = XFS_I(ioend->io_inode)->i_mount;
 189
 190                if (ioend->io_type == XFS_IO_UNWRITTEN)
 191                        queue_work(mp->m_unwritten_workqueue, &ioend->io_work);
 192                else if (ioend->io_append_trans ||
 193                         (ioend->io_isdirect && xfs_ioend_is_append(ioend)))
 194                        queue_work(mp->m_data_workqueue, &ioend->io_work);
 195                else
 196                        xfs_destroy_ioend(ioend);
 197        }
 198}
 199
 200/*
 201 * IO write completion.
 202 */
 203STATIC void
 204xfs_end_io(
 205        struct work_struct *work)
 206{
 207        xfs_ioend_t     *ioend = container_of(work, xfs_ioend_t, io_work);
 208        struct xfs_inode *ip = XFS_I(ioend->io_inode);
 209        int             error = 0;
 210
 211        if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
 212                ioend->io_error = -EIO;
 213                goto done;
 214        }
 215        if (ioend->io_error)
 216                goto done;
 217
 218        /*
 219         * For unwritten extents we need to issue transactions to convert a
 220         * range to normal written extens after the data I/O has finished.
 221         */
 222        if (ioend->io_type == XFS_IO_UNWRITTEN) {
 223                error = xfs_iomap_write_unwritten(ip, ioend->io_offset,
 224                                                  ioend->io_size);
 225        } else if (ioend->io_isdirect && xfs_ioend_is_append(ioend)) {
 226                /*
 227                 * For direct I/O we do not know if we need to allocate blocks
 228                 * or not so we can't preallocate an append transaction as that
 229                 * results in nested reservations and log space deadlocks. Hence
 230                 * allocate the transaction here. While this is sub-optimal and
 231                 * can block IO completion for some time, we're stuck with doing
 232                 * it this way until we can pass the ioend to the direct IO
 233                 * allocation callbacks and avoid nesting that way.
 234                 */
 235                error = xfs_setfilesize_trans_alloc(ioend);
 236                if (error)
 237                        goto done;
 238                error = xfs_setfilesize(ioend);
 239        } else if (ioend->io_append_trans) {
 240                error = xfs_setfilesize(ioend);
 241        } else {
 242                ASSERT(!xfs_ioend_is_append(ioend));
 243        }
 244
 245done:
 246        if (error)
 247                ioend->io_error = -error;
 248        xfs_destroy_ioend(ioend);
 249}
 250
 251/*
 252 * Call IO completion handling in caller context on the final put of an ioend.
 253 */
 254STATIC void
 255xfs_finish_ioend_sync(
 256        struct xfs_ioend        *ioend)
 257{
 258        if (atomic_dec_and_test(&ioend->io_remaining))
 259                xfs_end_io(&ioend->io_work);
 260}
 261
 262/*
 263 * Allocate and initialise an IO completion structure.
 264 * We need to track unwritten extent write completion here initially.
 265 * We'll need to extend this for updating the ondisk inode size later
 266 * (vs. incore size).
 267 */
 268STATIC xfs_ioend_t *
 269xfs_alloc_ioend(
 270        struct inode            *inode,
 271        unsigned int            type)
 272{
 273        xfs_ioend_t             *ioend;
 274
 275        ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);
 276
 277        /*
 278         * Set the count to 1 initially, which will prevent an I/O
 279         * completion callback from happening before we have started
 280         * all the I/O from calling the completion routine too early.
 281         */
 282        atomic_set(&ioend->io_remaining, 1);
 283        ioend->io_isasync = 0;
 284        ioend->io_isdirect = 0;
 285        ioend->io_error = 0;
 286        ioend->io_list = NULL;
 287        ioend->io_type = type;
 288        ioend->io_inode = inode;
 289        ioend->io_buffer_head = NULL;
 290        ioend->io_buffer_tail = NULL;
 291        ioend->io_offset = 0;
 292        ioend->io_size = 0;
 293        ioend->io_iocb = NULL;
 294        ioend->io_result = 0;
 295        ioend->io_append_trans = NULL;
 296
 297        INIT_WORK(&ioend->io_work, xfs_end_io);
 298        return ioend;
 299}
 300
 301STATIC int
 302xfs_map_blocks(
 303        struct inode            *inode,
 304        loff_t                  offset,
 305        struct xfs_bmbt_irec    *imap,
 306        int                     type,
 307        int                     nonblocking)
 308{
 309        struct xfs_inode        *ip = XFS_I(inode);
 310        struct xfs_mount        *mp = ip->i_mount;
 311        ssize_t                 count = 1 << inode->i_blkbits;
 312        xfs_fileoff_t           offset_fsb, end_fsb;
 313        int                     error = 0;
 314        int                     bmapi_flags = XFS_BMAPI_ENTIRE;
 315        int                     nimaps = 1;
 316
 317        if (XFS_FORCED_SHUTDOWN(mp))
 318                return -XFS_ERROR(EIO);
 319
 320        if (type == XFS_IO_UNWRITTEN)
 321                bmapi_flags |= XFS_BMAPI_IGSTATE;
 322
 323        if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) {
 324                if (nonblocking)
 325                        return -XFS_ERROR(EAGAIN);
 326                xfs_ilock(ip, XFS_ILOCK_SHARED);
 327        }
 328
 329        ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE ||
 330               (ip->i_df.if_flags & XFS_IFEXTENTS));
 331        ASSERT(offset <= mp->m_super->s_maxbytes);
 332
 333        if (offset + count > mp->m_super->s_maxbytes)
 334                count = mp->m_super->s_maxbytes - offset;
 335        end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count);
 336        offset_fsb = XFS_B_TO_FSBT(mp, offset);
 337        error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
 338                                imap, &nimaps, bmapi_flags);
 339        xfs_iunlock(ip, XFS_ILOCK_SHARED);
 340
 341        if (error)
 342                return -XFS_ERROR(error);
 343
 344        if (type == XFS_IO_DELALLOC &&
 345            (!nimaps || isnullstartblock(imap->br_startblock))) {
 346                error = xfs_iomap_write_allocate(ip, offset, count, imap);
 347                if (!error)
 348                        trace_xfs_map_blocks_alloc(ip, offset, count, type, imap);
 349                return -XFS_ERROR(error);
 350        }
 351
 352#ifdef DEBUG
 353        if (type == XFS_IO_UNWRITTEN) {
 354                ASSERT(nimaps);
 355                ASSERT(imap->br_startblock != HOLESTARTBLOCK);
 356                ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
 357        }
 358#endif
 359        if (nimaps)
 360                trace_xfs_map_blocks_found(ip, offset, count, type, imap);
 361        return 0;
 362}
 363
 364STATIC int
 365xfs_imap_valid(
 366        struct inode            *inode,
 367        struct xfs_bmbt_irec    *imap,
 368        xfs_off_t               offset)
 369{
 370        offset >>= inode->i_blkbits;
 371
 372        return offset >= imap->br_startoff &&
 373                offset < imap->br_startoff + imap->br_blockcount;
 374}
 375
 376/*
 377 * BIO completion handler for buffered IO.
 378 */
 379STATIC void
 380xfs_end_bio(
 381        struct bio              *bio,
 382        int                     error)
 383{
 384        xfs_ioend_t             *ioend = bio->bi_private;
 385
 386        ASSERT(atomic_read(&bio->bi_cnt) >= 1);
 387        ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;
 388
 389        /* Toss bio and pass work off to an xfsdatad thread */
 390        bio->bi_private = NULL;
 391        bio->bi_end_io = NULL;
 392        bio_put(bio);
 393
 394        xfs_finish_ioend(ioend);
 395}
 396
 397STATIC void
 398xfs_submit_ioend_bio(
 399        struct writeback_control *wbc,
 400        xfs_ioend_t             *ioend,
 401        struct bio              *bio)
 402{
 403        atomic_inc(&ioend->io_remaining);
 404        bio->bi_private = ioend;
 405        bio->bi_end_io = xfs_end_bio;
 406        submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, bio);
 407}
 408
 409STATIC struct bio *
 410xfs_alloc_ioend_bio(
 411        struct buffer_head      *bh)
 412{
 413        int                     nvecs = bio_get_nr_vecs(bh->b_bdev);
 414        struct bio              *bio = bio_alloc(GFP_NOIO, nvecs);
 415
 416        ASSERT(bio->bi_private == NULL);
 417        bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
 418        bio->bi_bdev = bh->b_bdev;
 419        return bio;
 420}
 421
 422STATIC void
 423xfs_start_buffer_writeback(
 424        struct buffer_head      *bh)
 425{
 426        ASSERT(buffer_mapped(bh));
 427        ASSERT(buffer_locked(bh));
 428        ASSERT(!buffer_delay(bh));
 429        ASSERT(!buffer_unwritten(bh));
 430
 431        mark_buffer_async_write(bh);
 432        set_buffer_uptodate(bh);
 433        clear_buffer_dirty(bh);
 434}
 435
 436STATIC void
 437xfs_start_page_writeback(
 438        struct page             *page,
 439        int                     clear_dirty,
 440        int                     buffers)
 441{
 442        ASSERT(PageLocked(page));
 443        ASSERT(!PageWriteback(page));
 444        if (clear_dirty)
 445                clear_page_dirty_for_io(page);
 446        set_page_writeback(page);
 447        unlock_page(page);
 448        /* If no buffers on the page are to be written, finish it here */
 449        if (!buffers)
 450                end_page_writeback(page);
 451}
 452
 453static inline int bio_add_buffer(struct bio *bio, struct buffer_head *bh)
 454{
 455        return bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
 456}
 457
 458/*
 459 * Submit all of the bios for all of the ioends we have saved up, covering the
 460 * initial writepage page and also any probed pages.
 461 *
 462 * Because we may have multiple ioends spanning a page, we need to start
 463 * writeback on all the buffers before we submit them for I/O. If we mark the
 464 * buffers as we got, then we can end up with a page that only has buffers
 465 * marked async write and I/O complete on can occur before we mark the other
 466 * buffers async write.
 467 *
 468 * The end result of this is that we trip a bug in end_page_writeback() because
 469 * we call it twice for the one page as the code in end_buffer_async_write()
 470 * assumes that all buffers on the page are started at the same time.
 471 *
 472 * The fix is two passes across the ioend list - one to start writeback on the
 473 * buffer_heads, and then submit them for I/O on the second pass.
 474 *
 475 * If @fail is non-zero, it means that we have a situation where some part of
 476 * the submission process has failed after we have marked paged for writeback
 477 * and unlocked them. In this situation, we need to fail the ioend chain rather
 478 * than submit it to IO. This typically only happens on a filesystem shutdown.
 479 */
 480STATIC void
 481xfs_submit_ioend(
 482        struct writeback_control *wbc,
 483        xfs_ioend_t             *ioend,
 484        int                     fail)
 485{
 486        xfs_ioend_t             *head = ioend;
 487        xfs_ioend_t             *next;
 488        struct buffer_head      *bh;
 489        struct bio              *bio;
 490        sector_t                lastblock = 0;
 491
 492        /* Pass 1 - start writeback */
 493        do {
 494                next = ioend->io_list;
 495                for (bh = ioend->io_buffer_head; bh; bh = bh->b_private)
 496                        xfs_start_buffer_writeback(bh);
 497        } while ((ioend = next) != NULL);
 498
 499        /* Pass 2 - submit I/O */
 500        ioend = head;
 501        do {
 502                next = ioend->io_list;
 503                bio = NULL;
 504
 505                /*
 506                 * If we are failing the IO now, just mark the ioend with an
 507                 * error and finish it. This will run IO completion immediately
 508                 * as there is only one reference to the ioend at this point in
 509                 * time.
 510                 */
 511                if (fail) {
 512                        ioend->io_error = -fail;
 513                        xfs_finish_ioend(ioend);
 514                        continue;
 515                }
 516
 517                for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {
 518
 519                        if (!bio) {
 520 retry:
 521                                bio = xfs_alloc_ioend_bio(bh);
 522                        } else if (bh->b_blocknr != lastblock + 1) {
 523                                xfs_submit_ioend_bio(wbc, ioend, bio);
 524                                goto retry;
 525                        }
 526
 527                        if (bio_add_buffer(bio, bh) != bh->b_size) {
 528                                xfs_submit_ioend_bio(wbc, ioend, bio);
 529                                goto retry;
 530                        }
 531
 532                        lastblock = bh->b_blocknr;
 533                }
 534                if (bio)
 535                        xfs_submit_ioend_bio(wbc, ioend, bio);
 536                xfs_finish_ioend(ioend);
 537        } while ((ioend = next) != NULL);
 538}
 539
 540/*
 541 * Cancel submission of all buffer_heads so far in this endio.
 542 * Toss the endio too.  Only ever called for the initial page
 543 * in a writepage request, so only ever one page.
 544 */
 545STATIC void
 546xfs_cancel_ioend(
 547        xfs_ioend_t             *ioend)
 548{
 549        xfs_ioend_t             *next;
 550        struct buffer_head      *bh, *next_bh;
 551
 552        do {
 553                next = ioend->io_list;
 554                bh = ioend->io_buffer_head;
 555                do {
 556                        next_bh = bh->b_private;
 557                        clear_buffer_async_write(bh);
 558                        unlock_buffer(bh);
 559                } while ((bh = next_bh) != NULL);
 560
 561                mempool_free(ioend, xfs_ioend_pool);
 562        } while ((ioend = next) != NULL);
 563}
 564
 565/*
 566 * Test to see if we've been building up a completion structure for
 567 * earlier buffers -- if so, we try to append to this ioend if we
 568 * can, otherwise we finish off any current ioend and start another.
 569 * Return true if we've finished the given ioend.
 570 */
 571STATIC void
 572xfs_add_to_ioend(
 573        struct inode            *inode,
 574        struct buffer_head      *bh,
 575        xfs_off_t               offset,
 576        unsigned int            type,
 577        xfs_ioend_t             **result,
 578        int                     need_ioend)
 579{
 580        xfs_ioend_t             *ioend = *result;
 581
 582        if (!ioend || need_ioend || type != ioend->io_type) {
 583                xfs_ioend_t     *previous = *result;
 584
 585                ioend = xfs_alloc_ioend(inode, type);
 586                ioend->io_offset = offset;
 587                ioend->io_buffer_head = bh;
 588                ioend->io_buffer_tail = bh;
 589                if (previous)
 590                        previous->io_list = ioend;
 591                *result = ioend;
 592        } else {
 593                ioend->io_buffer_tail->b_private = bh;
 594                ioend->io_buffer_tail = bh;
 595        }
 596
 597        bh->b_private = NULL;
 598        ioend->io_size += bh->b_size;
 599}
 600
 601STATIC void
 602xfs_map_buffer(
 603        struct inode            *inode,
 604        struct buffer_head      *bh,
 605        struct xfs_bmbt_irec    *imap,
 606        xfs_off_t               offset)
 607{
 608        sector_t                bn;
 609        struct xfs_mount        *m = XFS_I(inode)->i_mount;
 610        xfs_off_t               iomap_offset = XFS_FSB_TO_B(m, imap->br_startoff);
 611        xfs_daddr_t             iomap_bn = xfs_fsb_to_db(XFS_I(inode), imap->br_startblock);
 612
 613        ASSERT(imap->br_startblock != HOLESTARTBLOCK);
 614        ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
 615
 616        bn = (iomap_bn >> (inode->i_blkbits - BBSHIFT)) +
 617              ((offset - iomap_offset) >> inode->i_blkbits);
 618
 619        ASSERT(bn || XFS_IS_REALTIME_INODE(XFS_I(inode)));
 620
 621        bh->b_blocknr = bn;
 622        set_buffer_mapped(bh);
 623}
 624
 625STATIC void
 626xfs_map_at_offset(
 627        struct inode            *inode,
 628        struct buffer_head      *bh,
 629        struct xfs_bmbt_irec    *imap,
 630        xfs_off_t               offset)
 631{
 632        ASSERT(imap->br_startblock != HOLESTARTBLOCK);
 633        ASSERT(imap->br_startblock != DELAYSTARTBLOCK);
 634
 635        xfs_map_buffer(inode, bh, imap, offset);
 636        set_buffer_mapped(bh);
 637        clear_buffer_delay(bh);
 638        clear_buffer_unwritten(bh);
 639}
 640
 641/*
 642 * Test if a given page is suitable for writing as part of an unwritten
 643 * or delayed allocate extent.
 644 */
 645STATIC int
 646xfs_check_page_type(
 647        struct page             *page,
 648        unsigned int            type)
 649{
 650        if (PageWriteback(page))
 651                return 0;
 652
 653        if (page->mapping && page_has_buffers(page)) {
 654                struct buffer_head      *bh, *head;
 655                int                     acceptable = 0;
 656
 657                bh = head = page_buffers(page);
 658                do {
 659                        if (buffer_unwritten(bh))
 660                                acceptable += (type == XFS_IO_UNWRITTEN);
 661                        else if (buffer_delay(bh))
 662                                acceptable += (type == XFS_IO_DELALLOC);
 663                        else if (buffer_dirty(bh) && buffer_mapped(bh))
 664                                acceptable += (type == XFS_IO_OVERWRITE);
 665                        else
 666                                break;
 667                } while ((bh = bh->b_this_page) != head);
 668
 669                if (acceptable)
 670                        return 1;
 671        }
 672
 673        return 0;
 674}
 675
 676/*
 677 * Allocate & map buffers for page given the extent map. Write it out.
 678 * except for the original page of a writepage, this is called on
 679 * delalloc/unwritten pages only, for the original page it is possible
 680 * that the page has no mapping at all.
 681 */
 682STATIC int
 683xfs_convert_page(
 684        struct inode            *inode,
 685        struct page             *page,
 686        loff_t                  tindex,
 687        struct xfs_bmbt_irec    *imap,
 688        xfs_ioend_t             **ioendp,
 689        struct writeback_control *wbc)
 690{
 691        struct buffer_head      *bh, *head;
 692        xfs_off_t               end_offset;
 693        unsigned long           p_offset;
 694        unsigned int            type;
 695        int                     len, page_dirty;
 696        int                     count = 0, done = 0, uptodate = 1;
 697        xfs_off_t               offset = page_offset(page);
 698
 699        if (page->index != tindex)
 700                goto fail;
 701        if (!trylock_page(page))
 702                goto fail;
 703        if (PageWriteback(page))
 704                goto fail_unlock_page;
 705        if (page->mapping != inode->i_mapping)
 706                goto fail_unlock_page;
 707        if (!xfs_check_page_type(page, (*ioendp)->io_type))
 708                goto fail_unlock_page;
 709
 710        /*
 711         * page_dirty is initially a count of buffers on the page before
 712         * EOF and is decremented as we move each into a cleanable state.
 713         *
 714         * Derivation:
 715         *
 716         * End offset is the highest offset that this page should represent.
 717         * If we are on the last page, (end_offset & (PAGE_CACHE_SIZE - 1))
 718         * will evaluate non-zero and be less than PAGE_CACHE_SIZE and
 719         * hence give us the correct page_dirty count. On any other page,
 720         * it will be zero and in that case we need page_dirty to be the
 721         * count of buffers on the page.
 722         */
 723        end_offset = min_t(unsigned long long,
 724                        (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
 725                        i_size_read(inode));
 726
 727        len = 1 << inode->i_blkbits;
 728        p_offset = min_t(unsigned long, end_offset & (PAGE_CACHE_SIZE - 1),
 729                                        PAGE_CACHE_SIZE);
 730        p_offset = p_offset ? roundup(p_offset, len) : PAGE_CACHE_SIZE;
 731        page_dirty = p_offset / len;
 732
 733        bh = head = page_buffers(page);
 734        do {
 735                if (offset >= end_offset)
 736                        break;
 737                if (!buffer_uptodate(bh))
 738                        uptodate = 0;
 739                if (!(PageUptodate(page) || buffer_uptodate(bh))) {
 740                        done = 1;
 741                        continue;
 742                }
 743
 744                if (buffer_unwritten(bh) || buffer_delay(bh) ||
 745                    buffer_mapped(bh)) {
 746                        if (buffer_unwritten(bh))
 747                                type = XFS_IO_UNWRITTEN;
 748                        else if (buffer_delay(bh))
 749                                type = XFS_IO_DELALLOC;
 750                        else
 751                                type = XFS_IO_OVERWRITE;
 752
 753                        if (!xfs_imap_valid(inode, imap, offset)) {
 754                                done = 1;
 755                                continue;
 756                        }
 757
 758                        lock_buffer(bh);
 759                        if (type != XFS_IO_OVERWRITE)
 760                                xfs_map_at_offset(inode, bh, imap, offset);
 761                        xfs_add_to_ioend(inode, bh, offset, type,
 762                                         ioendp, done);
 763
 764                        page_dirty--;
 765                        count++;
 766                } else {
 767                        done = 1;
 768                }
 769        } while (offset += len, (bh = bh->b_this_page) != head);
 770
 771        if (uptodate && bh == head)
 772                SetPageUptodate(page);
 773
 774        if (count) {
 775                if (--wbc->nr_to_write <= 0 &&
 776                    wbc->sync_mode == WB_SYNC_NONE)
 777                        done = 1;
 778        }
 779        xfs_start_page_writeback(page, !page_dirty, count);
 780
 781        return done;
 782 fail_unlock_page:
 783        unlock_page(page);
 784 fail:
 785        return 1;
 786}
 787
 788/*
 789 * Convert & write out a cluster of pages in the same extent as defined
 790 * by mp and following the start page.
 791 */
 792STATIC void
 793xfs_cluster_write(
 794        struct inode            *inode,
 795        pgoff_t                 tindex,
 796        struct xfs_bmbt_irec    *imap,
 797        xfs_ioend_t             **ioendp,
 798        struct writeback_control *wbc,
 799        pgoff_t                 tlast)
 800{
 801        struct pagevec          pvec;
 802        int                     done = 0, i;
 803
 804        pagevec_init(&pvec, 0);
 805        while (!done && tindex <= tlast) {
 806                unsigned len = min_t(pgoff_t, PAGEVEC_SIZE, tlast - tindex + 1);
 807
 808                if (!pagevec_lookup(&pvec, inode->i_mapping, tindex, len))
 809                        break;
 810
 811                for (i = 0; i < pagevec_count(&pvec); i++) {
 812                        done = xfs_convert_page(inode, pvec.pages[i], tindex++,
 813                                        imap, ioendp, wbc);
 814                        if (done)
 815                                break;
 816                }
 817
 818                pagevec_release(&pvec);
 819                cond_resched();
 820        }
 821}
 822
 823STATIC void
 824xfs_vm_invalidatepage(
 825        struct page             *page,
 826        unsigned long           offset)
 827{
 828        trace_xfs_invalidatepage(page->mapping->host, page, offset);
 829        block_invalidatepage(page, offset);
 830}
 831
 832/*
 833 * If the page has delalloc buffers on it, we need to punch them out before we
 834 * invalidate the page. If we don't, we leave a stale delalloc mapping on the
 835 * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read
 836 * is done on that same region - the delalloc extent is returned when none is
 837 * supposed to be there.
 838 *
 839 * We prevent this by truncating away the delalloc regions on the page before
 840 * invalidating it. Because they are delalloc, we can do this without needing a
 841 * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this
 842 * truncation without a transaction as there is no space left for block
 843 * reservation (typically why we see a ENOSPC in writeback).
 844 *
 845 * This is not a performance critical path, so for now just do the punching a
 846 * buffer head at a time.
 847 */
 848STATIC void
 849xfs_aops_discard_page(
 850        struct page             *page)
 851{
 852        struct inode            *inode = page->mapping->host;
 853        struct xfs_inode        *ip = XFS_I(inode);
 854        struct buffer_head      *bh, *head;
 855        loff_t                  offset = page_offset(page);
 856
 857        if (!xfs_check_page_type(page, XFS_IO_DELALLOC))
 858                goto out_invalidate;
 859
 860        if (XFS_FORCED_SHUTDOWN(ip->i_mount))
 861                goto out_invalidate;
 862
 863        xfs_alert(ip->i_mount,
 864                "page discard on page %p, inode 0x%llx, offset %llu.",
 865                        page, ip->i_ino, offset);
 866
 867        xfs_ilock(ip, XFS_ILOCK_EXCL);
 868        bh = head = page_buffers(page);
 869        do {
 870                int             error;
 871                xfs_fileoff_t   start_fsb;
 872
 873                if (!buffer_delay(bh))
 874                        goto next_buffer;
 875
 876                start_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
 877                error = xfs_bmap_punch_delalloc_range(ip, start_fsb, 1);
 878                if (error) {
 879                        /* something screwed, just bail */
 880                        if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
 881                                xfs_alert(ip->i_mount,
 882                        "page discard unable to remove delalloc mapping.");
 883                        }
 884                        break;
 885                }
 886next_buffer:
 887                offset += 1 << inode->i_blkbits;
 888
 889        } while ((bh = bh->b_this_page) != head);
 890
 891        xfs_iunlock(ip, XFS_ILOCK_EXCL);
 892out_invalidate:
 893        xfs_vm_invalidatepage(page, 0);
 894        return;
 895}
 896
 897/*
 898 * Write out a dirty page.
 899 *
 900 * For delalloc space on the page we need to allocate space and flush it.
 901 * For unwritten space on the page we need to start the conversion to
 902 * regular allocated space.
 903 * For any other dirty buffer heads on the page we should flush them.
 904 */
 905STATIC int
 906xfs_vm_writepage(
 907        struct page             *page,
 908        struct writeback_control *wbc)
 909{
 910        struct inode            *inode = page->mapping->host;
 911        struct buffer_head      *bh, *head;
 912        struct xfs_bmbt_irec    imap;
 913        xfs_ioend_t             *ioend = NULL, *iohead = NULL;
 914        loff_t                  offset;
 915        unsigned int            type;
 916        __uint64_t              end_offset;
 917        pgoff_t                 end_index, last_index;
 918        ssize_t                 len;
 919        int                     err, imap_valid = 0, uptodate = 1;
 920        int                     count = 0;
 921        int                     nonblocking = 0;
 922
 923        trace_xfs_writepage(inode, page, 0);
 924
 925        ASSERT(page_has_buffers(page));
 926
 927        /*
 928         * Refuse to write the page out if we are called from reclaim context.
 929         *
 930         * This avoids stack overflows when called from deeply used stacks in
 931         * random callers for direct reclaim or memcg reclaim.  We explicitly
 932         * allow reclaim from kswapd as the stack usage there is relatively low.
 933         *
 934         * This should never happen except in the case of a VM regression so
 935         * warn about it.
 936         */
 937        if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
 938                        PF_MEMALLOC))
 939                goto redirty;
 940
 941        /*
 942         * Given that we do not allow direct reclaim to call us, we should
 943         * never be called while in a filesystem transaction.
 944         */
 945        if (WARN_ON(current->flags & PF_FSTRANS))
 946                goto redirty;
 947
 948        /* Is this page beyond the end of the file? */
 949        offset = i_size_read(inode);
 950        end_index = offset >> PAGE_CACHE_SHIFT;
 951        last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
 952        if (page->index >= end_index) {
 953                unsigned offset_into_page = offset & (PAGE_CACHE_SIZE - 1);
 954
 955                /*
 956                 * Just skip the page if it is fully outside i_size, e.g. due
 957                 * to a truncate operation that is in progress.
 958                 */
 959                if (page->index >= end_index + 1 || offset_into_page == 0) {
 960                        unlock_page(page);
 961                        return 0;
 962                }
 963
 964                /*
 965                 * The page straddles i_size.  It must be zeroed out on each
 966                 * and every writepage invocation because it may be mmapped.
 967                 * "A file is mapped in multiples of the page size.  For a file
 968                 * that is not a multiple of the  page size, the remaining
 969                 * memory is zeroed when mapped, and writes to that region are
 970                 * not written out to the file."
 971                 */
 972                zero_user_segment(page, offset_into_page, PAGE_CACHE_SIZE);
 973        }
 974
 975        end_offset = min_t(unsigned long long,
 976                        (xfs_off_t)(page->index + 1) << PAGE_CACHE_SHIFT,
 977                        offset);
 978        len = 1 << inode->i_blkbits;
 979
 980        bh = head = page_buffers(page);
 981        offset = page_offset(page);
 982        type = XFS_IO_OVERWRITE;
 983
 984        if (wbc->sync_mode == WB_SYNC_NONE)
 985                nonblocking = 1;
 986
 987        do {
 988                int new_ioend = 0;
 989
 990                if (offset >= end_offset)
 991                        break;
 992                if (!buffer_uptodate(bh))
 993                        uptodate = 0;
 994
 995                /*
 996                 * set_page_dirty dirties all buffers in a page, independent
 997                 * of their state.  The dirty state however is entirely
 998                 * meaningless for holes (!mapped && uptodate), so skip
 999                 * buffers covering holes here.
1000                 */
1001                if (!buffer_mapped(bh) && buffer_uptodate(bh)) {
1002                        imap_valid = 0;
1003                        continue;
1004                }
1005
1006                if (buffer_unwritten(bh)) {
1007                        if (type != XFS_IO_UNWRITTEN) {
1008                                type = XFS_IO_UNWRITTEN;
1009                                imap_valid = 0;
1010                        }
1011                } else if (buffer_delay(bh)) {
1012                        if (type != XFS_IO_DELALLOC) {
1013                                type = XFS_IO_DELALLOC;
1014                                imap_valid = 0;
1015                        }
1016                } else if (buffer_uptodate(bh)) {
1017                        if (type != XFS_IO_OVERWRITE) {
1018                                type = XFS_IO_OVERWRITE;
1019                                imap_valid = 0;
1020                        }
1021                } else {
1022                        if (PageUptodate(page))
1023                                ASSERT(buffer_mapped(bh));
1024                        /*
1025                         * This buffer is not uptodate and will not be
1026                         * written to disk.  Ensure that we will put any
1027                         * subsequent writeable buffers into a new
1028                         * ioend.
1029                         */
1030                        imap_valid = 0;
1031                        continue;
1032                }
1033
1034                if (imap_valid)
1035                        imap_valid = xfs_imap_valid(inode, &imap, offset);
1036                if (!imap_valid) {
1037                        /*
1038                         * If we didn't have a valid mapping then we need to
1039                         * put the new mapping into a separate ioend structure.
1040                         * This ensures non-contiguous extents always have
1041                         * separate ioends, which is particularly important
1042                         * for unwritten extent conversion at I/O completion
1043                         * time.
1044                         */
1045                        new_ioend = 1;
1046                        err = xfs_map_blocks(inode, offset, &imap, type,
1047                                             nonblocking);
1048                        if (err)
1049                                goto error;
1050                        imap_valid = xfs_imap_valid(inode, &imap, offset);
1051                }
1052                if (imap_valid) {
1053                        lock_buffer(bh);
1054                        if (type != XFS_IO_OVERWRITE)
1055                                xfs_map_at_offset(inode, bh, &imap, offset);
1056                        xfs_add_to_ioend(inode, bh, offset, type, &ioend,
1057                                         new_ioend);
1058                        count++;
1059                }
1060
1061                if (!iohead)
1062                        iohead = ioend;
1063
1064        } while (offset += len, ((bh = bh->b_this_page) != head));
1065
1066        if (uptodate && bh == head)
1067                SetPageUptodate(page);
1068
1069        xfs_start_page_writeback(page, 1, count);
1070
1071        /* if there is no IO to be submitted for this page, we are done */
1072        if (!ioend)
1073                return 0;
1074
1075        ASSERT(iohead);
1076
1077        /*
1078         * Any errors from this point onwards need tobe reported through the IO
1079         * completion path as we have marked the initial page as under writeback
1080         * and unlocked it.
1081         */
1082        if (imap_valid) {
1083                xfs_off_t               end_index;
1084
1085                end_index = imap.br_startoff + imap.br_blockcount;
1086
1087                /* to bytes */
1088                end_index <<= inode->i_blkbits;
1089
1090                /* to pages */
1091                end_index = (end_index - 1) >> PAGE_CACHE_SHIFT;
1092
1093                /* check against file size */
1094                if (end_index > last_index)
1095                        end_index = last_index;
1096
1097                xfs_cluster_write(inode, page->index + 1, &imap, &ioend,
1098                                  wbc, end_index);
1099        }
1100
1101
1102        /*
1103         * Reserve log space if we might write beyond the on-disk inode size.
1104         */
1105        err = 0;
1106        if (ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend))
1107                err = xfs_setfilesize_trans_alloc(ioend);
1108
1109        xfs_submit_ioend(wbc, iohead, err);
1110
1111        return 0;
1112
1113error:
1114        if (iohead)
1115                xfs_cancel_ioend(iohead);
1116
1117        if (err == -EAGAIN)
1118                goto redirty;
1119
1120        xfs_aops_discard_page(page);
1121        ClearPageUptodate(page);
1122        unlock_page(page);
1123        return err;
1124
1125redirty:
1126        redirty_page_for_writepage(wbc, page);
1127        unlock_page(page);
1128        return 0;
1129}
1130
1131STATIC int
1132xfs_vm_writepages(
1133        struct address_space    *mapping,
1134        struct writeback_control *wbc)
1135{
1136        xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
1137        return generic_writepages(mapping, wbc);
1138}
1139
1140/*
1141 * Called to move a page into cleanable state - and from there
1142 * to be released. The page should already be clean. We always
1143 * have buffer heads in this call.
1144 *
1145 * Returns 1 if the page is ok to release, 0 otherwise.
1146 */
1147STATIC int
1148xfs_vm_releasepage(
1149        struct page             *page,
1150        gfp_t                   gfp_mask)
1151{
1152        int                     delalloc, unwritten;
1153
1154        trace_xfs_releasepage(page->mapping->host, page, 0);
1155
1156        xfs_count_page_state(page, &delalloc, &unwritten);
1157
1158        if (WARN_ON(delalloc))
1159                return 0;
1160        if (WARN_ON(unwritten))
1161                return 0;
1162
1163        return try_to_free_buffers(page);
1164}
1165
1166STATIC int
1167__xfs_get_blocks(
1168        struct inode            *inode,
1169        sector_t                iblock,
1170        struct buffer_head      *bh_result,
1171        int                     create,
1172        int                     direct)
1173{
1174        struct xfs_inode        *ip = XFS_I(inode);
1175        struct xfs_mount        *mp = ip->i_mount;
1176        xfs_fileoff_t           offset_fsb, end_fsb;
1177        int                     error = 0;
1178        int                     lockmode = 0;
1179        struct xfs_bmbt_irec    imap;
1180        int                     nimaps = 1;
1181        xfs_off_t               offset;
1182        ssize_t                 size;
1183        int                     new = 0;
1184
1185        if (XFS_FORCED_SHUTDOWN(mp))
1186                return -XFS_ERROR(EIO);
1187
1188        offset = (xfs_off_t)iblock << inode->i_blkbits;
1189        ASSERT(bh_result->b_size >= (1 << inode->i_blkbits));
1190        size = bh_result->b_size;
1191
1192        if (!create && direct && offset >= i_size_read(inode))
1193                return 0;
1194
1195        /*
1196         * Direct I/O is usually done on preallocated files, so try getting
1197         * a block mapping without an exclusive lock first.  For buffered
1198         * writes we already have the exclusive iolock anyway, so avoiding
1199         * a lock roundtrip here by taking the ilock exclusive from the
1200         * beginning is a useful micro optimization.
1201         */
1202        if (create && !direct) {
1203                lockmode = XFS_ILOCK_EXCL;
1204                xfs_ilock(ip, lockmode);
1205        } else {
1206                lockmode = xfs_ilock_map_shared(ip);
1207        }
1208
1209        ASSERT(offset <= mp->m_super->s_maxbytes);
1210        if (offset + size > mp->m_super->s_maxbytes)
1211                size = mp->m_super->s_maxbytes - offset;
1212        end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + size);
1213        offset_fsb = XFS_B_TO_FSBT(mp, offset);
1214
1215        error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb,
1216                                &imap, &nimaps, XFS_BMAPI_ENTIRE);
1217        if (error)
1218                goto out_unlock;
1219
1220        if (create &&
1221            (!nimaps ||
1222             (imap.br_startblock == HOLESTARTBLOCK ||
1223              imap.br_startblock == DELAYSTARTBLOCK))) {
1224                if (direct || xfs_get_extsz_hint(ip)) {
1225                        /*
1226                         * Drop the ilock in preparation for starting the block
1227                         * allocation transaction.  It will be retaken
1228                         * exclusively inside xfs_iomap_write_direct for the
1229                         * actual allocation.
1230                         */
1231                        xfs_iunlock(ip, lockmode);
1232                        error = xfs_iomap_write_direct(ip, offset, size,
1233                                                       &imap, nimaps);
1234                        if (error)
1235                                return -error;
1236                        new = 1;
1237                } else {
1238                        /*
1239                         * Delalloc reservations do not require a transaction,
1240                         * we can go on without dropping the lock here. If we
1241                         * are allocating a new delalloc block, make sure that
1242                         * we set the new flag so that we mark the buffer new so
1243                         * that we know that it is newly allocated if the write
1244                         * fails.
1245                         */
1246                        if (nimaps && imap.br_startblock == HOLESTARTBLOCK)
1247                                new = 1;
1248                        error = xfs_iomap_write_delay(ip, offset, size, &imap);
1249                        if (error)
1250                                goto out_unlock;
1251
1252                        xfs_iunlock(ip, lockmode);
1253                }
1254
1255                trace_xfs_get_blocks_alloc(ip, offset, size, 0, &imap);
1256        } else if (nimaps) {
1257                trace_xfs_get_blocks_found(ip, offset, size, 0, &imap);
1258                xfs_iunlock(ip, lockmode);
1259        } else {
1260                trace_xfs_get_blocks_notfound(ip, offset, size);
1261                goto out_unlock;
1262        }
1263
1264        if (imap.br_startblock != HOLESTARTBLOCK &&
1265            imap.br_startblock != DELAYSTARTBLOCK) {
1266                /*
1267                 * For unwritten extents do not report a disk address on
1268                 * the read case (treat as if we're reading into a hole).
1269                 */
1270                if (create || !ISUNWRITTEN(&imap))
1271                        xfs_map_buffer(inode, bh_result, &imap, offset);
1272                if (create && ISUNWRITTEN(&imap)) {
1273                        if (direct)
1274                                bh_result->b_private = inode;
1275                        set_buffer_unwritten(bh_result);
1276                }
1277        }
1278
1279        /*
1280         * If this is a realtime file, data may be on a different device.
1281         * to that pointed to from the buffer_head b_bdev currently.
1282         */
1283        bh_result->b_bdev = xfs_find_bdev_for_inode(inode);
1284
1285        /*
1286         * If we previously allocated a block out beyond eof and we are now
1287         * coming back to use it then we will need to flag it as new even if it
1288         * has a disk address.
1289         *
1290         * With sub-block writes into unwritten extents we also need to mark
1291         * the buffer as new so that the unwritten parts of the buffer gets
1292         * correctly zeroed.
1293         */
1294        if (create &&
1295            ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
1296             (offset >= i_size_read(inode)) ||
1297             (new || ISUNWRITTEN(&imap))))
1298                set_buffer_new(bh_result);
1299
1300        if (imap.br_startblock == DELAYSTARTBLOCK) {
1301                BUG_ON(direct);
1302                if (create) {
1303                        set_buffer_uptodate(bh_result);
1304                        set_buffer_mapped(bh_result);
1305                        set_buffer_delay(bh_result);
1306                }
1307        }
1308
1309        /*
1310         * If this is O_DIRECT or the mpage code calling tell them how large
1311         * the mapping is, so that we can avoid repeated get_blocks calls.
1312         */
1313        if (direct || size > (1 << inode->i_blkbits)) {
1314                xfs_off_t               mapping_size;
1315
1316                mapping_size = imap.br_startoff + imap.br_blockcount - iblock;
1317                mapping_size <<= inode->i_blkbits;
1318
1319                ASSERT(mapping_size > 0);
1320                if (mapping_size > size)
1321                        mapping_size = size;
1322                if (mapping_size > LONG_MAX)
1323                        mapping_size = LONG_MAX;
1324
1325                bh_result->b_size = mapping_size;
1326        }
1327
1328        return 0;
1329
1330out_unlock:
1331        xfs_iunlock(ip, lockmode);
1332        return -error;
1333}
1334
1335int
1336xfs_get_blocks(
1337        struct inode            *inode,
1338        sector_t                iblock,
1339        struct buffer_head      *bh_result,
1340        int                     create)
1341{
1342        return __xfs_get_blocks(inode, iblock, bh_result, create, 0);
1343}
1344
1345STATIC int
1346xfs_get_blocks_direct(
1347        struct inode            *inode,
1348        sector_t                iblock,
1349        struct buffer_head      *bh_result,
1350        int                     create)
1351{
1352        return __xfs_get_blocks(inode, iblock, bh_result, create, 1);
1353}
1354
1355/*
1356 * Complete a direct I/O write request.
1357 *
1358 * If the private argument is non-NULL __xfs_get_blocks signals us that we
1359 * need to issue a transaction to convert the range from unwritten to written
1360 * extents.  In case this is regular synchronous I/O we just call xfs_end_io
1361 * to do this and we are done.  But in case this was a successful AIO
1362 * request this handler is called from interrupt context, from which we
1363 * can't start transactions.  In that case offload the I/O completion to
1364 * the workqueues we also use for buffered I/O completion.
1365 */
1366STATIC void
1367xfs_end_io_direct_write(
1368        struct kiocb            *iocb,
1369        loff_t                  offset,
1370        ssize_t                 size,
1371        void                    *private,
1372        int                     ret,
1373        bool                    is_async)
1374{
1375        struct xfs_ioend        *ioend = iocb->private;
1376
1377        /*
1378         * While the generic direct I/O code updates the inode size, it does
1379         * so only after the end_io handler is called, which means our
1380         * end_io handler thinks the on-disk size is outside the in-core
1381         * size.  To prevent this just update it a little bit earlier here.
1382         */
1383        if (offset + size > i_size_read(ioend->io_inode))
1384                i_size_write(ioend->io_inode, offset + size);
1385
1386        /*
1387         * blockdev_direct_IO can return an error even after the I/O
1388         * completion handler was called.  Thus we need to protect
1389         * against double-freeing.
1390         */
1391        iocb->private = NULL;
1392
1393        ioend->io_offset = offset;
1394        ioend->io_size = size;
1395        ioend->io_iocb = iocb;
1396        ioend->io_result = ret;
1397        if (private && size > 0)
1398                ioend->io_type = XFS_IO_UNWRITTEN;
1399
1400        if (is_async) {
1401                ioend->io_isasync = 1;
1402                xfs_finish_ioend(ioend);
1403        } else {
1404                xfs_finish_ioend_sync(ioend);
1405        }
1406}
1407
1408STATIC ssize_t
1409xfs_vm_direct_IO(
1410        int                     rw,
1411        struct kiocb            *iocb,
1412        const struct iovec      *iov,
1413        loff_t                  offset,
1414        unsigned long           nr_segs)
1415{
1416        struct inode            *inode = iocb->ki_filp->f_mapping->host;
1417        struct block_device     *bdev = xfs_find_bdev_for_inode(inode);
1418        struct xfs_ioend        *ioend = NULL;
1419        ssize_t                 ret;
1420
1421        if (rw & WRITE) {
1422                size_t size = iov_length(iov, nr_segs);
1423
1424                /*
1425                 * We cannot preallocate a size update transaction here as we
1426                 * don't know whether allocation is necessary or not. Hence we
1427                 * can only tell IO completion that one is necessary if we are
1428                 * not doing unwritten extent conversion.
1429                 */
1430                iocb->private = ioend = xfs_alloc_ioend(inode, XFS_IO_DIRECT);
1431                if (offset + size > XFS_I(inode)->i_d.di_size)
1432                        ioend->io_isdirect = 1;
1433
1434                ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
1435                                            offset, nr_segs,
1436                                            xfs_get_blocks_direct,
1437                                            xfs_end_io_direct_write, NULL, 0);
1438                if (ret != -EIOCBQUEUED && iocb->private)
1439                        goto out_destroy_ioend;
1440        } else {
1441                ret = __blockdev_direct_IO(rw, iocb, inode, bdev, iov,
1442                                            offset, nr_segs,
1443                                            xfs_get_blocks_direct,
1444                                            NULL, NULL, 0);
1445        }
1446
1447        return ret;
1448
1449out_destroy_ioend:
1450        xfs_destroy_ioend(ioend);
1451        return ret;
1452}
1453
1454/*
1455 * Punch out the delalloc blocks we have already allocated.
1456 *
1457 * Don't bother with xfs_setattr given that nothing can have made it to disk yet
1458 * as the page is still locked at this point.
1459 */
1460STATIC void
1461xfs_vm_kill_delalloc_range(
1462        struct inode            *inode,
1463        loff_t                  start,
1464        loff_t                  end)
1465{
1466        struct xfs_inode        *ip = XFS_I(inode);
1467        xfs_fileoff_t           start_fsb;
1468        xfs_fileoff_t           end_fsb;
1469        int                     error;
1470
1471        start_fsb = XFS_B_TO_FSB(ip->i_mount, start);
1472        end_fsb = XFS_B_TO_FSB(ip->i_mount, end);
1473        if (end_fsb <= start_fsb)
1474                return;
1475
1476        xfs_ilock(ip, XFS_ILOCK_EXCL);
1477        error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
1478                                                end_fsb - start_fsb);
1479        if (error) {
1480                /* something screwed, just bail */
1481                if (!XFS_FORCED_SHUTDOWN(ip->i_mount)) {
1482                        xfs_alert(ip->i_mount,
1483                "xfs_vm_write_failed: unable to clean up ino %lld",
1484                                        ip->i_ino);
1485                }
1486        }
1487        xfs_iunlock(ip, XFS_ILOCK_EXCL);
1488}
1489
1490STATIC void
1491xfs_vm_write_failed(
1492        struct inode            *inode,
1493        struct page             *page,
1494        loff_t                  pos,
1495        unsigned                len)
1496{
1497        loff_t                  block_offset = pos & PAGE_MASK;
1498        loff_t                  block_start;
1499        loff_t                  block_end;
1500        loff_t                  from = pos & (PAGE_CACHE_SIZE - 1);
1501        loff_t                  to = from + len;
1502        struct buffer_head      *bh, *head;
1503
1504        ASSERT(block_offset + from == pos);
1505
1506        head = page_buffers(page);
1507        block_start = 0;
1508        for (bh = head; bh != head || !block_start;
1509             bh = bh->b_this_page, block_start = block_end,
1510                                   block_offset += bh->b_size) {
1511                block_end = block_start + bh->b_size;
1512
1513                /* skip buffers before the write */
1514                if (block_end <= from)
1515                        continue;
1516
1517                /* if the buffer is after the write, we're done */
1518                if (block_start >= to)
1519                        break;
1520
1521                if (!buffer_delay(bh))
1522                        continue;
1523
1524                if (!buffer_new(bh) && block_offset < i_size_read(inode))
1525                        continue;
1526
1527                xfs_vm_kill_delalloc_range(inode, block_offset,
1528                                           block_offset + bh->b_size);
1529        }
1530
1531}
1532
1533/*
1534 * This used to call block_write_begin(), but it unlocks and releases the page
1535 * on error, and we need that page to be able to punch stale delalloc blocks out
1536 * on failure. hence we copy-n-waste it here and call xfs_vm_write_failed() at
1537 * the appropriate point.
1538 */
1539STATIC int
1540xfs_vm_write_begin(
1541        struct file             *file,
1542        struct address_space    *mapping,
1543        loff_t                  pos,
1544        unsigned                len,
1545        unsigned                flags,
1546        struct page             **pagep,
1547        void                    **fsdata)
1548{
1549        pgoff_t                 index = pos >> PAGE_CACHE_SHIFT;
1550        struct page             *page;
1551        int                     status;
1552
1553        ASSERT(len <= PAGE_CACHE_SIZE);
1554
1555        page = grab_cache_page_write_begin(mapping, index,
1556                                           flags | AOP_FLAG_NOFS);
1557        if (!page)
1558                return -ENOMEM;
1559
1560        status = __block_write_begin(page, pos, len, xfs_get_blocks);
1561        if (unlikely(status)) {
1562                struct inode    *inode = mapping->host;
1563
1564                xfs_vm_write_failed(inode, page, pos, len);
1565                unlock_page(page);
1566
1567                if (pos + len > i_size_read(inode))
1568                        truncate_pagecache(inode, pos + len, i_size_read(inode));
1569
1570                page_cache_release(page);
1571                page = NULL;
1572        }
1573
1574        *pagep = page;
1575        return status;
1576}
1577
1578/*
1579 * On failure, we only need to kill delalloc blocks beyond EOF because they
1580 * will never be written. For blocks within EOF, generic_write_end() zeros them
1581 * so they are safe to leave alone and be written with all the other valid data.
1582 */
1583STATIC int
1584xfs_vm_write_end(
1585        struct file             *file,
1586        struct address_space    *mapping,
1587        loff_t                  pos,
1588        unsigned                len,
1589        unsigned                copied,
1590        struct page             *page,
1591        void                    *fsdata)
1592{
1593        int                     ret;
1594
1595        ASSERT(len <= PAGE_CACHE_SIZE);
1596
1597        ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata);
1598        if (unlikely(ret < len)) {
1599                struct inode    *inode = mapping->host;
1600                size_t          isize = i_size_read(inode);
1601                loff_t          to = pos + len;
1602
1603                if (to > isize) {
1604                        truncate_pagecache(inode, to, isize);
1605                        xfs_vm_kill_delalloc_range(inode, isize, to);
1606                }
1607        }
1608        return ret;
1609}
1610
1611STATIC sector_t
1612xfs_vm_bmap(
1613        struct address_space    *mapping,
1614        sector_t                block)
1615{
1616        struct inode            *inode = (struct inode *)mapping->host;
1617        struct xfs_inode        *ip = XFS_I(inode);
1618
1619        trace_xfs_vm_bmap(XFS_I(inode));
1620        xfs_ilock(ip, XFS_IOLOCK_SHARED);
1621        xfs_flush_pages(ip, (xfs_off_t)0, -1, 0, FI_REMAPF);
1622        xfs_iunlock(ip, XFS_IOLOCK_SHARED);
1623        return generic_block_bmap(mapping, block, xfs_get_blocks);
1624}
1625
1626STATIC int
1627xfs_vm_readpage(
1628        struct file             *unused,
1629        struct page             *page)
1630{
1631        return mpage_readpage(page, xfs_get_blocks);
1632}
1633
1634STATIC int
1635xfs_vm_readpages(
1636        struct file             *unused,
1637        struct address_space    *mapping,
1638        struct list_head        *pages,
1639        unsigned                nr_pages)
1640{
1641        return mpage_readpages(mapping, pages, nr_pages, xfs_get_blocks);
1642}
1643
1644const struct address_space_operations xfs_address_space_operations = {
1645        .readpage               = xfs_vm_readpage,
1646        .readpages              = xfs_vm_readpages,
1647        .writepage              = xfs_vm_writepage,
1648        .writepages             = xfs_vm_writepages,
1649        .releasepage            = xfs_vm_releasepage,
1650        .invalidatepage         = xfs_vm_invalidatepage,
1651        .write_begin            = xfs_vm_write_begin,
1652        .write_end              = xfs_vm_write_end,
1653        .bmap                   = xfs_vm_bmap,
1654        .direct_IO              = xfs_vm_direct_IO,
1655        .migratepage            = buffer_migrate_page,
1656        .is_partially_uptodate  = block_is_partially_uptodate,
1657        .error_remove_page      = generic_error_remove_page,
1658};
1659
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.