linux/fs/direct-io.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * fs/direct-io.c
   4 *
   5 * Copyright (C) 2002, Linus Torvalds.
   6 *
   7 * O_DIRECT
   8 *
   9 * 04Jul2002    Andrew Morton
  10 *              Initial version
  11 * 11Sep2002    janetinc@us.ibm.com
  12 *              added readv/writev support.
  13 * 29Oct2002    Andrew Morton
  14 *              rewrote bio_add_page() support.
  15 * 30Oct2002    pbadari@us.ibm.com
  16 *              added support for non-aligned IO.
  17 * 06Nov2002    pbadari@us.ibm.com
  18 *              added asynchronous IO support.
  19 * 21Jul2003    nathans@sgi.com
  20 *              added IO completion notifier.
  21 */
  22
  23#include <linux/kernel.h>
  24#include <linux/module.h>
  25#include <linux/types.h>
  26#include <linux/fs.h>
  27#include <linux/mm.h>
  28#include <linux/slab.h>
  29#include <linux/highmem.h>
  30#include <linux/pagemap.h>
  31#include <linux/task_io_accounting_ops.h>
  32#include <linux/bio.h>
  33#include <linux/wait.h>
  34#include <linux/err.h>
  35#include <linux/blkdev.h>
  36#include <linux/buffer_head.h>
  37#include <linux/rwsem.h>
  38#include <linux/uio.h>
  39#include <linux/atomic.h>
  40#include <linux/prefetch.h>
  41
  42#include "internal.h"
  43
  44/*
  45 * How many user pages to map in one call to get_user_pages().  This determines
  46 * the size of a structure in the slab cache
  47 */
  48#define DIO_PAGES       64
  49
  50/*
  51 * Flags for dio_complete()
  52 */
  53#define DIO_COMPLETE_ASYNC              0x01    /* This is async IO */
  54#define DIO_COMPLETE_INVALIDATE         0x02    /* Can invalidate pages */
  55
  56/*
  57 * This code generally works in units of "dio_blocks".  A dio_block is
  58 * somewhere between the hard sector size and the filesystem block size.  it
  59 * is determined on a per-invocation basis.   When talking to the filesystem
  60 * we need to convert dio_blocks to fs_blocks by scaling the dio_block quantity
  61 * down by dio->blkfactor.  Similarly, fs-blocksize quantities are converted
  62 * to bio_block quantities by shifting left by blkfactor.
  63 *
  64 * If blkfactor is zero then the user's request was aligned to the filesystem's
  65 * blocksize.
  66 */
  67
  68/* dio_state only used in the submission path */
  69
  70struct dio_submit {
  71        struct bio *bio;                /* bio under assembly */
  72        unsigned blkbits;               /* doesn't change */
  73        unsigned blkfactor;             /* When we're using an alignment which
  74                                           is finer than the filesystem's soft
  75                                           blocksize, this specifies how much
  76                                           finer.  blkfactor=2 means 1/4-block
  77                                           alignment.  Does not change */
  78        unsigned start_zero_done;       /* flag: sub-blocksize zeroing has
  79                                           been performed at the start of a
  80                                           write */
  81        int pages_in_io;                /* approximate total IO pages */
  82        sector_t block_in_file;         /* Current offset into the underlying
  83                                           file in dio_block units. */
  84        unsigned blocks_available;      /* At block_in_file.  changes */
  85        int reap_counter;               /* rate limit reaping */
  86        sector_t final_block_in_request;/* doesn't change */
  87        int boundary;                   /* prev block is at a boundary */
  88        get_block_t *get_block;         /* block mapping function */
  89        dio_submit_t *submit_io;        /* IO submition function */
  90
  91        loff_t logical_offset_in_bio;   /* current first logical block in bio */
  92        sector_t final_block_in_bio;    /* current final block in bio + 1 */
  93        sector_t next_block_for_io;     /* next block to be put under IO,
  94                                           in dio_blocks units */
  95
  96        /*
  97         * Deferred addition of a page to the dio.  These variables are
  98         * private to dio_send_cur_page(), submit_page_section() and
  99         * dio_bio_add_page().
 100         */
 101        struct page *cur_page;          /* The page */
 102        unsigned cur_page_offset;       /* Offset into it, in bytes */
 103        unsigned cur_page_len;          /* Nr of bytes at cur_page_offset */
 104        sector_t cur_page_block;        /* Where it starts */
 105        loff_t cur_page_fs_offset;      /* Offset in file */
 106
 107        struct iov_iter *iter;
 108        /*
 109         * Page queue.  These variables belong to dio_refill_pages() and
 110         * dio_get_page().
 111         */
 112        unsigned head;                  /* next page to process */
 113        unsigned tail;                  /* last valid page + 1 */
 114        size_t from, to;
 115};
 116
 117/* dio_state communicated between submission path and end_io */
 118struct dio {
 119        int flags;                      /* doesn't change */
 120        int op;
 121        int op_flags;
 122        blk_qc_t bio_cookie;
 123        struct gendisk *bio_disk;
 124        struct inode *inode;
 125        loff_t i_size;                  /* i_size when submitted */
 126        dio_iodone_t *end_io;           /* IO completion function */
 127
 128        void *private;                  /* copy from map_bh.b_private */
 129
 130        /* BIO completion state */
 131        spinlock_t bio_lock;            /* protects BIO fields below */
 132        int page_errors;                /* errno from get_user_pages() */
 133        int is_async;                   /* is IO async ? */
 134        bool defer_completion;          /* defer AIO completion to workqueue? */
 135        bool should_dirty;              /* if pages should be dirtied */
 136        int io_error;                   /* IO error in completion path */
 137        unsigned long refcount;         /* direct_io_worker() and bios */
 138        struct bio *bio_list;           /* singly linked via bi_private */
 139        struct task_struct *waiter;     /* waiting task (NULL if none) */
 140
 141        /* AIO related stuff */
 142        struct kiocb *iocb;             /* kiocb */
 143        ssize_t result;                 /* IO result */
 144
 145        /*
 146         * pages[] (and any fields placed after it) are not zeroed out at
 147         * allocation time.  Don't add new fields after pages[] unless you
 148         * wish that they not be zeroed.
 149         */
 150        union {
 151                struct page *pages[DIO_PAGES];  /* page buffer */
 152                struct work_struct complete_work;/* deferred AIO completion */
 153        };
 154} ____cacheline_aligned_in_smp;
 155
 156static struct kmem_cache *dio_cache __read_mostly;
 157
 158/*
 159 * How many pages are in the queue?
 160 */
 161static inline unsigned dio_pages_present(struct dio_submit *sdio)
 162{
 163        return sdio->tail - sdio->head;
 164}
 165
 166/*
 167 * Go grab and pin some userspace pages.   Typically we'll get 64 at a time.
 168 */
 169static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)
 170{
 171        ssize_t ret;
 172
 173        ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES,
 174                                &sdio->from);
 175
 176        if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) {
 177                struct page *page = ZERO_PAGE(0);
 178                /*
 179                 * A memory fault, but the filesystem has some outstanding
 180                 * mapped blocks.  We need to use those blocks up to avoid
 181                 * leaking stale data in the file.
 182                 */
 183                if (dio->page_errors == 0)
 184                        dio->page_errors = ret;
 185                get_page(page);
 186                dio->pages[0] = page;
 187                sdio->head = 0;
 188                sdio->tail = 1;
 189                sdio->from = 0;
 190                sdio->to = PAGE_SIZE;
 191                return 0;
 192        }
 193
 194        if (ret >= 0) {
 195                iov_iter_advance(sdio->iter, ret);
 196                ret += sdio->from;
 197                sdio->head = 0;
 198                sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
 199                sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1;
 200                return 0;
 201        }
 202        return ret;     
 203}
 204
 205/*
 206 * Get another userspace page.  Returns an ERR_PTR on error.  Pages are
 207 * buffered inside the dio so that we can call get_user_pages() against a
 208 * decent number of pages, less frequently.  To provide nicer use of the
 209 * L1 cache.
 210 */
 211static inline struct page *dio_get_page(struct dio *dio,
 212                                        struct dio_submit *sdio)
 213{
 214        if (dio_pages_present(sdio) == 0) {
 215                int ret;
 216
 217                ret = dio_refill_pages(dio, sdio);
 218                if (ret)
 219                        return ERR_PTR(ret);
 220                BUG_ON(dio_pages_present(sdio) == 0);
 221        }
 222        return dio->pages[sdio->head];
 223}
 224
 225/*
 226 * dio_complete() - called when all DIO BIO I/O has been completed
 227 *
 228 * This drops i_dio_count, lets interested parties know that a DIO operation
 229 * has completed, and calculates the resulting return code for the operation.
 230 *
 231 * It lets the filesystem know if it registered an interest earlier via
 232 * get_block.  Pass the private field of the map buffer_head so that
 233 * filesystems can use it to hold additional state between get_block calls and
 234 * dio_complete.
 235 */
 236static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags)
 237{
 238        loff_t offset = dio->iocb->ki_pos;
 239        ssize_t transferred = 0;
 240        int err;
 241
 242        /*
 243         * AIO submission can race with bio completion to get here while
 244         * expecting to have the last io completed by bio completion.
 245         * In that case -EIOCBQUEUED is in fact not an error we want
 246         * to preserve through this call.
 247         */
 248        if (ret == -EIOCBQUEUED)
 249                ret = 0;
 250
 251        if (dio->result) {
 252                transferred = dio->result;
 253
 254                /* Check for short read case */
 255                if ((dio->op == REQ_OP_READ) &&
 256                    ((offset + transferred) > dio->i_size))
 257                        transferred = dio->i_size - offset;
 258                /* ignore EFAULT if some IO has been done */
 259                if (unlikely(ret == -EFAULT) && transferred)
 260                        ret = 0;
 261        }
 262
 263        if (ret == 0)
 264                ret = dio->page_errors;
 265        if (ret == 0)
 266                ret = dio->io_error;
 267        if (ret == 0)
 268                ret = transferred;
 269
 270        if (dio->end_io) {
 271                // XXX: ki_pos??
 272                err = dio->end_io(dio->iocb, offset, ret, dio->private);
 273                if (err)
 274                        ret = err;
 275        }
 276
 277        /*
 278         * Try again to invalidate clean pages which might have been cached by
 279         * non-direct readahead, or faulted in by get_user_pages() if the source
 280         * of the write was an mmap'ed region of the file we're writing.  Either
 281         * one is a pretty crazy thing to do, so we don't support it 100%.  If
 282         * this invalidation fails, tough, the write still worked...
 283         *
 284         * And this page cache invalidation has to be after dio->end_io(), as
 285         * some filesystems convert unwritten extents to real allocations in
 286         * end_io() when necessary, otherwise a racing buffer read would cache
 287         * zeros from unwritten extents.
 288         */
 289        if (flags & DIO_COMPLETE_INVALIDATE &&
 290            ret > 0 && dio->op == REQ_OP_WRITE &&
 291            dio->inode->i_mapping->nrpages) {
 292                err = invalidate_inode_pages2_range(dio->inode->i_mapping,
 293                                        offset >> PAGE_SHIFT,
 294                                        (offset + ret - 1) >> PAGE_SHIFT);
 295                if (err)
 296                        dio_warn_stale_pagecache(dio->iocb->ki_filp);
 297        }
 298
 299        inode_dio_end(dio->inode);
 300
 301        if (flags & DIO_COMPLETE_ASYNC) {
 302                /*
 303                 * generic_write_sync expects ki_pos to have been updated
 304                 * already, but the submission path only does this for
 305                 * synchronous I/O.
 306                 */
 307                dio->iocb->ki_pos += transferred;
 308
 309                if (ret > 0 && dio->op == REQ_OP_WRITE)
 310                        ret = generic_write_sync(dio->iocb, ret);
 311                dio->iocb->ki_complete(dio->iocb, ret, 0);
 312        }
 313
 314        kmem_cache_free(dio_cache, dio);
 315        return ret;
 316}
 317
 318static void dio_aio_complete_work(struct work_struct *work)
 319{
 320        struct dio *dio = container_of(work, struct dio, complete_work);
 321
 322        dio_complete(dio, 0, DIO_COMPLETE_ASYNC | DIO_COMPLETE_INVALIDATE);
 323}
 324
 325static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio);
 326
 327/*
 328 * Asynchronous IO callback. 
 329 */
 330static void dio_bio_end_aio(struct bio *bio)
 331{
 332        struct dio *dio = bio->bi_private;
 333        unsigned long remaining;
 334        unsigned long flags;
 335        bool defer_completion = false;
 336
 337        /* cleanup the bio */
 338        dio_bio_complete(dio, bio);
 339
 340        spin_lock_irqsave(&dio->bio_lock, flags);
 341        remaining = --dio->refcount;
 342        if (remaining == 1 && dio->waiter)
 343                wake_up_process(dio->waiter);
 344        spin_unlock_irqrestore(&dio->bio_lock, flags);
 345
 346        if (remaining == 0) {
 347                /*
 348                 * Defer completion when defer_completion is set or
 349                 * when the inode has pages mapped and this is AIO write.
 350                 * We need to invalidate those pages because there is a
 351                 * chance they contain stale data in the case buffered IO
 352                 * went in between AIO submission and completion into the
 353                 * same region.
 354                 */
 355                if (dio->result)
 356                        defer_completion = dio->defer_completion ||
 357                                           (dio->op == REQ_OP_WRITE &&
 358                                            dio->inode->i_mapping->nrpages);
 359                if (defer_completion) {
 360                        INIT_WORK(&dio->complete_work, dio_aio_complete_work);
 361                        queue_work(dio->inode->i_sb->s_dio_done_wq,
 362                                   &dio->complete_work);
 363                } else {
 364                        dio_complete(dio, 0, DIO_COMPLETE_ASYNC);
 365                }
 366        }
 367}
 368
 369/*
 370 * The BIO completion handler simply queues the BIO up for the process-context
 371 * handler.
 372 *
 373 * During I/O bi_private points at the dio.  After I/O, bi_private is used to
 374 * implement a singly-linked list of completed BIOs, at dio->bio_list.
 375 */
 376static void dio_bio_end_io(struct bio *bio)
 377{
 378        struct dio *dio = bio->bi_private;
 379        unsigned long flags;
 380
 381        spin_lock_irqsave(&dio->bio_lock, flags);
 382        bio->bi_private = dio->bio_list;
 383        dio->bio_list = bio;
 384        if (--dio->refcount == 1 && dio->waiter)
 385                wake_up_process(dio->waiter);
 386        spin_unlock_irqrestore(&dio->bio_lock, flags);
 387}
 388
 389static inline void
 390dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
 391              struct block_device *bdev,
 392              sector_t first_sector, int nr_vecs)
 393{
 394        struct bio *bio;
 395
 396        /*
 397         * bio_alloc() is guaranteed to return a bio when allowed to sleep and
 398         * we request a valid number of vectors.
 399         */
 400        bio = bio_alloc(GFP_KERNEL, nr_vecs);
 401
 402        bio_set_dev(bio, bdev);
 403        bio->bi_iter.bi_sector = first_sector;
 404        bio_set_op_attrs(bio, dio->op, dio->op_flags);
 405        if (dio->is_async)
 406                bio->bi_end_io = dio_bio_end_aio;
 407        else
 408                bio->bi_end_io = dio_bio_end_io;
 409
 410        bio->bi_write_hint = dio->iocb->ki_hint;
 411
 412        sdio->bio = bio;
 413        sdio->logical_offset_in_bio = sdio->cur_page_fs_offset;
 414}
 415
 416/*
 417 * In the AIO read case we speculatively dirty the pages before starting IO.
 418 * During IO completion, any of these pages which happen to have been written
 419 * back will be redirtied by bio_check_pages_dirty().
 420 *
 421 * bios hold a dio reference between submit_bio and ->end_io.
 422 */
 423static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
 424{
 425        struct bio *bio = sdio->bio;
 426        unsigned long flags;
 427
 428        bio->bi_private = dio;
 429        /* don't account direct I/O as memory stall */
 430        bio_clear_flag(bio, BIO_WORKINGSET);
 431
 432        spin_lock_irqsave(&dio->bio_lock, flags);
 433        dio->refcount++;
 434        spin_unlock_irqrestore(&dio->bio_lock, flags);
 435
 436        if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty)
 437                bio_set_pages_dirty(bio);
 438
 439        dio->bio_disk = bio->bi_bdev->bd_disk;
 440
 441        if (sdio->submit_io) {
 442                sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio);
 443                dio->bio_cookie = BLK_QC_T_NONE;
 444        } else
 445                dio->bio_cookie = submit_bio(bio);
 446
 447        sdio->bio = NULL;
 448        sdio->boundary = 0;
 449        sdio->logical_offset_in_bio = 0;
 450}
 451
 452/*
 453 * Release any resources in case of a failure
 454 */
 455static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)
 456{
 457        while (sdio->head < sdio->tail)
 458                put_page(dio->pages[sdio->head++]);
 459}
 460
 461/*
 462 * Wait for the next BIO to complete.  Remove it and return it.  NULL is
 463 * returned once all BIOs have been completed.  This must only be called once
 464 * all bios have been issued so that dio->refcount can only decrease.  This
 465 * requires that the caller hold a reference on the dio.
 466 */
 467static struct bio *dio_await_one(struct dio *dio)
 468{
 469        unsigned long flags;
 470        struct bio *bio = NULL;
 471
 472        spin_lock_irqsave(&dio->bio_lock, flags);
 473
 474        /*
 475         * Wait as long as the list is empty and there are bios in flight.  bio
 476         * completion drops the count, maybe adds to the list, and wakes while
 477         * holding the bio_lock so we don't need set_current_state()'s barrier
 478         * and can call it after testing our condition.
 479         */
 480        while (dio->refcount > 1 && dio->bio_list == NULL) {
 481                __set_current_state(TASK_UNINTERRUPTIBLE);
 482                dio->waiter = current;
 483                spin_unlock_irqrestore(&dio->bio_lock, flags);
 484                if (!(dio->iocb->ki_flags & IOCB_HIPRI) ||
 485                    !blk_poll(dio->bio_disk->queue, dio->bio_cookie, true))
 486                        blk_io_schedule();
 487                /* wake up sets us TASK_RUNNING */
 488                spin_lock_irqsave(&dio->bio_lock, flags);
 489                dio->waiter = NULL;
 490        }
 491        if (dio->bio_list) {
 492                bio = dio->bio_list;
 493                dio->bio_list = bio->bi_private;
 494        }
 495        spin_unlock_irqrestore(&dio->bio_lock, flags);
 496        return bio;
 497}
 498
 499/*
 500 * Process one completed BIO.  No locks are held.
 501 */
 502static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)
 503{
 504        blk_status_t err = bio->bi_status;
 505        bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty;
 506
 507        if (err) {
 508                if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT))
 509                        dio->io_error = -EAGAIN;
 510                else
 511                        dio->io_error = -EIO;
 512        }
 513
 514        if (dio->is_async && should_dirty) {
 515                bio_check_pages_dirty(bio);     /* transfers ownership */
 516        } else {
 517                bio_release_pages(bio, should_dirty);
 518                bio_put(bio);
 519        }
 520        return err;
 521}
 522
 523/*
 524 * Wait on and process all in-flight BIOs.  This must only be called once
 525 * all bios have been issued so that the refcount can only decrease.
 526 * This just waits for all bios to make it through dio_bio_complete.  IO
 527 * errors are propagated through dio->io_error and should be propagated via
 528 * dio_complete().
 529 */
 530static void dio_await_completion(struct dio *dio)
 531{
 532        struct bio *bio;
 533        do {
 534                bio = dio_await_one(dio);
 535                if (bio)
 536                        dio_bio_complete(dio, bio);
 537        } while (bio);
 538}
 539
 540/*
 541 * A really large O_DIRECT read or write can generate a lot of BIOs.  So
 542 * to keep the memory consumption sane we periodically reap any completed BIOs
 543 * during the BIO generation phase.
 544 *
 545 * This also helps to limit the peak amount of pinned userspace memory.
 546 */
 547static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)
 548{
 549        int ret = 0;
 550
 551        if (sdio->reap_counter++ >= 64) {
 552                while (dio->bio_list) {
 553                        unsigned long flags;
 554                        struct bio *bio;
 555                        int ret2;
 556
 557                        spin_lock_irqsave(&dio->bio_lock, flags);
 558                        bio = dio->bio_list;
 559                        dio->bio_list = bio->bi_private;
 560                        spin_unlock_irqrestore(&dio->bio_lock, flags);
 561                        ret2 = blk_status_to_errno(dio_bio_complete(dio, bio));
 562                        if (ret == 0)
 563                                ret = ret2;
 564                }
 565                sdio->reap_counter = 0;
 566        }
 567        return ret;
 568}
 569
 570/*
 571 * Create workqueue for deferred direct IO completions. We allocate the
 572 * workqueue when it's first needed. This avoids creating workqueue for
 573 * filesystems that don't need it and also allows us to create the workqueue
 574 * late enough so the we can include s_id in the name of the workqueue.
 575 */
 576int sb_init_dio_done_wq(struct super_block *sb)
 577{
 578        struct workqueue_struct *old;
 579        struct workqueue_struct *wq = alloc_workqueue("dio/%s",
 580                                                      WQ_MEM_RECLAIM, 0,
 581                                                      sb->s_id);
 582        if (!wq)
 583                return -ENOMEM;
 584        /*
 585         * This has to be atomic as more DIOs can race to create the workqueue
 586         */
 587        old = cmpxchg(&sb->s_dio_done_wq, NULL, wq);
 588        /* Someone created workqueue before us? Free ours... */
 589        if (old)
 590                destroy_workqueue(wq);
 591        return 0;
 592}
 593
 594static int dio_set_defer_completion(struct dio *dio)
 595{
 596        struct super_block *sb = dio->inode->i_sb;
 597
 598        if (dio->defer_completion)
 599                return 0;
 600        dio->defer_completion = true;
 601        if (!sb->s_dio_done_wq)
 602                return sb_init_dio_done_wq(sb);
 603        return 0;
 604}
 605
 606/*
 607 * Call into the fs to map some more disk blocks.  We record the current number
 608 * of available blocks at sdio->blocks_available.  These are in units of the
 609 * fs blocksize, i_blocksize(inode).
 610 *
 611 * The fs is allowed to map lots of blocks at once.  If it wants to do that,
 612 * it uses the passed inode-relative block number as the file offset, as usual.
 613 *
 614 * get_block() is passed the number of i_blkbits-sized blocks which direct_io
 615 * has remaining to do.  The fs should not map more than this number of blocks.
 616 *
 617 * If the fs has mapped a lot of blocks, it should populate bh->b_size to
 618 * indicate how much contiguous disk space has been made available at
 619 * bh->b_blocknr.
 620 *
 621 * If *any* of the mapped blocks are new, then the fs must set buffer_new().
 622 * This isn't very efficient...
 623 *
 624 * In the case of filesystem holes: the fs may return an arbitrarily-large
 625 * hole by returning an appropriate value in b_size and by clearing
 626 * buffer_mapped().  However the direct-io code will only process holes one
 627 * block at a time - it will repeatedly call get_block() as it walks the hole.
 628 */
 629static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,
 630                           struct buffer_head *map_bh)
 631{
 632        int ret;
 633        sector_t fs_startblk;   /* Into file, in filesystem-sized blocks */
 634        sector_t fs_endblk;     /* Into file, in filesystem-sized blocks */
 635        unsigned long fs_count; /* Number of filesystem-sized blocks */
 636        int create;
 637        unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor;
 638        loff_t i_size;
 639
 640        /*
 641         * If there was a memory error and we've overwritten all the
 642         * mapped blocks then we can now return that memory error
 643         */
 644        ret = dio->page_errors;
 645        if (ret == 0) {
 646                BUG_ON(sdio->block_in_file >= sdio->final_block_in_request);
 647                fs_startblk = sdio->block_in_file >> sdio->blkfactor;
 648                fs_endblk = (sdio->final_block_in_request - 1) >>
 649                                        sdio->blkfactor;
 650                fs_count = fs_endblk - fs_startblk + 1;
 651
 652                map_bh->b_state = 0;
 653                map_bh->b_size = fs_count << i_blkbits;
 654
 655                /*
 656                 * For writes that could fill holes inside i_size on a
 657                 * DIO_SKIP_HOLES filesystem we forbid block creations: only
 658                 * overwrites are permitted. We will return early to the caller
 659                 * once we see an unmapped buffer head returned, and the caller
 660                 * will fall back to buffered I/O.
 661                 *
 662                 * Otherwise the decision is left to the get_blocks method,
 663                 * which may decide to handle it or also return an unmapped
 664                 * buffer head.
 665                 */
 666                create = dio->op == REQ_OP_WRITE;
 667                if (dio->flags & DIO_SKIP_HOLES) {
 668                        i_size = i_size_read(dio->inode);
 669                        if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits)
 670                                create = 0;
 671                }
 672
 673                ret = (*sdio->get_block)(dio->inode, fs_startblk,
 674                                                map_bh, create);
 675
 676                /* Store for completion */
 677                dio->private = map_bh->b_private;
 678
 679                if (ret == 0 && buffer_defer_completion(map_bh))
 680                        ret = dio_set_defer_completion(dio);
 681        }
 682        return ret;
 683}
 684
 685/*
 686 * There is no bio.  Make one now.
 687 */
 688static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,
 689                sector_t start_sector, struct buffer_head *map_bh)
 690{
 691        sector_t sector;
 692        int ret, nr_pages;
 693
 694        ret = dio_bio_reap(dio, sdio);
 695        if (ret)
 696                goto out;
 697        sector = start_sector << (sdio->blkbits - 9);
 698        nr_pages = bio_max_segs(sdio->pages_in_io);
 699        BUG_ON(nr_pages <= 0);
 700        dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);
 701        sdio->boundary = 0;
 702out:
 703        return ret;
 704}
 705
 706/*
 707 * Attempt to put the current chunk of 'cur_page' into the current BIO.  If
 708 * that was successful then update final_block_in_bio and take a ref against
 709 * the just-added page.
 710 *
 711 * Return zero on success.  Non-zero means the caller needs to start a new BIO.
 712 */
 713static inline int dio_bio_add_page(struct dio_submit *sdio)
 714{
 715        int ret;
 716
 717        ret = bio_add_page(sdio->bio, sdio->cur_page,
 718                        sdio->cur_page_len, sdio->cur_page_offset);
 719        if (ret == sdio->cur_page_len) {
 720                /*
 721                 * Decrement count only, if we are done with this page
 722                 */
 723                if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE)
 724                        sdio->pages_in_io--;
 725                get_page(sdio->cur_page);
 726                sdio->final_block_in_bio = sdio->cur_page_block +
 727                        (sdio->cur_page_len >> sdio->blkbits);
 728                ret = 0;
 729        } else {
 730                ret = 1;
 731        }
 732        return ret;
 733}
 734                
 735/*
 736 * Put cur_page under IO.  The section of cur_page which is described by
 737 * cur_page_offset,cur_page_len is put into a BIO.  The section of cur_page
 738 * starts on-disk at cur_page_block.
 739 *
 740 * We take a ref against the page here (on behalf of its presence in the bio).
 741 *
 742 * The caller of this function is responsible for removing cur_page from the
 743 * dio, and for dropping the refcount which came from that presence.
 744 */
 745static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,
 746                struct buffer_head *map_bh)
 747{
 748        int ret = 0;
 749
 750        if (sdio->bio) {
 751                loff_t cur_offset = sdio->cur_page_fs_offset;
 752                loff_t bio_next_offset = sdio->logical_offset_in_bio +
 753                        sdio->bio->bi_iter.bi_size;
 754
 755                /*
 756                 * See whether this new request is contiguous with the old.
 757                 *
 758                 * Btrfs cannot handle having logically non-contiguous requests
 759                 * submitted.  For example if you have
 760                 *
 761                 * Logical:  [0-4095][HOLE][8192-12287]
 762                 * Physical: [0-4095]      [4096-8191]
 763                 *
 764                 * We cannot submit those pages together as one BIO.  So if our
 765                 * current logical offset in the file does not equal what would
 766                 * be the next logical offset in the bio, submit the bio we
 767                 * have.
 768                 */
 769                if (sdio->final_block_in_bio != sdio->cur_page_block ||
 770                    cur_offset != bio_next_offset)
 771                        dio_bio_submit(dio, sdio);
 772        }
 773
 774        if (sdio->bio == NULL) {
 775                ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
 776                if (ret)
 777                        goto out;
 778        }
 779
 780        if (dio_bio_add_page(sdio) != 0) {
 781                dio_bio_submit(dio, sdio);
 782                ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
 783                if (ret == 0) {
 784                        ret = dio_bio_add_page(sdio);
 785                        BUG_ON(ret != 0);
 786                }
 787        }
 788out:
 789        return ret;
 790}
 791
 792/*
 793 * An autonomous function to put a chunk of a page under deferred IO.
 794 *
 795 * The caller doesn't actually know (or care) whether this piece of page is in
 796 * a BIO, or is under IO or whatever.  We just take care of all possible 
 797 * situations here.  The separation between the logic of do_direct_IO() and
 798 * that of submit_page_section() is important for clarity.  Please don't break.
 799 *
 800 * The chunk of page starts on-disk at blocknr.
 801 *
 802 * We perform deferred IO, by recording the last-submitted page inside our
 803 * private part of the dio structure.  If possible, we just expand the IO
 804 * across that page here.
 805 *
 806 * If that doesn't work out then we put the old page into the bio and add this
 807 * page to the dio instead.
 808 */
 809static inline int
 810submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,
 811                    unsigned offset, unsigned len, sector_t blocknr,
 812                    struct buffer_head *map_bh)
 813{
 814        int ret = 0;
 815        int boundary = sdio->boundary;  /* dio_send_cur_page may clear it */
 816
 817        if (dio->op == REQ_OP_WRITE) {
 818                /*
 819                 * Read accounting is performed in submit_bio()
 820                 */
 821                task_io_account_write(len);
 822        }
 823
 824        /*
 825         * Can we just grow the current page's presence in the dio?
 826         */
 827        if (sdio->cur_page == page &&
 828            sdio->cur_page_offset + sdio->cur_page_len == offset &&
 829            sdio->cur_page_block +
 830            (sdio->cur_page_len >> sdio->blkbits) == blocknr) {
 831                sdio->cur_page_len += len;
 832                goto out;
 833        }
 834
 835        /*
 836         * If there's a deferred page already there then send it.
 837         */
 838        if (sdio->cur_page) {
 839                ret = dio_send_cur_page(dio, sdio, map_bh);
 840                put_page(sdio->cur_page);
 841                sdio->cur_page = NULL;
 842                if (ret)
 843                        return ret;
 844        }
 845
 846        get_page(page);         /* It is in dio */
 847        sdio->cur_page = page;
 848        sdio->cur_page_offset = offset;
 849        sdio->cur_page_len = len;
 850        sdio->cur_page_block = blocknr;
 851        sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits;
 852out:
 853        /*
 854         * If boundary then we want to schedule the IO now to
 855         * avoid metadata seeks.
 856         */
 857        if (boundary) {
 858                ret = dio_send_cur_page(dio, sdio, map_bh);
 859                if (sdio->bio)
 860                        dio_bio_submit(dio, sdio);
 861                put_page(sdio->cur_page);
 862                sdio->cur_page = NULL;
 863        }
 864        return ret;
 865}
 866
 867/*
 868 * If we are not writing the entire block and get_block() allocated
 869 * the block for us, we need to fill-in the unused portion of the
 870 * block with zeros. This happens only if user-buffer, fileoffset or
 871 * io length is not filesystem block-size multiple.
 872 *
 873 * `end' is zero if we're doing the start of the IO, 1 at the end of the
 874 * IO.
 875 */
 876static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,
 877                int end, struct buffer_head *map_bh)
 878{
 879        unsigned dio_blocks_per_fs_block;
 880        unsigned this_chunk_blocks;     /* In dio_blocks */
 881        unsigned this_chunk_bytes;
 882        struct page *page;
 883
 884        sdio->start_zero_done = 1;
 885        if (!sdio->blkfactor || !buffer_new(map_bh))
 886                return;
 887
 888        dio_blocks_per_fs_block = 1 << sdio->blkfactor;
 889        this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1);
 890
 891        if (!this_chunk_blocks)
 892                return;
 893
 894        /*
 895         * We need to zero out part of an fs block.  It is either at the
 896         * beginning or the end of the fs block.
 897         */
 898        if (end) 
 899                this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks;
 900
 901        this_chunk_bytes = this_chunk_blocks << sdio->blkbits;
 902
 903        page = ZERO_PAGE(0);
 904        if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,
 905                                sdio->next_block_for_io, map_bh))
 906                return;
 907
 908        sdio->next_block_for_io += this_chunk_blocks;
 909}
 910
 911/*
 912 * Walk the user pages, and the file, mapping blocks to disk and generating
 913 * a sequence of (page,offset,len,block) mappings.  These mappings are injected
 914 * into submit_page_section(), which takes care of the next stage of submission
 915 *
 916 * Direct IO against a blockdev is different from a file.  Because we can
 917 * happily perform page-sized but 512-byte aligned IOs.  It is important that
 918 * blockdev IO be able to have fine alignment and large sizes.
 919 *
 920 * So what we do is to permit the ->get_block function to populate bh.b_size
 921 * with the size of IO which is permitted at this offset and this i_blkbits.
 922 *
 923 * For best results, the blockdev should be set up with 512-byte i_blkbits and
 924 * it should set b_size to PAGE_SIZE or more inside get_block().  This gives
 925 * fine alignment but still allows this function to work in PAGE_SIZE units.
 926 */
 927static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,
 928                        struct buffer_head *map_bh)
 929{
 930        const unsigned blkbits = sdio->blkbits;
 931        const unsigned i_blkbits = blkbits + sdio->blkfactor;
 932        int ret = 0;
 933
 934        while (sdio->block_in_file < sdio->final_block_in_request) {
 935                struct page *page;
 936                size_t from, to;
 937
 938                page = dio_get_page(dio, sdio);
 939                if (IS_ERR(page)) {
 940                        ret = PTR_ERR(page);
 941                        goto out;
 942                }
 943                from = sdio->head ? 0 : sdio->from;
 944                to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE;
 945                sdio->head++;
 946
 947                while (from < to) {
 948                        unsigned this_chunk_bytes;      /* # of bytes mapped */
 949                        unsigned this_chunk_blocks;     /* # of blocks */
 950                        unsigned u;
 951
 952                        if (sdio->blocks_available == 0) {
 953                                /*
 954                                 * Need to go and map some more disk
 955                                 */
 956                                unsigned long blkmask;
 957                                unsigned long dio_remainder;
 958
 959                                ret = get_more_blocks(dio, sdio, map_bh);
 960                                if (ret) {
 961                                        put_page(page);
 962                                        goto out;
 963                                }
 964                                if (!buffer_mapped(map_bh))
 965                                        goto do_holes;
 966
 967                                sdio->blocks_available =
 968                                                map_bh->b_size >> blkbits;
 969                                sdio->next_block_for_io =
 970                                        map_bh->b_blocknr << sdio->blkfactor;
 971                                if (buffer_new(map_bh)) {
 972                                        clean_bdev_aliases(
 973                                                map_bh->b_bdev,
 974                                                map_bh->b_blocknr,
 975                                                map_bh->b_size >> i_blkbits);
 976                                }
 977
 978                                if (!sdio->blkfactor)
 979                                        goto do_holes;
 980
 981                                blkmask = (1 << sdio->blkfactor) - 1;
 982                                dio_remainder = (sdio->block_in_file & blkmask);
 983
 984                                /*
 985                                 * If we are at the start of IO and that IO
 986                                 * starts partway into a fs-block,
 987                                 * dio_remainder will be non-zero.  If the IO
 988                                 * is a read then we can simply advance the IO
 989                                 * cursor to the first block which is to be
 990                                 * read.  But if the IO is a write and the
 991                                 * block was newly allocated we cannot do that;
 992                                 * the start of the fs block must be zeroed out
 993                                 * on-disk
 994                                 */
 995                                if (!buffer_new(map_bh))
 996                                        sdio->next_block_for_io += dio_remainder;
 997                                sdio->blocks_available -= dio_remainder;
 998                        }
 999do_holes:
1000                        /* Handle holes */
1001                        if (!buffer_mapped(map_bh)) {
1002                                loff_t i_size_aligned;
1003
1004                                /* AKPM: eargh, -ENOTBLK is a hack */
1005                                if (dio->op == REQ_OP_WRITE) {
1006                                        put_page(page);
1007                                        return -ENOTBLK;
1008                                }
1009
1010                                /*
1011                                 * Be sure to account for a partial block as the
1012                                 * last block in the file
1013                                 */
1014                                i_size_aligned = ALIGN(i_size_read(dio->inode),
1015                                                        1 << blkbits);
1016                                if (sdio->block_in_file >=
1017                                                i_size_aligned >> blkbits) {
1018                                        /* We hit eof */
1019                                        put_page(page);
1020                                        goto out;
1021                                }
1022                                zero_user(page, from, 1 << blkbits);
1023                                sdio->block_in_file++;
1024                                from += 1 << blkbits;
1025                                dio->result += 1 << blkbits;
1026                                goto next_block;
1027                        }
1028
1029                        /*
1030                         * If we're performing IO which has an alignment which
1031                         * is finer than the underlying fs, go check to see if
1032                         * we must zero out the start of this block.
1033                         */
1034                        if (unlikely(sdio->blkfactor && !sdio->start_zero_done))
1035                                dio_zero_block(dio, sdio, 0, map_bh);
1036
1037                        /*
1038                         * Work out, in this_chunk_blocks, how much disk we
1039                         * can add to this page
1040                         */
1041                        this_chunk_blocks = sdio->blocks_available;
1042                        u = (to - from) >> blkbits;
1043                        if (this_chunk_blocks > u)
1044                                this_chunk_blocks = u;
1045                        u = sdio->final_block_in_request - sdio->block_in_file;
1046                        if (this_chunk_blocks > u)
1047                                this_chunk_blocks = u;
1048                        this_chunk_bytes = this_chunk_blocks << blkbits;
1049                        BUG_ON(this_chunk_bytes == 0);
1050
1051                        if (this_chunk_blocks == sdio->blocks_available)
1052                                sdio->boundary = buffer_boundary(map_bh);
1053                        ret = submit_page_section(dio, sdio, page,
1054                                                  from,
1055                                                  this_chunk_bytes,
1056                                                  sdio->next_block_for_io,
1057                                                  map_bh);
1058                        if (ret) {
1059                                put_page(page);
1060                                goto out;
1061                        }
1062                        sdio->next_block_for_io += this_chunk_blocks;
1063
1064                        sdio->block_in_file += this_chunk_blocks;
1065                        from += this_chunk_bytes;
1066                        dio->result += this_chunk_bytes;
1067                        sdio->blocks_available -= this_chunk_blocks;
1068next_block:
1069                        BUG_ON(sdio->block_in_file > sdio->final_block_in_request);
1070                        if (sdio->block_in_file == sdio->final_block_in_request)
1071                                break;
1072                }
1073
1074                /* Drop the ref which was taken in get_user_pages() */
1075                put_page(page);
1076        }
1077out:
1078        return ret;
1079}
1080
1081static inline int drop_refcount(struct dio *dio)
1082{
1083        int ret2;
1084        unsigned long flags;
1085
1086        /*
1087         * Sync will always be dropping the final ref and completing the
1088         * operation.  AIO can if it was a broken operation described above or
1089         * in fact if all the bios race to complete before we get here.  In
1090         * that case dio_complete() translates the EIOCBQUEUED into the proper
1091         * return code that the caller will hand to ->complete().
1092         *
1093         * This is managed by the bio_lock instead of being an atomic_t so that
1094         * completion paths can drop their ref and use the remaining count to
1095         * decide to wake the submission path atomically.
1096         */
1097        spin_lock_irqsave(&dio->bio_lock, flags);
1098        ret2 = --dio->refcount;
1099        spin_unlock_irqrestore(&dio->bio_lock, flags);
1100        return ret2;
1101}
1102
1103/*
1104 * This is a library function for use by filesystem drivers.
1105 *
1106 * The locking rules are governed by the flags parameter:
1107 *  - if the flags value contains DIO_LOCKING we use a fancy locking
1108 *    scheme for dumb filesystems.
1109 *    For writes this function is called under i_mutex and returns with
1110 *    i_mutex held, for reads, i_mutex is not held on entry, but it is
1111 *    taken and dropped again before returning.
1112 *  - if the flags value does NOT contain DIO_LOCKING we don't use any
1113 *    internal locking but rather rely on the filesystem to synchronize
1114 *    direct I/O reads/writes versus each other and truncate.
1115 *
1116 * To help with locking against truncate we incremented the i_dio_count
1117 * counter before starting direct I/O, and decrement it once we are done.
1118 * Truncate can wait for it to reach zero to provide exclusion.  It is
1119 * expected that filesystem provide exclusion between new direct I/O
1120 * and truncates.  For DIO_LOCKING filesystems this is done by i_mutex,
1121 * but other filesystems need to take care of this on their own.
1122 *
1123 * NOTE: if you pass "sdio" to anything by pointer make sure that function
1124 * is always inlined. Otherwise gcc is unable to split the structure into
1125 * individual fields and will generate much worse code. This is important
1126 * for the whole file.
1127 */
1128static inline ssize_t
1129do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
1130                      struct block_device *bdev, struct iov_iter *iter,
1131                      get_block_t get_block, dio_iodone_t end_io,
1132                      dio_submit_t submit_io, int flags)
1133{
1134        unsigned i_blkbits = READ_ONCE(inode->i_blkbits);
1135        unsigned blkbits = i_blkbits;
1136        unsigned blocksize_mask = (1 << blkbits) - 1;
1137        ssize_t retval = -EINVAL;
1138        const size_t count = iov_iter_count(iter);
1139        loff_t offset = iocb->ki_pos;
1140        const loff_t end = offset + count;
1141        struct dio *dio;
1142        struct dio_submit sdio = { 0, };
1143        struct buffer_head map_bh = { 0, };
1144        struct blk_plug plug;
1145        unsigned long align = offset | iov_iter_alignment(iter);
1146
1147        /*
1148         * Avoid references to bdev if not absolutely needed to give
1149         * the early prefetch in the caller enough time.
1150         */
1151
1152        /* watch out for a 0 len io from a tricksy fs */
1153        if (iov_iter_rw(iter) == READ && !count)
1154                return 0;
1155
1156        dio = kmem_cache_alloc(dio_cache, GFP_KERNEL);
1157        if (!dio)
1158                return -ENOMEM;
1159        /*
1160         * Believe it or not, zeroing out the page array caused a .5%
1161         * performance regression in a database benchmark.  So, we take
1162         * care to only zero out what's needed.
1163         */
1164        memset(dio, 0, offsetof(struct dio, pages));
1165
1166        dio->flags = flags;
1167        if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) {
1168                /* will be released by direct_io_worker */
1169                inode_lock(inode);
1170        }
1171
1172        /* Once we sampled i_size check for reads beyond EOF */
1173        dio->i_size = i_size_read(inode);
1174        if (iov_iter_rw(iter) == READ && offset >= dio->i_size) {
1175                retval = 0;
1176                goto fail_dio;
1177        }
1178
1179        if (align & blocksize_mask) {
1180                if (bdev)
1181                        blkbits = blksize_bits(bdev_logical_block_size(bdev));
1182                blocksize_mask = (1 << blkbits) - 1;
1183                if (align & blocksize_mask)
1184                        goto fail_dio;
1185        }
1186
1187        if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) {
1188                struct address_space *mapping = iocb->ki_filp->f_mapping;
1189
1190                retval = filemap_write_and_wait_range(mapping, offset, end - 1);
1191                if (retval)
1192                        goto fail_dio;
1193        }
1194
1195        /*
1196         * For file extending writes updating i_size before data writeouts
1197         * complete can expose uninitialized blocks in dumb filesystems.
1198         * In that case we need to wait for I/O completion even if asked
1199         * for an asynchronous write.
1200         */
1201        if (is_sync_kiocb(iocb))
1202                dio->is_async = false;
1203        else if (iov_iter_rw(iter) == WRITE && end > i_size_read(inode))
1204                dio->is_async = false;
1205        else
1206                dio->is_async = true;
1207
1208        dio->inode = inode;
1209        if (iov_iter_rw(iter) == WRITE) {
1210                dio->op = REQ_OP_WRITE;
1211                dio->op_flags = REQ_SYNC | REQ_IDLE;
1212                if (iocb->ki_flags & IOCB_NOWAIT)
1213                        dio->op_flags |= REQ_NOWAIT;
1214        } else {
1215                dio->op = REQ_OP_READ;
1216        }
1217        if (iocb->ki_flags & IOCB_HIPRI)
1218                dio->op_flags |= REQ_HIPRI;
1219
1220        /*
1221         * For AIO O_(D)SYNC writes we need to defer completions to a workqueue
1222         * so that we can call ->fsync.
1223         */
1224        if (dio->is_async && iov_iter_rw(iter) == WRITE) {
1225                retval = 0;
1226                if (iocb->ki_flags & IOCB_DSYNC)
1227                        retval = dio_set_defer_completion(dio);
1228                else if (!dio->inode->i_sb->s_dio_done_wq) {
1229                        /*
1230                         * In case of AIO write racing with buffered read we
1231                         * need to defer completion. We can't decide this now,
1232                         * however the workqueue needs to be initialized here.
1233                         */
1234                        retval = sb_init_dio_done_wq(dio->inode->i_sb);
1235                }
1236                if (retval)
1237                        goto fail_dio;
1238        }
1239
1240        /*
1241         * Will be decremented at I/O completion time.
1242         */
1243        inode_dio_begin(inode);
1244
1245        retval = 0;
1246        sdio.blkbits = blkbits;
1247        sdio.blkfactor = i_blkbits - blkbits;
1248        sdio.block_in_file = offset >> blkbits;
1249
1250        sdio.get_block = get_block;
1251        dio->end_io = end_io;
1252        sdio.submit_io = submit_io;
1253        sdio.final_block_in_bio = -1;
1254        sdio.next_block_for_io = -1;
1255
1256        dio->iocb = iocb;
1257
1258        spin_lock_init(&dio->bio_lock);
1259        dio->refcount = 1;
1260
1261        dio->should_dirty = iter_is_iovec(iter) && iov_iter_rw(iter) == READ;
1262        sdio.iter = iter;
1263        sdio.final_block_in_request = end >> blkbits;
1264
1265        /*
1266         * In case of non-aligned buffers, we may need 2 more
1267         * pages since we need to zero out first and last block.
1268         */
1269        if (unlikely(sdio.blkfactor))
1270                sdio.pages_in_io = 2;
1271
1272        sdio.pages_in_io += iov_iter_npages(iter, INT_MAX);
1273
1274        blk_start_plug(&plug);
1275
1276        retval = do_direct_IO(dio, &sdio, &map_bh);
1277        if (retval)
1278                dio_cleanup(dio, &sdio);
1279
1280        if (retval == -ENOTBLK) {
1281                /*
1282                 * The remaining part of the request will be
1283                 * handled by buffered I/O when we return
1284                 */
1285                retval = 0;
1286        }
1287        /*
1288         * There may be some unwritten disk at the end of a part-written
1289         * fs-block-sized block.  Go zero that now.
1290         */
1291        dio_zero_block(dio, &sdio, 1, &map_bh);
1292
1293        if (sdio.cur_page) {
1294                ssize_t ret2;
1295
1296                ret2 = dio_send_cur_page(dio, &sdio, &map_bh);
1297                if (retval == 0)
1298                        retval = ret2;
1299                put_page(sdio.cur_page);
1300                sdio.cur_page = NULL;
1301        }
1302        if (sdio.bio)
1303                dio_bio_submit(dio, &sdio);
1304
1305        blk_finish_plug(&plug);
1306
1307        /*
1308         * It is possible that, we return short IO due to end of file.
1309         * In that case, we need to release all the pages we got hold on.
1310         */
1311        dio_cleanup(dio, &sdio);
1312
1313        /*
1314         * All block lookups have been performed. For READ requests
1315         * we can let i_mutex go now that its achieved its purpose
1316         * of protecting us from looking up uninitialized blocks.
1317         */
1318        if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING))
1319                inode_unlock(dio->inode);
1320
1321        /*
1322         * The only time we want to leave bios in flight is when a successful
1323         * partial aio read or full aio write have been setup.  In that case
1324         * bio completion will call aio_complete.  The only time it's safe to
1325         * call aio_complete is when we return -EIOCBQUEUED, so we key on that.
1326         * This had *better* be the only place that raises -EIOCBQUEUED.
1327         */
1328        BUG_ON(retval == -EIOCBQUEUED);
1329        if (dio->is_async && retval == 0 && dio->result &&
1330            (iov_iter_rw(iter) == READ || dio->result == count))
1331                retval = -EIOCBQUEUED;
1332        else
1333                dio_await_completion(dio);
1334
1335        if (drop_refcount(dio) == 0) {
1336                retval = dio_complete(dio, retval, DIO_COMPLETE_INVALIDATE);
1337        } else
1338                BUG_ON(retval != -EIOCBQUEUED);
1339
1340        return retval;
1341
1342fail_dio:
1343        if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ)
1344                inode_unlock(inode);
1345
1346        kmem_cache_free(dio_cache, dio);
1347        return retval;
1348}
1349
1350ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
1351                             struct block_device *bdev, struct iov_iter *iter,
1352                             get_block_t get_block,
1353                             dio_iodone_t end_io, dio_submit_t submit_io,
1354                             int flags)
1355{
1356        /*
1357         * The block device state is needed in the end to finally
1358         * submit everything.  Since it's likely to be cache cold
1359         * prefetch it here as first thing to hide some of the
1360         * latency.
1361         *
1362         * Attempt to prefetch the pieces we likely need later.
1363         */
1364        prefetch(&bdev->bd_disk->part_tbl);
1365        prefetch(bdev->bd_disk->queue);
1366        prefetch((char *)bdev->bd_disk->queue + SMP_CACHE_BYTES);
1367
1368        return do_blockdev_direct_IO(iocb, inode, bdev, iter, get_block,
1369                                     end_io, submit_io, flags);
1370}
1371
1372EXPORT_SYMBOL(__blockdev_direct_IO);
1373
1374static __init int dio_init(void)
1375{
1376        dio_cache = KMEM_CACHE(dio, SLAB_PANIC);
1377        return 0;
1378}
1379module_init(dio_init)
1380
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.