linux/fs/bio.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License version 2 as
   6 * published by the Free Software Foundation.
   7 *
   8 * This program is distributed in the hope that it will be useful,
   9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 * GNU General Public License for more details.
  12 *
  13 * You should have received a copy of the GNU General Public Licens
  14 * along with this program; if not, write to the Free Software
  15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-
  16 *
  17 */
  18#include <linux/mm.h>
  19#include <linux/swap.h>
  20#include <linux/bio.h>
  21#include <linux/blkdev.h>
  22#include <linux/uio.h>
  23#include <linux/iocontext.h>
  24#include <linux/slab.h>
  25#include <linux/init.h>
  26#include <linux/kernel.h>
  27#include <linux/export.h>
  28#include <linux/mempool.h>
  29#include <linux/workqueue.h>
  30#include <linux/cgroup.h>
  31#include <scsi/sg.h>            /* for struct sg_iovec */
  32
  33#include <trace/events/block.h>
  34
  35/*
  36 * Test patch to inline a certain number of bi_io_vec's inside the bio
  37 * itself, to shrink a bio data allocation from two mempool calls to one
  38 */
  39#define BIO_INLINE_VECS         4
  40
  41static mempool_t *bio_split_pool __read_mostly;
  42
  43/*
  44 * if you change this list, also change bvec_alloc or things will
  45 * break badly! cannot be bigger than what you can fit into an
  46 * unsigned short
  47 */
  48#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
  49static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
  50        BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
  51};
  52#undef BV
  53
  54/*
  55 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
  56 * IO code that does not need private memory pools.
  57 */
  58struct bio_set *fs_bio_set;
  59EXPORT_SYMBOL(fs_bio_set);
  60
  61/*
  62 * Our slab pool management
  63 */
  64struct bio_slab {
  65        struct kmem_cache *slab;
  66        unsigned int slab_ref;
  67        unsigned int slab_size;
  68        char name[8];
  69};
  70static DEFINE_MUTEX(bio_slab_lock);
  71static struct bio_slab *bio_slabs;
  72static unsigned int bio_slab_nr, bio_slab_max;
  73
  74static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
  75{
  76        unsigned int sz = sizeof(struct bio) + extra_size;
  77        struct kmem_cache *slab = NULL;
  78        struct bio_slab *bslab, *new_bio_slabs;
  79        unsigned int new_bio_slab_max;
  80        unsigned int i, entry = -1;
  81
  82        mutex_lock(&bio_slab_lock);
  83
  84        i = 0;
  85        while (i < bio_slab_nr) {
  86                bslab = &bio_slabs[i];
  87
  88                if (!bslab->slab && entry == -1)
  89                        entry = i;
  90                else if (bslab->slab_size == sz) {
  91                        slab = bslab->slab;
  92                        bslab->slab_ref++;
  93                        break;
  94                }
  95                i++;
  96        }
  97
  98        if (slab)
  99                goto out_unlock;
 100
 101        if (bio_slab_nr == bio_slab_max && entry == -1) {
 102                new_bio_slab_max = bio_slab_max << 1;
 103                new_bio_slabs = krealloc(bio_slabs,
 104                                         new_bio_slab_max * sizeof(struct bio_slab),
 105                                         GFP_KERNEL);
 106                if (!new_bio_slabs)
 107                        goto out_unlock;
 108                bio_slab_max = new_bio_slab_max;
 109                bio_slabs = new_bio_slabs;
 110        }
 111        if (entry == -1)
 112                entry = bio_slab_nr++;
 113
 114        bslab = &bio_slabs[entry];
 115
 116        snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
 117        slab = kmem_cache_create(bslab->name, sz, 0, SLAB_HWCACHE_ALIGN, NULL);
 118        if (!slab)
 119                goto out_unlock;
 120
 121        printk(KERN_INFO "bio: create slab <%s> at %d\n", bslab->name, entry);
 122        bslab->slab = slab;
 123        bslab->slab_ref = 1;
 124        bslab->slab_size = sz;
 125out_unlock:
 126        mutex_unlock(&bio_slab_lock);
 127        return slab;
 128}
 129
 130static void bio_put_slab(struct bio_set *bs)
 131{
 132        struct bio_slab *bslab = NULL;
 133        unsigned int i;
 134
 135        mutex_lock(&bio_slab_lock);
 136
 137        for (i = 0; i < bio_slab_nr; i++) {
 138                if (bs->bio_slab == bio_slabs[i].slab) {
 139                        bslab = &bio_slabs[i];
 140                        break;
 141                }
 142        }
 143
 144        if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
 145                goto out;
 146
 147        WARN_ON(!bslab->slab_ref);
 148
 149        if (--bslab->slab_ref)
 150                goto out;
 151
 152        kmem_cache_destroy(bslab->slab);
 153        bslab->slab = NULL;
 154
 155out:
 156        mutex_unlock(&bio_slab_lock);
 157}
 158
 159unsigned int bvec_nr_vecs(unsigned short idx)
 160{
 161        return bvec_slabs[idx].nr_vecs;
 162}
 163
 164void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
 165{
 166        BIO_BUG_ON(idx >= BIOVEC_NR_POOLS);
 167
 168        if (idx == BIOVEC_MAX_IDX)
 169                mempool_free(bv, pool);
 170        else {
 171                struct biovec_slab *bvs = bvec_slabs + idx;
 172
 173                kmem_cache_free(bvs->slab, bv);
 174        }
 175}
 176
 177struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
 178                           mempool_t *pool)
 179{
 180        struct bio_vec *bvl;
 181
 182        /*
 183         * see comment near bvec_array define!
 184         */
 185        switch (nr) {
 186        case 1:
 187                *idx = 0;
 188                break;
 189        case 2 ... 4:
 190                *idx = 1;
 191                break;
 192        case 5 ... 16:
 193                *idx = 2;
 194                break;
 195        case 17 ... 64:
 196                *idx = 3;
 197                break;
 198        case 65 ... 128:
 199                *idx = 4;
 200                break;
 201        case 129 ... BIO_MAX_PAGES:
 202                *idx = 5;
 203                break;
 204        default:
 205                return NULL;
 206        }
 207
 208        /*
 209         * idx now points to the pool we want to allocate from. only the
 210         * 1-vec entry pool is mempool backed.
 211         */
 212        if (*idx == BIOVEC_MAX_IDX) {
 213fallback:
 214                bvl = mempool_alloc(pool, gfp_mask);
 215        } else {
 216                struct biovec_slab *bvs = bvec_slabs + *idx;
 217                gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO);
 218
 219                /*
 220                 * Make this allocation restricted and don't dump info on
 221                 * allocation failures, since we'll fallback to the mempool
 222                 * in case of failure.
 223                 */
 224                __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
 225
 226                /*
 227                 * Try a slab allocation. If this fails and __GFP_WAIT
 228                 * is set, retry with the 1-entry mempool
 229                 */
 230                bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
 231                if (unlikely(!bvl && (gfp_mask & __GFP_WAIT))) {
 232                        *idx = BIOVEC_MAX_IDX;
 233                        goto fallback;
 234                }
 235        }
 236
 237        return bvl;
 238}
 239
 240static void __bio_free(struct bio *bio)
 241{
 242        bio_disassociate_task(bio);
 243
 244        if (bio_integrity(bio))
 245                bio_integrity_free(bio);
 246}
 247
 248static void bio_free(struct bio *bio)
 249{
 250        struct bio_set *bs = bio->bi_pool;
 251        void *p;
 252
 253        __bio_free(bio);
 254
 255        if (bs) {
 256                if (bio_flagged(bio, BIO_OWNS_VEC))
 257                        bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio));
 258
 259                /*
 260                 * If we have front padding, adjust the bio pointer before freeing
 261                 */
 262                p = bio;
 263                p -= bs->front_pad;
 264
 265                mempool_free(p, bs->bio_pool);
 266        } else {
 267                /* Bio was allocated by bio_kmalloc() */
 268                kfree(bio);
 269        }
 270}
 271
 272void bio_init(struct bio *bio)
 273{
 274        memset(bio, 0, sizeof(*bio));
 275        bio->bi_flags = 1 << BIO_UPTODATE;
 276        atomic_set(&bio->bi_cnt, 1);
 277}
 278EXPORT_SYMBOL(bio_init);
 279
 280/**
 281 * bio_reset - reinitialize a bio
 282 * @bio:        bio to reset
 283 *
 284 * Description:
 285 *   After calling bio_reset(), @bio will be in the same state as a freshly
 286 *   allocated bio returned bio bio_alloc_bioset() - the only fields that are
 287 *   preserved are the ones that are initialized by bio_alloc_bioset(). See
 288 *   comment in struct bio.
 289 */
 290void bio_reset(struct bio *bio)
 291{
 292        unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
 293
 294        __bio_free(bio);
 295
 296        memset(bio, 0, BIO_RESET_BYTES);
 297        bio->bi_flags = flags|(1 << BIO_UPTODATE);
 298}
 299EXPORT_SYMBOL(bio_reset);
 300
 301static void bio_alloc_rescue(struct work_struct *work)
 302{
 303        struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
 304        struct bio *bio;
 305
 306        while (1) {
 307                spin_lock(&bs->rescue_lock);
 308                bio = bio_list_pop(&bs->rescue_list);
 309                spin_unlock(&bs->rescue_lock);
 310
 311                if (!bio)
 312                        break;
 313
 314                generic_make_request(bio);
 315        }
 316}
 317
 318static void punt_bios_to_rescuer(struct bio_set *bs)
 319{
 320        struct bio_list punt, nopunt;
 321        struct bio *bio;
 322
 323        /*
 324         * In order to guarantee forward progress we must punt only bios that
 325         * were allocated from this bio_set; otherwise, if there was a bio on
 326         * there for a stacking driver higher up in the stack, processing it
 327         * could require allocating bios from this bio_set, and doing that from
 328         * our own rescuer would be bad.
 329         *
 330         * Since bio lists are singly linked, pop them all instead of trying to
 331         * remove from the middle of the list:
 332         */
 333
 334        bio_list_init(&punt);
 335        bio_list_init(&nopunt);
 336
 337        while ((bio = bio_list_pop(current->bio_list)))
 338                bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
 339
 340        *current->bio_list = nopunt;
 341
 342        spin_lock(&bs->rescue_lock);
 343        bio_list_merge(&bs->rescue_list, &punt);
 344        spin_unlock(&bs->rescue_lock);
 345
 346        queue_work(bs->rescue_workqueue, &bs->rescue_work);
 347}
 348
 349/**
 350 * bio_alloc_bioset - allocate a bio for I/O
 351 * @gfp_mask:   the GFP_ mask given to the slab allocator
 352 * @nr_iovecs:  number of iovecs to pre-allocate
 353 * @bs:         the bio_set to allocate from.
 354 *
 355 * Description:
 356 *   If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
 357 *   backed by the @bs's mempool.
 358 *
 359 *   When @bs is not NULL, if %__GFP_WAIT is set then bio_alloc will always be
 360 *   able to allocate a bio. This is due to the mempool guarantees. To make this
 361 *   work, callers must never allocate more than 1 bio at a time from this pool.
 362 *   Callers that need to allocate more than 1 bio must always submit the
 363 *   previously allocated bio for IO before attempting to allocate a new one.
 364 *   Failure to do so can cause deadlocks under memory pressure.
 365 *
 366 *   Note that when running under generic_make_request() (i.e. any block
 367 *   driver), bios are not submitted until after you return - see the code in
 368 *   generic_make_request() that converts recursion into iteration, to prevent
 369 *   stack overflows.
 370 *
 371 *   This would normally mean allocating multiple bios under
 372 *   generic_make_request() would be susceptible to deadlocks, but we have
 373 *   deadlock avoidance code that resubmits any blocked bios from a rescuer
 374 *   thread.
 375 *
 376 *   However, we do not guarantee forward progress for allocations from other
 377 *   mempools. Doing multiple allocations from the same mempool under
 378 *   generic_make_request() should be avoided - instead, use bio_set's front_pad
 379 *   for per bio allocations.
 380 *
 381 *   RETURNS:
 382 *   Pointer to new bio on success, NULL on failure.
 383 */
 384struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
 385{
 386        gfp_t saved_gfp = gfp_mask;
 387        unsigned front_pad;
 388        unsigned inline_vecs;
 389        unsigned long idx = BIO_POOL_NONE;
 390        struct bio_vec *bvl = NULL;
 391        struct bio *bio;
 392        void *p;
 393
 394        if (!bs) {
 395                if (nr_iovecs > UIO_MAXIOV)
 396                        return NULL;
 397
 398                p = kmalloc(sizeof(struct bio) +
 399                            nr_iovecs * sizeof(struct bio_vec),
 400                            gfp_mask);
 401                front_pad = 0;
 402                inline_vecs = nr_iovecs;
 403        } else {
 404                /*
 405                 * generic_make_request() converts recursion to iteration; this
 406                 * means if we're running beneath it, any bios we allocate and
 407                 * submit will not be submitted (and thus freed) until after we
 408                 * return.
 409                 *
 410                 * This exposes us to a potential deadlock if we allocate
 411                 * multiple bios from the same bio_set() while running
 412                 * underneath generic_make_request(). If we were to allocate
 413                 * multiple bios (say a stacking block driver that was splitting
 414                 * bios), we would deadlock if we exhausted the mempool's
 415                 * reserve.
 416                 *
 417                 * We solve this, and guarantee forward progress, with a rescuer
 418                 * workqueue per bio_set. If we go to allocate and there are
 419                 * bios on current->bio_list, we first try the allocation
 420                 * without __GFP_WAIT; if that fails, we punt those bios we
 421                 * would be blocking to the rescuer workqueue before we retry
 422                 * with the original gfp_flags.
 423                 */
 424
 425                if (current->bio_list && !bio_list_empty(current->bio_list))
 426                        gfp_mask &= ~__GFP_WAIT;
 427
 428                p = mempool_alloc(bs->bio_pool, gfp_mask);
 429                if (!p && gfp_mask != saved_gfp) {
 430                        punt_bios_to_rescuer(bs);
 431                        gfp_mask = saved_gfp;
 432                        p = mempool_alloc(bs->bio_pool, gfp_mask);
 433                }
 434
 435                front_pad = bs->front_pad;
 436                inline_vecs = BIO_INLINE_VECS;
 437        }
 438
 439        if (unlikely(!p))
 440                return NULL;
 441
 442        bio = p + front_pad;
 443        bio_init(bio);
 444
 445        if (nr_iovecs > inline_vecs) {
 446                bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
 447                if (!bvl && gfp_mask != saved_gfp) {
 448                        punt_bios_to_rescuer(bs);
 449                        gfp_mask = saved_gfp;
 450                        bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
 451                }
 452
 453                if (unlikely(!bvl))
 454                        goto err_free;
 455
 456                bio->bi_flags |= 1 << BIO_OWNS_VEC;
 457        } else if (nr_iovecs) {
 458                bvl = bio->bi_inline_vecs;
 459        }
 460
 461        bio->bi_pool = bs;
 462        bio->bi_flags |= idx << BIO_POOL_OFFSET;
 463        bio->bi_max_vecs = nr_iovecs;
 464        bio->bi_io_vec = bvl;
 465        return bio;
 466
 467err_free:
 468        mempool_free(p, bs->bio_pool);
 469        return NULL;
 470}
 471EXPORT_SYMBOL(bio_alloc_bioset);
 472
 473void zero_fill_bio(struct bio *bio)
 474{
 475        unsigned long flags;
 476        struct bio_vec *bv;
 477        int i;
 478
 479        bio_for_each_segment(bv, bio, i) {
 480                char *data = bvec_kmap_irq(bv, &flags);
 481                memset(data, 0, bv->bv_len);
 482                flush_dcache_page(bv->bv_page);
 483                bvec_kunmap_irq(data, &flags);
 484        }
 485}
 486EXPORT_SYMBOL(zero_fill_bio);
 487
 488/**
 489 * bio_put - release a reference to a bio
 490 * @bio:   bio to release reference to
 491 *
 492 * Description:
 493 *   Put a reference to a &struct bio, either one you have gotten with
 494 *   bio_alloc, bio_get or bio_clone. The last put of a bio will free it.
 495 **/
 496void bio_put(struct bio *bio)
 497{
 498        BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
 499
 500        /*
 501         * last put frees it
 502         */
 503        if (atomic_dec_and_test(&bio->bi_cnt))
 504                bio_free(bio);
 505}
 506EXPORT_SYMBOL(bio_put);
 507
 508inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
 509{
 510        if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
 511                blk_recount_segments(q, bio);
 512
 513        return bio->bi_phys_segments;
 514}
 515EXPORT_SYMBOL(bio_phys_segments);
 516
 517/**
 518 *      __bio_clone     -       clone a bio
 519 *      @bio: destination bio
 520 *      @bio_src: bio to clone
 521 *
 522 *      Clone a &bio. Caller will own the returned bio, but not
 523 *      the actual data it points to. Reference count of returned
 524 *      bio will be one.
 525 */
 526void __bio_clone(struct bio *bio, struct bio *bio_src)
 527{
 528        memcpy(bio->bi_io_vec, bio_src->bi_io_vec,
 529                bio_src->bi_max_vecs * sizeof(struct bio_vec));
 530
 531        /*
 532         * most users will be overriding ->bi_bdev with a new target,
 533         * so we don't set nor calculate new physical/hw segment counts here
 534         */
 535        bio->bi_sector = bio_src->bi_sector;
 536        bio->bi_bdev = bio_src->bi_bdev;
 537        bio->bi_flags |= 1 << BIO_CLONED;
 538        bio->bi_rw = bio_src->bi_rw;
 539        bio->bi_vcnt = bio_src->bi_vcnt;
 540        bio->bi_size = bio_src->bi_size;
 541        bio->bi_idx = bio_src->bi_idx;
 542}
 543EXPORT_SYMBOL(__bio_clone);
 544
 545/**
 546 *      bio_clone_bioset -      clone a bio
 547 *      @bio: bio to clone
 548 *      @gfp_mask: allocation priority
 549 *      @bs: bio_set to allocate from
 550 *
 551 *      Like __bio_clone, only also allocates the returned bio
 552 */
 553struct bio *bio_clone_bioset(struct bio *bio, gfp_t gfp_mask,
 554                             struct bio_set *bs)
 555{
 556        struct bio *b;
 557
 558        b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, bs);
 559        if (!b)
 560                return NULL;
 561
 562        __bio_clone(b, bio);
 563
 564        if (bio_integrity(bio)) {
 565                int ret;
 566
 567                ret = bio_integrity_clone(b, bio, gfp_mask);
 568
 569                if (ret < 0) {
 570                        bio_put(b);
 571                        return NULL;
 572                }
 573        }
 574
 575        return b;
 576}
 577EXPORT_SYMBOL(bio_clone_bioset);
 578
 579/**
 580 *      bio_get_nr_vecs         - return approx number of vecs
 581 *      @bdev:  I/O target
 582 *
 583 *      Return the approximate number of pages we can send to this target.
 584 *      There's no guarantee that you will be able to fit this number of pages
 585 *      into a bio, it does not account for dynamic restrictions that vary
 586 *      on offset.
 587 */
 588int bio_get_nr_vecs(struct block_device *bdev)
 589{
 590        struct request_queue *q = bdev_get_queue(bdev);
 591        int nr_pages;
 592
 593        nr_pages = min_t(unsigned,
 594                     queue_max_segments(q),
 595                     queue_max_sectors(q) / (PAGE_SIZE >> 9) + 1);
 596
 597        return min_t(unsigned, nr_pages, BIO_MAX_PAGES);
 598
 599}
 600EXPORT_SYMBOL(bio_get_nr_vecs);
 601
 602static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
 603                          *page, unsigned int len, unsigned int offset,
 604                          unsigned short max_sectors)
 605{
 606        int retried_segments = 0;
 607        struct bio_vec *bvec;
 608
 609        /*
 610         * cloned bio must not modify vec list
 611         */
 612        if (unlikely(bio_flagged(bio, BIO_CLONED)))
 613                return 0;
 614
 615        if (((bio->bi_size + len) >> 9) > max_sectors)
 616                return 0;
 617
 618        /*
 619         * For filesystems with a blocksize smaller than the pagesize
 620         * we will often be called with the same page as last time and
 621         * a consecutive offset.  Optimize this special case.
 622         */
 623        if (bio->bi_vcnt > 0) {
 624                struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
 625
 626                if (page == prev->bv_page &&
 627                    offset == prev->bv_offset + prev->bv_len) {
 628                        unsigned int prev_bv_len = prev->bv_len;
 629                        prev->bv_len += len;
 630
 631                        if (q->merge_bvec_fn) {
 632                                struct bvec_merge_data bvm = {
 633                                        /* prev_bvec is already charged in
 634                                           bi_size, discharge it in order to
 635                                           simulate merging updated prev_bvec
 636                                           as new bvec. */
 637                                        .bi_bdev = bio->bi_bdev,
 638                                        .bi_sector = bio->bi_sector,
 639                                        .bi_size = bio->bi_size - prev_bv_len,
 640                                        .bi_rw = bio->bi_rw,
 641                                };
 642
 643                                if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) {
 644                                        prev->bv_len -= len;
 645                                        return 0;
 646                                }
 647                        }
 648
 649                        goto done;
 650                }
 651        }
 652
 653        if (bio->bi_vcnt >= bio->bi_max_vecs)
 654                return 0;
 655
 656        /*
 657         * we might lose a segment or two here, but rather that than
 658         * make this too complex.
 659         */
 660
 661        while (bio->bi_phys_segments >= queue_max_segments(q)) {
 662
 663                if (retried_segments)
 664                        return 0;
 665
 666                retried_segments = 1;
 667                blk_recount_segments(q, bio);
 668        }
 669
 670        /*
 671         * setup the new entry, we might clear it again later if we
 672         * cannot add the page
 673         */
 674        bvec = &bio->bi_io_vec[bio->bi_vcnt];
 675        bvec->bv_page = page;
 676        bvec->bv_len = len;
 677        bvec->bv_offset = offset;
 678
 679        /*
 680         * if queue has other restrictions (eg varying max sector size
 681         * depending on offset), it can specify a merge_bvec_fn in the
 682         * queue to get further control
 683         */
 684        if (q->merge_bvec_fn) {
 685                struct bvec_merge_data bvm = {
 686                        .bi_bdev = bio->bi_bdev,
 687                        .bi_sector = bio->bi_sector,
 688                        .bi_size = bio->bi_size,
 689                        .bi_rw = bio->bi_rw,
 690                };
 691
 692                /*
 693                 * merge_bvec_fn() returns number of bytes it can accept
 694                 * at this offset
 695                 */
 696                if (q->merge_bvec_fn(q, &bvm, bvec) < bvec->bv_len) {
 697                        bvec->bv_page = NULL;
 698                        bvec->bv_len = 0;
 699                        bvec->bv_offset = 0;
 700                        return 0;
 701                }
 702        }
 703
 704        /* If we may be able to merge these biovecs, force a recount */
 705        if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
 706                bio->bi_flags &= ~(1 << BIO_SEG_VALID);
 707
 708        bio->bi_vcnt++;
 709        bio->bi_phys_segments++;
 710 done:
 711        bio->bi_size += len;
 712        return len;
 713}
 714
 715/**
 716 *      bio_add_pc_page -       attempt to add page to bio
 717 *      @q: the target queue
 718 *      @bio: destination bio
 719 *      @page: page to add
 720 *      @len: vec entry length
 721 *      @offset: vec entry offset
 722 *
 723 *      Attempt to add a page to the bio_vec maplist. This can fail for a
 724 *      number of reasons, such as the bio being full or target block device
 725 *      limitations. The target block device must allow bio's up to PAGE_SIZE,
 726 *      so it is always possible to add a single page to an empty bio.
 727 *
 728 *      This should only be used by REQ_PC bios.
 729 */
 730int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
 731                    unsigned int len, unsigned int offset)
 732{
 733        return __bio_add_page(q, bio, page, len, offset,
 734                              queue_max_hw_sectors(q));
 735}
 736EXPORT_SYMBOL(bio_add_pc_page);
 737
 738/**
 739 *      bio_add_page    -       attempt to add page to bio
 740 *      @bio: destination bio
 741 *      @page: page to add
 742 *      @len: vec entry length
 743 *      @offset: vec entry offset
 744 *
 745 *      Attempt to add a page to the bio_vec maplist. This can fail for a
 746 *      number of reasons, such as the bio being full or target block device
 747 *      limitations. The target block device must allow bio's up to PAGE_SIZE,
 748 *      so it is always possible to add a single page to an empty bio.
 749 */
 750int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
 751                 unsigned int offset)
 752{
 753        struct request_queue *q = bdev_get_queue(bio->bi_bdev);
 754        return __bio_add_page(q, bio, page, len, offset, queue_max_sectors(q));
 755}
 756EXPORT_SYMBOL(bio_add_page);
 757
 758struct submit_bio_ret {
 759        struct completion event;
 760        int error;
 761};
 762
 763static void submit_bio_wait_endio(struct bio *bio, int error)
 764{
 765        struct submit_bio_ret *ret = bio->bi_private;
 766
 767        ret->error = error;
 768        complete(&ret->event);
 769}
 770
 771/**
 772 * submit_bio_wait - submit a bio, and wait until it completes
 773 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
 774 * @bio: The &struct bio which describes the I/O
 775 *
 776 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
 777 * bio_endio() on failure.
 778 */
 779int submit_bio_wait(int rw, struct bio *bio)
 780{
 781        struct submit_bio_ret ret;
 782
 783        rw |= REQ_SYNC;
 784        init_completion(&ret.event);
 785        bio->bi_private = &ret;
 786        bio->bi_end_io = submit_bio_wait_endio;
 787        submit_bio(rw, bio);
 788        wait_for_completion(&ret.event);
 789
 790        return ret.error;
 791}
 792EXPORT_SYMBOL(submit_bio_wait);
 793
 794/**
 795 * bio_advance - increment/complete a bio by some number of bytes
 796 * @bio:        bio to advance
 797 * @bytes:      number of bytes to complete
 798 *
 799 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
 800 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
 801 * be updated on the last bvec as well.
 802 *
 803 * @bio will then represent the remaining, uncompleted portion of the io.
 804 */
 805void bio_advance(struct bio *bio, unsigned bytes)
 806{
 807        if (bio_integrity(bio))
 808                bio_integrity_advance(bio, bytes);
 809
 810        bio->bi_sector += bytes >> 9;
 811        bio->bi_size -= bytes;
 812
 813        if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK)
 814                return;
 815
 816        while (bytes) {
 817                if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
 818                        WARN_ONCE(1, "bio idx %d >= vcnt %d\n",
 819                                  bio->bi_idx, bio->bi_vcnt);
 820                        break;
 821                }
 822
 823                if (bytes >= bio_iovec(bio)->bv_len) {
 824                        bytes -= bio_iovec(bio)->bv_len;
 825                        bio->bi_idx++;
 826                } else {
 827                        bio_iovec(bio)->bv_len -= bytes;
 828                        bio_iovec(bio)->bv_offset += bytes;
 829                        bytes = 0;
 830                }
 831        }
 832}
 833EXPORT_SYMBOL(bio_advance);
 834
 835/**
 836 * bio_alloc_pages - allocates a single page for each bvec in a bio
 837 * @bio: bio to allocate pages for
 838 * @gfp_mask: flags for allocation
 839 *
 840 * Allocates pages up to @bio->bi_vcnt.
 841 *
 842 * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
 843 * freed.
 844 */
 845int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
 846{
 847        int i;
 848        struct bio_vec *bv;
 849
 850        bio_for_each_segment_all(bv, bio, i) {
 851                bv->bv_page = alloc_page(gfp_mask);
 852                if (!bv->bv_page) {
 853                        while (--bv >= bio->bi_io_vec)
 854                                __free_page(bv->bv_page);
 855                        return -ENOMEM;
 856                }
 857        }
 858
 859        return 0;
 860}
 861EXPORT_SYMBOL(bio_alloc_pages);
 862
 863/**
 864 * bio_copy_data - copy contents of data buffers from one chain of bios to
 865 * another
 866 * @src: source bio list
 867 * @dst: destination bio list
 868 *
 869 * If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
 870 * @src and @dst as linked lists of bios.
 871 *
 872 * Stops when it reaches the end of either @src or @dst - that is, copies
 873 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
 874 */
 875void bio_copy_data(struct bio *dst, struct bio *src)
 876{
 877        struct bio_vec *src_bv, *dst_bv;
 878        unsigned src_offset, dst_offset, bytes;
 879        void *src_p, *dst_p;
 880
 881        src_bv = bio_iovec(src);
 882        dst_bv = bio_iovec(dst);
 883
 884        src_offset = src_bv->bv_offset;
 885        dst_offset = dst_bv->bv_offset;
 886
 887        while (1) {
 888                if (src_offset == src_bv->bv_offset + src_bv->bv_len) {
 889                        src_bv++;
 890                        if (src_bv == bio_iovec_idx(src, src->bi_vcnt)) {
 891                                src = src->bi_next;
 892                                if (!src)
 893                                        break;
 894
 895                                src_bv = bio_iovec(src);
 896                        }
 897
 898                        src_offset = src_bv->bv_offset;
 899                }
 900
 901                if (dst_offset == dst_bv->bv_offset + dst_bv->bv_len) {
 902                        dst_bv++;
 903                        if (dst_bv == bio_iovec_idx(dst, dst->bi_vcnt)) {
 904                                dst = dst->bi_next;
 905                                if (!dst)
 906                                        break;
 907
 908                                dst_bv = bio_iovec(dst);
 909                        }
 910
 911                        dst_offset = dst_bv->bv_offset;
 912                }
 913
 914                bytes = min(dst_bv->bv_offset + dst_bv->bv_len - dst_offset,
 915                            src_bv->bv_offset + src_bv->bv_len - src_offset);
 916
 917                src_p = kmap_atomic(src_bv->bv_page);
 918                dst_p = kmap_atomic(dst_bv->bv_page);
 919
 920                memcpy(dst_p + dst_bv->bv_offset,
 921                       src_p + src_bv->bv_offset,
 922                       bytes);
 923
 924                kunmap_atomic(dst_p);
 925                kunmap_atomic(src_p);
 926
 927                src_offset += bytes;
 928                dst_offset += bytes;
 929        }
 930}
 931EXPORT_SYMBOL(bio_copy_data);
 932
 933struct bio_map_data {
 934        struct bio_vec *iovecs;
 935        struct sg_iovec *sgvecs;
 936        int nr_sgvecs;
 937        int is_our_pages;
 938};
 939
 940static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
 941                             struct sg_iovec *iov, int iov_count,
 942                             int is_our_pages)
 943{
 944        memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt);
 945        memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
 946        bmd->nr_sgvecs = iov_count;
 947        bmd->is_our_pages = is_our_pages;
 948        bio->bi_private = bmd;
 949}
 950
 951static void bio_free_map_data(struct bio_map_data *bmd)
 952{
 953        kfree(bmd->iovecs);
 954        kfree(bmd->sgvecs);
 955        kfree(bmd);
 956}
 957
 958static struct bio_map_data *bio_alloc_map_data(int nr_segs,
 959                                               unsigned int iov_count,
 960                                               gfp_t gfp_mask)
 961{
 962        struct bio_map_data *bmd;
 963
 964        if (iov_count > UIO_MAXIOV)
 965                return NULL;
 966
 967        bmd = kmalloc(sizeof(*bmd), gfp_mask);
 968        if (!bmd)
 969                return NULL;
 970
 971        bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, gfp_mask);
 972        if (!bmd->iovecs) {
 973                kfree(bmd);
 974                return NULL;
 975        }
 976
 977        bmd->sgvecs = kmalloc(sizeof(struct sg_iovec) * iov_count, gfp_mask);
 978        if (bmd->sgvecs)
 979                return bmd;
 980
 981        kfree(bmd->iovecs);
 982        kfree(bmd);
 983        return NULL;
 984}
 985
 986static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs,
 987                          struct sg_iovec *iov, int iov_count,
 988                          int to_user, int from_user, int do_free_page)
 989{
 990        int ret = 0, i;
 991        struct bio_vec *bvec;
 992        int iov_idx = 0;
 993        unsigned int iov_off = 0;
 994
 995        bio_for_each_segment_all(bvec, bio, i) {
 996                char *bv_addr = page_address(bvec->bv_page);
 997                unsigned int bv_len = iovecs[i].bv_len;
 998
 999                while (bv_len && iov_idx < iov_count) {
1000                        unsigned int bytes;
1001                        char __user *iov_addr;
1002
1003                        bytes = min_t(unsigned int,
1004                                      iov[iov_idx].iov_len - iov_off, bv_len);
1005                        iov_addr = iov[iov_idx].iov_base + iov_off;
1006
1007                        if (!ret) {
1008                                if (to_user)
1009                                        ret = copy_to_user(iov_addr, bv_addr,
1010                                                           bytes);
1011
1012                                if (from_user)
1013                                        ret = copy_from_user(bv_addr, iov_addr,
1014                                                             bytes);
1015
1016                                if (ret)
1017                                        ret = -EFAULT;
1018                        }
1019
1020                        bv_len -= bytes;
1021                        bv_addr += bytes;
1022                        iov_addr += bytes;
1023                        iov_off += bytes;
1024
1025                        if (iov[iov_idx].iov_len == iov_off) {
1026                                iov_idx++;
1027                                iov_off = 0;
1028                        }
1029                }
1030
1031                if (do_free_page)
1032                        __free_page(bvec->bv_page);
1033        }
1034
1035        return ret;
1036}
1037
1038/**
1039 *      bio_uncopy_user -       finish previously mapped bio
1040 *      @bio: bio being terminated
1041 *
1042 *      Free pages allocated from bio_copy_user() and write back data
1043 *      to user space in case of a read.
1044 */
1045int bio_uncopy_user(struct bio *bio)
1046{
1047        struct bio_map_data *bmd = bio->bi_private;
1048        int ret = 0;
1049
1050        if (!bio_flagged(bio, BIO_NULL_MAPPED))
1051                ret = __bio_copy_iov(bio, bmd->iovecs, bmd->sgvecs,
1052                                     bmd->nr_sgvecs, bio_data_dir(bio) == READ,
1053                                     0, bmd->is_our_pages);
1054        bio_free_map_data(bmd);
1055        bio_put(bio);
1056        return ret;
1057}
1058EXPORT_SYMBOL(bio_uncopy_user);
1059
1060/**
1061 *      bio_copy_user_iov       -       copy user data to bio
1062 *      @q: destination block queue
1063 *      @map_data: pointer to the rq_map_data holding pages (if necessary)
1064 *      @iov:   the iovec.
1065 *      @iov_count: number of elements in the iovec
1066 *      @write_to_vm: bool indicating writing to pages or not
1067 *      @gfp_mask: memory allocation flags
1068 *
1069 *      Prepares and returns a bio for indirect user io, bouncing data
1070 *      to/from kernel pages as necessary. Must be paired with
1071 *      call bio_uncopy_user() on io completion.
1072 */
1073struct bio *bio_copy_user_iov(struct request_queue *q,
1074                              struct rq_map_data *map_data,
1075                              struct sg_iovec *iov, int iov_count,
1076                              int write_to_vm, gfp_t gfp_mask)
1077{
1078        struct bio_map_data *bmd;
1079        struct bio_vec *bvec;
1080        struct page *page;
1081        struct bio *bio;
1082        int i, ret;
1083        int nr_pages = 0;
1084        unsigned int len = 0;
1085        unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0;
1086
1087        for (i = 0; i < iov_count; i++) {
1088                unsigned long uaddr;
1089                unsigned long end;
1090                unsigned long start;
1091
1092                uaddr = (unsigned long)iov[i].iov_base;
1093                end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1094                start = uaddr >> PAGE_SHIFT;
1095
1096                /*
1097                 * Overflow, abort
1098                 */
1099                if (end < start)
1100                        return ERR_PTR(-EINVAL);
1101
1102                nr_pages += end - start;
1103                len += iov[i].iov_len;
1104        }
1105
1106        if (offset)
1107                nr_pages++;
1108
1109        bmd = bio_alloc_map_data(nr_pages, iov_count, gfp_mask);
1110        if (!bmd)
1111                return ERR_PTR(-ENOMEM);
1112
1113        ret = -ENOMEM;
1114        bio = bio_kmalloc(gfp_mask, nr_pages);
1115        if (!bio)
1116                goto out_bmd;
1117
1118        if (!write_to_vm)
1119                bio->bi_rw |= REQ_WRITE;
1120
1121        ret = 0;
1122
1123        if (map_data) {
1124                nr_pages = 1 << map_data->page_order;
1125                i = map_data->offset / PAGE_SIZE;
1126        }
1127        while (len) {
1128                unsigned int bytes = PAGE_SIZE;
1129
1130                bytes -= offset;
1131
1132                if (bytes > len)
1133                        bytes = len;
1134
1135                if (map_data) {
1136                        if (i == map_data->nr_entries * nr_pages) {
1137                                ret = -ENOMEM;
1138                                break;
1139                        }
1140
1141                        page = map_data->pages[i / nr_pages];
1142                        page += (i % nr_pages);
1143
1144                        i++;
1145                } else {
1146                        page = alloc_page(q->bounce_gfp | gfp_mask);
1147                        if (!page) {
1148                                ret = -ENOMEM;
1149                                break;
1150                        }
1151                }
1152
1153                if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes)
1154                        break;
1155
1156                len -= bytes;
1157                offset = 0;
1158        }
1159
1160        if (ret)
1161                goto cleanup;
1162
1163        /*
1164         * success
1165         */
1166        if ((!write_to_vm && (!map_data || !map_data->null_mapped)) ||
1167            (map_data && map_data->from_user)) {
1168                ret = __bio_copy_iov(bio, bio->bi_io_vec, iov, iov_count, 0, 1, 0);
1169                if (ret)
1170                        goto cleanup;
1171        }
1172
1173        bio_set_map_data(bmd, bio, iov, iov_count, map_data ? 0 : 1);
1174        return bio;
1175cleanup:
1176        if (!map_data)
1177                bio_for_each_segment_all(bvec, bio, i)
1178                        __free_page(bvec->bv_page);
1179
1180        bio_put(bio);
1181out_bmd:
1182        bio_free_map_data(bmd);
1183        return ERR_PTR(ret);
1184}
1185
1186/**
1187 *      bio_copy_user   -       copy user data to bio
1188 *      @q: destination block queue
1189 *      @map_data: pointer to the rq_map_data holding pages (if necessary)
1190 *      @uaddr: start of user address
1191 *      @len: length in bytes
1192 *      @write_to_vm: bool indicating writing to pages or not
1193 *      @gfp_mask: memory allocation flags
1194 *
1195 *      Prepares and returns a bio for indirect user io, bouncing data
1196 *      to/from kernel pages as necessary. Must be paired with
1197 *      call bio_uncopy_user() on io completion.
1198 */
1199struct bio *bio_copy_user(struct request_queue *q, struct rq_map_data *map_data,
1200                          unsigned long uaddr, unsigned int len,
1201                          int write_to_vm, gfp_t gfp_mask)
1202{
1203        struct sg_iovec iov;
1204
1205        iov.iov_base = (void __user *)uaddr;
1206        iov.iov_len = len;
1207
1208        return bio_copy_user_iov(q, map_data, &iov, 1, write_to_vm, gfp_mask);
1209}
1210EXPORT_SYMBOL(bio_copy_user);
1211
1212static struct bio *__bio_map_user_iov(struct request_queue *q,
1213                                      struct block_device *bdev,
1214                                      struct sg_iovec *iov, int iov_count,
1215                                      int write_to_vm, gfp_t gfp_mask)
1216{
1217        int i, j;
1218        int nr_pages = 0;
1219        struct page **pages;
1220        struct bio *bio;
1221        int cur_page = 0;
1222        int ret, offset;
1223
1224        for (i = 0; i < iov_count; i++) {
1225                unsigned long uaddr = (unsigned long)iov[i].iov_base;
1226                unsigned long len = iov[i].iov_len;
1227                unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1228                unsigned long start = uaddr >> PAGE_SHIFT;
1229
1230                /*
1231                 * Overflow, abort
1232                 */
1233                if (end < start)
1234                        return ERR_PTR(-EINVAL);
1235
1236                nr_pages += end - start;
1237                /*
1238                 * buffer must be aligned to at least hardsector size for now
1239                 */
1240                if (uaddr & queue_dma_alignment(q))
1241                        return ERR_PTR(-EINVAL);
1242        }
1243
1244        if (!nr_pages)
1245                return ERR_PTR(-EINVAL);
1246
1247        bio = bio_kmalloc(gfp_mask, nr_pages);
1248        if (!bio)
1249                return ERR_PTR(-ENOMEM);
1250
1251        ret = -ENOMEM;
1252        pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
1253        if (!pages)
1254                goto out;
1255
1256        for (i = 0; i < iov_count; i++) {
1257                unsigned long uaddr = (unsigned long)iov[i].iov_base;
1258                unsigned long len = iov[i].iov_len;
1259                unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1260                unsigned long start = uaddr >> PAGE_SHIFT;
1261                const int local_nr_pages = end - start;
1262                const int page_limit = cur_page + local_nr_pages;
1263
1264                ret = get_user_pages_fast(uaddr, local_nr_pages,
1265                                write_to_vm, &pages[cur_page]);
1266                if (ret < local_nr_pages) {
1267                        ret = -EFAULT;
1268                        goto out_unmap;
1269                }
1270
1271                offset = uaddr & ~PAGE_MASK;
1272                for (j = cur_page; j < page_limit; j++) {
1273                        unsigned int bytes = PAGE_SIZE - offset;
1274
1275                        if (len <= 0)
1276                                break;
1277                        
1278                        if (bytes > len)
1279                                bytes = len;
1280
1281                        /*
1282                         * sorry...
1283                         */
1284                        if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
1285                                            bytes)
1286                                break;
1287
1288                        len -= bytes;
1289                        offset = 0;
1290                }
1291
1292                cur_page = j;
1293                /*
1294                 * release the pages we didn't map into the bio, if any
1295                 */
1296                while (j < page_limit)
1297                        page_cache_release(pages[j++]);
1298        }
1299
1300        kfree(pages);
1301
1302        /*
1303         * set data direction, and check if mapped pages need bouncing
1304         */
1305        if (!write_to_vm)
1306                bio->bi_rw |= REQ_WRITE;
1307
1308        bio->bi_bdev = bdev;
1309        bio->bi_flags |= (1 << BIO_USER_MAPPED);
1310        return bio;
1311
1312 out_unmap:
1313        for (i = 0; i < nr_pages; i++) {
1314                if(!pages[i])
1315                        break;
1316                page_cache_release(pages[i]);
1317        }
1318 out:
1319        kfree(pages);
1320        bio_put(bio);
1321        return ERR_PTR(ret);
1322}
1323
1324/**
1325 *      bio_map_user    -       map user address into bio
1326 *      @q: the struct request_queue for the bio
1327 *      @bdev: destination block device
1328 *      @uaddr: start of user address
1329 *      @len: length in bytes
1330 *      @write_to_vm: bool indicating writing to pages or not
1331 *      @gfp_mask: memory allocation flags
1332 *
1333 *      Map the user space address into a bio suitable for io to a block
1334 *      device. Returns an error pointer in case of error.
1335 */
1336struct bio *bio_map_user(struct request_queue *q, struct block_device *bdev,
1337                         unsigned long uaddr, unsigned int len, int write_to_vm,
1338                         gfp_t gfp_mask)
1339{
1340        struct sg_iovec iov;
1341
1342        iov.iov_base = (void __user *)uaddr;
1343        iov.iov_len = len;
1344
1345        return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm, gfp_mask);
1346}
1347EXPORT_SYMBOL(bio_map_user);
1348
1349/**
1350 *      bio_map_user_iov - map user sg_iovec table into bio
1351 *      @q: the struct request_queue for the bio
1352 *      @bdev: destination block device
1353 *      @iov:   the iovec.
1354 *      @iov_count: number of elements in the iovec
1355 *      @write_to_vm: bool indicating writing to pages or not
1356 *      @gfp_mask: memory allocation flags
1357 *
1358 *      Map the user space address into a bio suitable for io to a block
1359 *      device. Returns an error pointer in case of error.
1360 */
1361struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
1362                             struct sg_iovec *iov, int iov_count,
1363                             int write_to_vm, gfp_t gfp_mask)
1364{
1365        struct bio *bio;
1366
1367        bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm,
1368                                 gfp_mask);
1369        if (IS_ERR(bio))
1370                return bio;
1371
1372        /*
1373         * subtle -- if __bio_map_user() ended up bouncing a bio,
1374         * it would normally disappear when its bi_end_io is run.
1375         * however, we need it for the unmap, so grab an extra
1376         * reference to it
1377         */
1378        bio_get(bio);
1379
1380        return bio;
1381}
1382
1383static void __bio_unmap_user(struct bio *bio)
1384{
1385        struct bio_vec *bvec;
1386        int i;
1387
1388        /*
1389         * make sure we dirty pages we wrote to
1390         */
1391        bio_for_each_segment_all(bvec, bio, i) {
1392                if (bio_data_dir(bio) == READ)
1393                        set_page_dirty_lock(bvec->bv_page);
1394
1395                page_cache_release(bvec->bv_page);
1396        }
1397
1398        bio_put(bio);
1399}
1400
1401/**
1402 *      bio_unmap_user  -       unmap a bio
1403 *      @bio:           the bio being unmapped
1404 *
1405 *      Unmap a bio previously mapped by bio_map_user(). Must be called with
1406 *      a process context.
1407 *
1408 *      bio_unmap_user() may sleep.
1409 */
1410void bio_unmap_user(struct bio *bio)
1411{
1412        __bio_unmap_user(bio);
1413        bio_put(bio);
1414}
1415EXPORT_SYMBOL(bio_unmap_user);
1416
1417static void bio_map_kern_endio(struct bio *bio, int err)
1418{
1419        bio_put(bio);
1420}
1421
1422static struct bio *__bio_map_kern(struct request_queue *q, void *data,
1423                                  unsigned int len, gfp_t gfp_mask)
1424{
1425        unsigned long kaddr = (unsigned long)data;
1426        unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1427        unsigned long start = kaddr >> PAGE_SHIFT;
1428        const int nr_pages = end - start;
1429        int offset, i;
1430        struct bio *bio;
1431
1432        bio = bio_kmalloc(gfp_mask, nr_pages);
1433        if (!bio)
1434                return ERR_PTR(-ENOMEM);
1435
1436        offset = offset_in_page(kaddr);
1437        for (i = 0; i < nr_pages; i++) {
1438                unsigned int bytes = PAGE_SIZE - offset;
1439
1440                if (len <= 0)
1441                        break;
1442
1443                if (bytes > len)
1444                        bytes = len;
1445
1446                if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
1447                                    offset) < bytes)
1448                        break;
1449
1450                data += bytes;
1451                len -= bytes;
1452                offset = 0;
1453        }
1454
1455        bio->bi_end_io = bio_map_kern_endio;
1456        return bio;
1457}
1458
1459/**
1460 *      bio_map_kern    -       map kernel address into bio
1461 *      @q: the struct request_queue for the bio
1462 *      @data: pointer to buffer to map
1463 *      @len: length in bytes
1464 *      @gfp_mask: allocation flags for bio allocation
1465 *
1466 *      Map the kernel address into a bio suitable for io to a block
1467 *      device. Returns an error pointer in case of error.
1468 */
1469struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1470                         gfp_t gfp_mask)
1471{
1472        struct bio *bio;
1473
1474        bio = __bio_map_kern(q, data, len, gfp_mask);
1475        if (IS_ERR(bio))
1476                return bio;
1477
1478        if (bio->bi_size == len)
1479                return bio;
1480
1481        /*
1482         * Don't support partial mappings.
1483         */
1484        bio_put(bio);
1485        return ERR_PTR(-EINVAL);
1486}
1487EXPORT_SYMBOL(bio_map_kern);
1488
1489static void bio_copy_kern_endio(struct bio *bio, int err)
1490{
1491        struct bio_vec *bvec;
1492        const int read = bio_data_dir(bio) == READ;
1493        struct bio_map_data *bmd = bio->bi_private;
1494        int i;
1495        char *p = bmd->sgvecs[0].iov_base;
1496
1497        bio_for_each_segment_all(bvec, bio, i) {
1498                char *addr = page_address(bvec->bv_page);
1499                int len = bmd->iovecs[i].bv_len;
1500
1501                if (read)
1502                        memcpy(p, addr, len);
1503
1504                __free_page(bvec->bv_page);
1505                p += len;
1506        }
1507
1508        bio_free_map_data(bmd);
1509        bio_put(bio);
1510}
1511
1512/**
1513 *      bio_copy_kern   -       copy kernel address into bio
1514 *      @q: the struct request_queue for the bio
1515 *      @data: pointer to buffer to copy
1516 *      @len: length in bytes
1517 *      @gfp_mask: allocation flags for bio and page allocation
1518 *      @reading: data direction is READ
1519 *
1520 *      copy the kernel address into a bio suitable for io to a block
1521 *      device. Returns an error pointer in case of error.
1522 */
1523struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
1524                          gfp_t gfp_mask, int reading)
1525{
1526        struct bio *bio;
1527        struct bio_vec *bvec;
1528        int i;
1529
1530        bio = bio_copy_user(q, NULL, (unsigned long)data, len, 1, gfp_mask);
1531        if (IS_ERR(bio))
1532                return bio;
1533
1534        if (!reading) {
1535                void *p = data;
1536
1537                bio_for_each_segment_all(bvec, bio, i) {
1538                        char *addr = page_address(bvec->bv_page);
1539
1540                        memcpy(addr, p, bvec->bv_len);
1541                        p += bvec->bv_len;
1542                }
1543        }
1544
1545        bio->bi_end_io = bio_copy_kern_endio;
1546
1547        return bio;
1548}
1549EXPORT_SYMBOL(bio_copy_kern);
1550
1551/*
1552 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1553 * for performing direct-IO in BIOs.
1554 *
1555 * The problem is that we cannot run set_page_dirty() from interrupt context
1556 * because the required locks are not interrupt-safe.  So what we can do is to
1557 * mark the pages dirty _before_ performing IO.  And in interrupt context,
1558 * check that the pages are still dirty.   If so, fine.  If not, redirty them
1559 * in process context.
1560 *
1561 * We special-case compound pages here: normally this means reads into hugetlb
1562 * pages.  The logic in here doesn't really work right for compound pages
1563 * because the VM does not uniformly chase down the head page in all cases.
1564 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1565 * handle them at all.  So we skip compound pages here at an early stage.
1566 *
1567 * Note that this code is very hard to test under normal circumstances because
1568 * direct-io pins the pages with get_user_pages().  This makes
1569 * is_page_cache_freeable return false, and the VM will not clean the pages.
1570 * But other code (eg, flusher threads) could clean the pages if they are mapped
1571 * pagecache.
1572 *
1573 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1574 * deferred bio dirtying paths.
1575 */
1576
1577/*
1578 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1579 */
1580void bio_set_pages_dirty(struct bio *bio)
1581{
1582        struct bio_vec *bvec;
1583        int i;
1584
1585        bio_for_each_segment_all(bvec, bio, i) {
1586                struct page *page = bvec->bv_page;
1587
1588                if (page && !PageCompound(page))
1589                        set_page_dirty_lock(page);
1590        }
1591}
1592
1593static void bio_release_pages(struct bio *bio)
1594{
1595        struct bio_vec *bvec;
1596        int i;
1597
1598        bio_for_each_segment_all(bvec, bio, i) {
1599                struct page *page = bvec->bv_page;
1600
1601                if (page)
1602                        put_page(page);
1603        }
1604}
1605
1606/*
1607 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1608 * If they are, then fine.  If, however, some pages are clean then they must
1609 * have been written out during the direct-IO read.  So we take another ref on
1610 * the BIO and the offending pages and re-dirty the pages in process context.
1611 *
1612 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1613 * here on.  It will run one page_cache_release() against each page and will
1614 * run one bio_put() against the BIO.
1615 */
1616
1617static void bio_dirty_fn(struct work_struct *work);
1618
1619static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
1620static DEFINE_SPINLOCK(bio_dirty_lock);
1621static struct bio *bio_dirty_list;
1622
1623/*
1624 * This runs in process context
1625 */
1626static void bio_dirty_fn(struct work_struct *work)
1627{
1628        unsigned long flags;
1629        struct bio *bio;
1630
1631        spin_lock_irqsave(&bio_dirty_lock, flags);
1632        bio = bio_dirty_list;
1633        bio_dirty_list = NULL;
1634        spin_unlock_irqrestore(&bio_dirty_lock, flags);
1635
1636        while (bio) {
1637                struct bio *next = bio->bi_private;
1638
1639                bio_set_pages_dirty(bio);
1640                bio_release_pages(bio);
1641                bio_put(bio);
1642                bio = next;
1643        }
1644}
1645
1646void bio_check_pages_dirty(struct bio *bio)
1647{
1648        struct bio_vec *bvec;
1649        int nr_clean_pages = 0;
1650        int i;
1651
1652        bio_for_each_segment_all(bvec, bio, i) {
1653                struct page *page = bvec->bv_page;
1654
1655                if (PageDirty(page) || PageCompound(page)) {
1656                        page_cache_release(page);
1657                        bvec->bv_page = NULL;
1658                } else {
1659                        nr_clean_pages++;
1660                }
1661        }
1662
1663        if (nr_clean_pages) {
1664                unsigned long flags;
1665
1666                spin_lock_irqsave(&bio_dirty_lock, flags);
1667                bio->bi_private = bio_dirty_list;
1668                bio_dirty_list = bio;
1669                spin_unlock_irqrestore(&bio_dirty_lock, flags);
1670                schedule_work(&bio_dirty_work);
1671        } else {
1672                bio_put(bio);
1673        }
1674}
1675
1676#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1677void bio_flush_dcache_pages(struct bio *bi)
1678{
1679        int i;
1680        struct bio_vec *bvec;
1681
1682        bio_for_each_segment(bvec, bi, i)
1683                flush_dcache_page(bvec->bv_page);
1684}
1685EXPORT_SYMBOL(bio_flush_dcache_pages);
1686#endif
1687
1688/**
1689 * bio_endio - end I/O on a bio
1690 * @bio:        bio
1691 * @error:      error, if any
1692 *
1693 * Description:
1694 *   bio_endio() will end I/O on the whole bio. bio_endio() is the
1695 *   preferred way to end I/O on a bio, it takes care of clearing
1696 *   BIO_UPTODATE on error. @error is 0 on success, and and one of the
1697 *   established -Exxxx (-EIO, for instance) error values in case
1698 *   something went wrong. No one should call bi_end_io() directly on a
1699 *   bio unless they own it and thus know that it has an end_io
1700 *   function.
1701 **/
1702void bio_endio(struct bio *bio, int error)
1703{
1704        if (error)
1705                clear_bit(BIO_UPTODATE, &bio->bi_flags);
1706        else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1707                error = -EIO;
1708
1709        if (bio->bi_end_io)
1710                bio->bi_end_io(bio, error);
1711}
1712EXPORT_SYMBOL(bio_endio);
1713
1714void bio_pair_release(struct bio_pair *bp)
1715{
1716        if (atomic_dec_and_test(&bp->cnt)) {
1717                struct bio *master = bp->bio1.bi_private;
1718
1719                bio_endio(master, bp->error);
1720                mempool_free(bp, bp->bio2.bi_private);
1721        }
1722}
1723EXPORT_SYMBOL(bio_pair_release);
1724
1725static void bio_pair_end_1(struct bio *bi, int err)
1726{
1727        struct bio_pair *bp = container_of(bi, struct bio_pair, bio1);
1728
1729        if (err)
1730                bp->error = err;
1731
1732        bio_pair_release(bp);
1733}
1734
1735static void bio_pair_end_2(struct bio *bi, int err)
1736{
1737        struct bio_pair *bp = container_of(bi, struct bio_pair, bio2);
1738
1739        if (err)
1740                bp->error = err;
1741
1742        bio_pair_release(bp);
1743}
1744
1745/*
1746 * split a bio - only worry about a bio with a single page in its iovec
1747 */
1748struct bio_pair *bio_split(struct bio *bi, int first_sectors)
1749{
1750        struct bio_pair *bp = mempool_alloc(bio_split_pool, GFP_NOIO);
1751
1752        if (!bp)
1753                return bp;
1754
1755        trace_block_split(bdev_get_queue(bi->bi_bdev), bi,
1756                                bi->bi_sector + first_sectors);
1757
1758        BUG_ON(bio_segments(bi) > 1);
1759        atomic_set(&bp->cnt, 3);
1760        bp->error = 0;
1761        bp->bio1 = *bi;
1762        bp->bio2 = *bi;
1763        bp->bio2.bi_sector += first_sectors;
1764        bp->bio2.bi_size -= first_sectors << 9;
1765        bp->bio1.bi_size = first_sectors << 9;
1766
1767        if (bi->bi_vcnt != 0) {
1768                bp->bv1 = *bio_iovec(bi);
1769                bp->bv2 = *bio_iovec(bi);
1770
1771                if (bio_is_rw(bi)) {
1772                        bp->bv2.bv_offset += first_sectors << 9;
1773                        bp->bv2.bv_len -= first_sectors << 9;
1774                        bp->bv1.bv_len = first_sectors << 9;
1775                }
1776
1777                bp->bio1.bi_io_vec = &bp->bv1;
1778                bp->bio2.bi_io_vec = &bp->bv2;
1779
1780                bp->bio1.bi_max_vecs = 1;
1781                bp->bio2.bi_max_vecs = 1;
1782        }
1783
1784        bp->bio1.bi_end_io = bio_pair_end_1;
1785        bp->bio2.bi_end_io = bio_pair_end_2;
1786
1787        bp->bio1.bi_private = bi;
1788        bp->bio2.bi_private = bio_split_pool;
1789
1790        if (bio_integrity(bi))
1791                bio_integrity_split(bi, bp, first_sectors);
1792
1793        return bp;
1794}
1795EXPORT_SYMBOL(bio_split);
1796
1797/**
1798 *      bio_sector_offset - Find hardware sector offset in bio
1799 *      @bio:           bio to inspect
1800 *      @index:         bio_vec index
1801 *      @offset:        offset in bv_page
1802 *
1803 *      Return the number of hardware sectors between beginning of bio
1804 *      and an end point indicated by a bio_vec index and an offset
1805 *      within that vector's page.
1806 */
1807sector_t bio_sector_offset(struct bio *bio, unsigned short index,
1808                           unsigned int offset)
1809{
1810        unsigned int sector_sz;
1811        struct bio_vec *bv;
1812        sector_t sectors;
1813        int i;
1814
1815        sector_sz = queue_logical_block_size(bio->bi_bdev->bd_disk->queue);
1816        sectors = 0;
1817
1818        if (index >= bio->bi_idx)
1819                index = bio->bi_vcnt - 1;
1820
1821        bio_for_each_segment_all(bv, bio, i) {
1822                if (i == index) {
1823                        if (offset > bv->bv_offset)
1824                                sectors += (offset - bv->bv_offset) / sector_sz;
1825                        break;
1826                }
1827
1828                sectors += bv->bv_len / sector_sz;
1829        }
1830
1831        return sectors;
1832}
1833EXPORT_SYMBOL(bio_sector_offset);
1834
1835/*
1836 * create memory pools for biovec's in a bio_set.
1837 * use the global biovec slabs created for general use.
1838 */
1839mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries)
1840{
1841        struct biovec_slab *bp = bvec_slabs + BIOVEC_MAX_IDX;
1842
1843        return mempool_create_slab_pool(pool_entries, bp->slab);
1844}
1845
1846void bioset_free(struct bio_set *bs)
1847{
1848        if (bs->rescue_workqueue)
1849                destroy_workqueue(bs->rescue_workqueue);
1850
1851        if (bs->bio_pool)
1852                mempool_destroy(bs->bio_pool);
1853
1854        if (bs->bvec_pool)
1855                mempool_destroy(bs->bvec_pool);
1856
1857        bioset_integrity_free(bs);
1858        bio_put_slab(bs);
1859
1860        kfree(bs);
1861}
1862EXPORT_SYMBOL(bioset_free);
1863
1864/**
1865 * bioset_create  - Create a bio_set
1866 * @pool_size:  Number of bio and bio_vecs to cache in the mempool
1867 * @front_pad:  Number of bytes to allocate in front of the returned bio
1868 *
1869 * Description:
1870 *    Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1871 *    to ask for a number of bytes to be allocated in front of the bio.
1872 *    Front pad allocation is useful for embedding the bio inside
1873 *    another structure, to avoid allocating extra data to go with the bio.
1874 *    Note that the bio must be embedded at the END of that structure always,
1875 *    or things will break badly.
1876 */
1877struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
1878{
1879        unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1880        struct bio_set *bs;
1881
1882        bs = kzalloc(sizeof(*bs), GFP_KERNEL);
1883        if (!bs)
1884                return NULL;
1885
1886        bs->front_pad = front_pad;
1887
1888        spin_lock_init(&bs->rescue_lock);
1889        bio_list_init(&bs->rescue_list);
1890        INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1891
1892        bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
1893        if (!bs->bio_slab) {
1894                kfree(bs);
1895                return NULL;
1896        }
1897
1898        bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);
1899        if (!bs->bio_pool)
1900                goto bad;
1901
1902        bs->bvec_pool = biovec_create_pool(bs, pool_size);
1903        if (!bs->bvec_pool)
1904                goto bad;
1905
1906        bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
1907        if (!bs->rescue_workqueue)
1908                goto bad;
1909
1910        return bs;
1911bad:
1912        bioset_free(bs);
1913        return NULL;
1914}
1915EXPORT_SYMBOL(bioset_create);
1916
1917#ifdef CONFIG_BLK_CGROUP
1918/**
1919 * bio_associate_current - associate a bio with %current
1920 * @bio: target bio
1921 *
1922 * Associate @bio with %current if it hasn't been associated yet.  Block
1923 * layer will treat @bio as if it were issued by %current no matter which
1924 * task actually issues it.
1925 *
1926 * This function takes an extra reference of @task's io_context and blkcg
1927 * which will be put when @bio is released.  The caller must own @bio,
1928 * ensure %current->io_context exists, and is responsible for synchronizing
1929 * calls to this function.
1930 */
1931int bio_associate_current(struct bio *bio)
1932{
1933        struct io_context *ioc;
1934        struct cgroup_subsys_state *css;
1935
1936        if (bio->bi_ioc)
1937                return -EBUSY;
1938
1939        ioc = current->io_context;
1940        if (!ioc)
1941                return -ENOENT;
1942
1943        /* acquire active ref on @ioc and associate */
1944        get_io_context_active(ioc);
1945        bio->bi_ioc = ioc;
1946
1947        /* associate blkcg if exists */
1948        rcu_read_lock();
1949        css = task_subsys_state(current, blkio_subsys_id);
1950        if (css && css_tryget(css))
1951                bio->bi_css = css;
1952        rcu_read_unlock();
1953
1954        return 0;
1955}
1956
1957/**
1958 * bio_disassociate_task - undo bio_associate_current()
1959 * @bio: target bio
1960 */
1961void bio_disassociate_task(struct bio *bio)
1962{
1963        if (bio->bi_ioc) {
1964                put_io_context(bio->bi_ioc);
1965                bio->bi_ioc = NULL;
1966        }
1967        if (bio->bi_css) {
1968                css_put(bio->bi_css);
1969                bio->bi_css = NULL;
1970        }
1971}
1972
1973#endif /* CONFIG_BLK_CGROUP */
1974
1975static void __init biovec_init_slabs(void)
1976{
1977        int i;
1978
1979        for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1980                int size;
1981                struct biovec_slab *bvs = bvec_slabs + i;
1982
1983                if (bvs->nr_vecs <= BIO_INLINE_VECS) {
1984                        bvs->slab = NULL;
1985                        continue;
1986                }
1987
1988                size = bvs->nr_vecs * sizeof(struct bio_vec);
1989                bvs->slab = kmem_cache_create(bvs->name, size, 0,
1990                                SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
1991        }
1992}
1993
1994static int __init init_bio(void)
1995{
1996        bio_slab_max = 2;
1997        bio_slab_nr = 0;
1998        bio_slabs = kzalloc(bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL);
1999        if (!bio_slabs)
2000                panic("bio: can't allocate bios\n");
2001
2002        bio_integrity_init();
2003        biovec_init_slabs();
2004
2005        fs_bio_set = bioset_create(BIO_POOL_SIZE, 0);
2006        if (!fs_bio_set)
2007                panic("bio: can't allocate bios\n");
2008
2009        if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
2010                panic("bio: can't create integrity pool\n");
2011
2012        bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES,
2013                                                     sizeof(struct bio_pair));
2014        if (!bio_split_pool)
2015                panic("bio: can't create split pool\n");
2016
2017        return 0;
2018}
2019subsys_initcall(init_bio);
2020
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.