linux/drivers/md/dm.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
   3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
   4 *
   5 * This file is released under the GPL.
   6 */
   7
   8#include "dm.h"
   9#include "dm-uevent.h"
  10
  11#include <linux/init.h>
  12#include <linux/module.h>
  13#include <linux/mutex.h>
  14#include <linux/moduleparam.h>
  15#include <linux/blkpg.h>
  16#include <linux/bio.h>
  17#include <linux/mempool.h>
  18#include <linux/slab.h>
  19#include <linux/idr.h>
  20#include <linux/hdreg.h>
  21#include <linux/delay.h>
  22
  23#include <trace/events/block.h>
  24
  25#define DM_MSG_PREFIX "core"
  26
  27#ifdef CONFIG_PRINTK
  28/*
  29 * ratelimit state to be used in DMXXX_LIMIT().
  30 */
  31DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
  32                       DEFAULT_RATELIMIT_INTERVAL,
  33                       DEFAULT_RATELIMIT_BURST);
  34EXPORT_SYMBOL(dm_ratelimit_state);
  35#endif
  36
  37/*
  38 * Cookies are numeric values sent with CHANGE and REMOVE
  39 * uevents while resuming, removing or renaming the device.
  40 */
  41#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
  42#define DM_COOKIE_LENGTH 24
  43
  44static const char *_name = DM_NAME;
  45
  46static unsigned int major = 0;
  47static unsigned int _major = 0;
  48
  49static DEFINE_IDR(_minor_idr);
  50
  51static DEFINE_SPINLOCK(_minor_lock);
  52/*
  53 * For bio-based dm.
  54 * One of these is allocated per bio.
  55 */
  56struct dm_io {
  57        struct mapped_device *md;
  58        int error;
  59        atomic_t io_count;
  60        struct bio *bio;
  61        unsigned long start_time;
  62        spinlock_t endio_lock;
  63};
  64
  65/*
  66 * For request-based dm.
  67 * One of these is allocated per request.
  68 */
  69struct dm_rq_target_io {
  70        struct mapped_device *md;
  71        struct dm_target *ti;
  72        struct request *orig, clone;
  73        int error;
  74        union map_info info;
  75};
  76
  77/*
  78 * For request-based dm - the bio clones we allocate are embedded in these
  79 * structs.
  80 *
  81 * We allocate these with bio_alloc_bioset, using the front_pad parameter when
  82 * the bioset is created - this means the bio has to come at the end of the
  83 * struct.
  84 */
  85struct dm_rq_clone_bio_info {
  86        struct bio *orig;
  87        struct dm_rq_target_io *tio;
  88        struct bio clone;
  89};
  90
  91union map_info *dm_get_mapinfo(struct bio *bio)
  92{
  93        if (bio && bio->bi_private)
  94                return &((struct dm_target_io *)bio->bi_private)->info;
  95        return NULL;
  96}
  97
  98union map_info *dm_get_rq_mapinfo(struct request *rq)
  99{
 100        if (rq && rq->end_io_data)
 101                return &((struct dm_rq_target_io *)rq->end_io_data)->info;
 102        return NULL;
 103}
 104EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo);
 105
 106#define MINOR_ALLOCED ((void *)-1)
 107
 108/*
 109 * Bits for the md->flags field.
 110 */
 111#define DMF_BLOCK_IO_FOR_SUSPEND 0
 112#define DMF_SUSPENDED 1
 113#define DMF_FROZEN 2
 114#define DMF_FREEING 3
 115#define DMF_DELETING 4
 116#define DMF_NOFLUSH_SUSPENDING 5
 117#define DMF_MERGE_IS_OPTIONAL 6
 118
 119/*
 120 * Work processed by per-device workqueue.
 121 */
 122struct mapped_device {
 123        struct rw_semaphore io_lock;
 124        struct mutex suspend_lock;
 125        rwlock_t map_lock;
 126        atomic_t holders;
 127        atomic_t open_count;
 128
 129        unsigned long flags;
 130
 131        struct request_queue *queue;
 132        unsigned type;
 133        /* Protect queue and type against concurrent access. */
 134        struct mutex type_lock;
 135
 136        struct target_type *immutable_target_type;
 137
 138        struct gendisk *disk;
 139        char name[16];
 140
 141        void *interface_ptr;
 142
 143        /*
 144         * A list of ios that arrived while we were suspended.
 145         */
 146        atomic_t pending[2];
 147        wait_queue_head_t wait;
 148        struct work_struct work;
 149        struct bio_list deferred;
 150        spinlock_t deferred_lock;
 151
 152        /*
 153         * Processing queue (flush)
 154         */
 155        struct workqueue_struct *wq;
 156
 157        /*
 158         * The current mapping.
 159         */
 160        struct dm_table *map;
 161
 162        /*
 163         * io objects are allocated from here.
 164         */
 165        mempool_t *io_pool;
 166
 167        struct bio_set *bs;
 168
 169        /*
 170         * Event handling.
 171         */
 172        atomic_t event_nr;
 173        wait_queue_head_t eventq;
 174        atomic_t uevent_seq;
 175        struct list_head uevent_list;
 176        spinlock_t uevent_lock; /* Protect access to uevent_list */
 177
 178        /*
 179         * freeze/thaw support require holding onto a super block
 180         */
 181        struct super_block *frozen_sb;
 182        struct block_device *bdev;
 183
 184        /* forced geometry settings */
 185        struct hd_geometry geometry;
 186
 187        /* sysfs handle */
 188        struct kobject kobj;
 189
 190        /* zero-length flush that will be cloned and submitted to targets */
 191        struct bio flush_bio;
 192};
 193
 194/*
 195 * For mempools pre-allocation at the table loading time.
 196 */
 197struct dm_md_mempools {
 198        mempool_t *io_pool;
 199        struct bio_set *bs;
 200};
 201
 202#define MIN_IOS 256
 203static struct kmem_cache *_io_cache;
 204static struct kmem_cache *_rq_tio_cache;
 205
 206static int __init local_init(void)
 207{
 208        int r = -ENOMEM;
 209
 210        /* allocate a slab for the dm_ios */
 211        _io_cache = KMEM_CACHE(dm_io, 0);
 212        if (!_io_cache)
 213                return r;
 214
 215        _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
 216        if (!_rq_tio_cache)
 217                goto out_free_io_cache;
 218
 219        r = dm_uevent_init();
 220        if (r)
 221                goto out_free_rq_tio_cache;
 222
 223        _major = major;
 224        r = register_blkdev(_major, _name);
 225        if (r < 0)
 226                goto out_uevent_exit;
 227
 228        if (!_major)
 229                _major = r;
 230
 231        return 0;
 232
 233out_uevent_exit:
 234        dm_uevent_exit();
 235out_free_rq_tio_cache:
 236        kmem_cache_destroy(_rq_tio_cache);
 237out_free_io_cache:
 238        kmem_cache_destroy(_io_cache);
 239
 240        return r;
 241}
 242
 243static void local_exit(void)
 244{
 245        kmem_cache_destroy(_rq_tio_cache);
 246        kmem_cache_destroy(_io_cache);
 247        unregister_blkdev(_major, _name);
 248        dm_uevent_exit();
 249
 250        _major = 0;
 251
 252        DMINFO("cleaned up");
 253}
 254
 255static int (*_inits[])(void) __initdata = {
 256        local_init,
 257        dm_target_init,
 258        dm_linear_init,
 259        dm_stripe_init,
 260        dm_io_init,
 261        dm_kcopyd_init,
 262        dm_interface_init,
 263};
 264
 265static void (*_exits[])(void) = {
 266        local_exit,
 267        dm_target_exit,
 268        dm_linear_exit,
 269        dm_stripe_exit,
 270        dm_io_exit,
 271        dm_kcopyd_exit,
 272        dm_interface_exit,
 273};
 274
 275static int __init dm_init(void)
 276{
 277        const int count = ARRAY_SIZE(_inits);
 278
 279        int r, i;
 280
 281        for (i = 0; i < count; i++) {
 282                r = _inits[i]();
 283                if (r)
 284                        goto bad;
 285        }
 286
 287        return 0;
 288
 289      bad:
 290        while (i--)
 291                _exits[i]();
 292
 293        return r;
 294}
 295
 296static void __exit dm_exit(void)
 297{
 298        int i = ARRAY_SIZE(_exits);
 299
 300        while (i--)
 301                _exits[i]();
 302
 303        /*
 304         * Should be empty by this point.
 305         */
 306        idr_destroy(&_minor_idr);
 307}
 308
 309/*
 310 * Block device functions
 311 */
 312int dm_deleting_md(struct mapped_device *md)
 313{
 314        return test_bit(DMF_DELETING, &md->flags);
 315}
 316
 317static int dm_blk_open(struct block_device *bdev, fmode_t mode)
 318{
 319        struct mapped_device *md;
 320
 321        spin_lock(&_minor_lock);
 322
 323        md = bdev->bd_disk->private_data;
 324        if (!md)
 325                goto out;
 326
 327        if (test_bit(DMF_FREEING, &md->flags) ||
 328            dm_deleting_md(md)) {
 329                md = NULL;
 330                goto out;
 331        }
 332
 333        dm_get(md);
 334        atomic_inc(&md->open_count);
 335
 336out:
 337        spin_unlock(&_minor_lock);
 338
 339        return md ? 0 : -ENXIO;
 340}
 341
 342static void dm_blk_close(struct gendisk *disk, fmode_t mode)
 343{
 344        struct mapped_device *md = disk->private_data;
 345
 346        spin_lock(&_minor_lock);
 347
 348        atomic_dec(&md->open_count);
 349        dm_put(md);
 350
 351        spin_unlock(&_minor_lock);
 352}
 353
 354int dm_open_count(struct mapped_device *md)
 355{
 356        return atomic_read(&md->open_count);
 357}
 358
 359/*
 360 * Guarantees nothing is using the device before it's deleted.
 361 */
 362int dm_lock_for_deletion(struct mapped_device *md)
 363{
 364        int r = 0;
 365
 366        spin_lock(&_minor_lock);
 367
 368        if (dm_open_count(md))
 369                r = -EBUSY;
 370        else
 371                set_bit(DMF_DELETING, &md->flags);
 372
 373        spin_unlock(&_minor_lock);
 374
 375        return r;
 376}
 377
 378static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 379{
 380        struct mapped_device *md = bdev->bd_disk->private_data;
 381
 382        return dm_get_geometry(md, geo);
 383}
 384
 385static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
 386                        unsigned int cmd, unsigned long arg)
 387{
 388        struct mapped_device *md = bdev->bd_disk->private_data;
 389        struct dm_table *map = dm_get_live_table(md);
 390        struct dm_target *tgt;
 391        int r = -ENOTTY;
 392
 393        if (!map || !dm_table_get_size(map))
 394                goto out;
 395
 396        /* We only support devices that have a single target */
 397        if (dm_table_get_num_targets(map) != 1)
 398                goto out;
 399
 400        tgt = dm_table_get_target(map, 0);
 401
 402        if (dm_suspended_md(md)) {
 403                r = -EAGAIN;
 404                goto out;
 405        }
 406
 407        if (tgt->type->ioctl)
 408                r = tgt->type->ioctl(tgt, cmd, arg);
 409
 410out:
 411        dm_table_put(map);
 412
 413        return r;
 414}
 415
 416static struct dm_io *alloc_io(struct mapped_device *md)
 417{
 418        return mempool_alloc(md->io_pool, GFP_NOIO);
 419}
 420
 421static void free_io(struct mapped_device *md, struct dm_io *io)
 422{
 423        mempool_free(io, md->io_pool);
 424}
 425
 426static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
 427{
 428        bio_put(&tio->clone);
 429}
 430
 431static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md,
 432                                            gfp_t gfp_mask)
 433{
 434        return mempool_alloc(md->io_pool, gfp_mask);
 435}
 436
 437static void free_rq_tio(struct dm_rq_target_io *tio)
 438{
 439        mempool_free(tio, tio->md->io_pool);
 440}
 441
 442static int md_in_flight(struct mapped_device *md)
 443{
 444        return atomic_read(&md->pending[READ]) +
 445               atomic_read(&md->pending[WRITE]);
 446}
 447
 448static void start_io_acct(struct dm_io *io)
 449{
 450        struct mapped_device *md = io->md;
 451        int cpu;
 452        int rw = bio_data_dir(io->bio);
 453
 454        io->start_time = jiffies;
 455
 456        cpu = part_stat_lock();
 457        part_round_stats(cpu, &dm_disk(md)->part0);
 458        part_stat_unlock();
 459        atomic_set(&dm_disk(md)->part0.in_flight[rw],
 460                atomic_inc_return(&md->pending[rw]));
 461}
 462
 463static void end_io_acct(struct dm_io *io)
 464{
 465        struct mapped_device *md = io->md;
 466        struct bio *bio = io->bio;
 467        unsigned long duration = jiffies - io->start_time;
 468        int pending, cpu;
 469        int rw = bio_data_dir(bio);
 470
 471        cpu = part_stat_lock();
 472        part_round_stats(cpu, &dm_disk(md)->part0);
 473        part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
 474        part_stat_unlock();
 475
 476        /*
 477         * After this is decremented the bio must not be touched if it is
 478         * a flush.
 479         */
 480        pending = atomic_dec_return(&md->pending[rw]);
 481        atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
 482        pending += atomic_read(&md->pending[rw^0x1]);
 483
 484        /* nudge anyone waiting on suspend queue */
 485        if (!pending)
 486                wake_up(&md->wait);
 487}
 488
 489/*
 490 * Add the bio to the list of deferred io.
 491 */
 492static void queue_io(struct mapped_device *md, struct bio *bio)
 493{
 494        unsigned long flags;
 495
 496        spin_lock_irqsave(&md->deferred_lock, flags);
 497        bio_list_add(&md->deferred, bio);
 498        spin_unlock_irqrestore(&md->deferred_lock, flags);
 499        queue_work(md->wq, &md->work);
 500}
 501
 502/*
 503 * Everyone (including functions in this file), should use this
 504 * function to access the md->map field, and make sure they call
 505 * dm_table_put() when finished.
 506 */
 507struct dm_table *dm_get_live_table(struct mapped_device *md)
 508{
 509        struct dm_table *t;
 510        unsigned long flags;
 511
 512        read_lock_irqsave(&md->map_lock, flags);
 513        t = md->map;
 514        if (t)
 515                dm_table_get(t);
 516        read_unlock_irqrestore(&md->map_lock, flags);
 517
 518        return t;
 519}
 520
 521/*
 522 * Get the geometry associated with a dm device
 523 */
 524int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
 525{
 526        *geo = md->geometry;
 527
 528        return 0;
 529}
 530
 531/*
 532 * Set the geometry of a device.
 533 */
 534int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
 535{
 536        sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
 537
 538        if (geo->start > sz) {
 539                DMWARN("Start sector is beyond the geometry limits.");
 540                return -EINVAL;
 541        }
 542
 543        md->geometry = *geo;
 544
 545        return 0;
 546}
 547
 548/*-----------------------------------------------------------------
 549 * CRUD START:
 550 *   A more elegant soln is in the works that uses the queue
 551 *   merge fn, unfortunately there are a couple of changes to
 552 *   the block layer that I want to make for this.  So in the
 553 *   interests of getting something for people to use I give
 554 *   you this clearly demarcated crap.
 555 *---------------------------------------------------------------*/
 556
 557static int __noflush_suspending(struct mapped_device *md)
 558{
 559        return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
 560}
 561
 562/*
 563 * Decrements the number of outstanding ios that a bio has been
 564 * cloned into, completing the original io if necc.
 565 */
 566static void dec_pending(struct dm_io *io, int error)
 567{
 568        unsigned long flags;
 569        int io_error;
 570        struct bio *bio;
 571        struct mapped_device *md = io->md;
 572
 573        /* Push-back supersedes any I/O errors */
 574        if (unlikely(error)) {
 575                spin_lock_irqsave(&io->endio_lock, flags);
 576                if (!(io->error > 0 && __noflush_suspending(md)))
 577                        io->error = error;
 578                spin_unlock_irqrestore(&io->endio_lock, flags);
 579        }
 580
 581        if (atomic_dec_and_test(&io->io_count)) {
 582                if (io->error == DM_ENDIO_REQUEUE) {
 583                        /*
 584                         * Target requested pushing back the I/O.
 585                         */
 586                        spin_lock_irqsave(&md->deferred_lock, flags);
 587                        if (__noflush_suspending(md))
 588                                bio_list_add_head(&md->deferred, io->bio);
 589                        else
 590                                /* noflush suspend was interrupted. */
 591                                io->error = -EIO;
 592                        spin_unlock_irqrestore(&md->deferred_lock, flags);
 593                }
 594
 595                io_error = io->error;
 596                bio = io->bio;
 597                end_io_acct(io);
 598                free_io(md, io);
 599
 600                if (io_error == DM_ENDIO_REQUEUE)
 601                        return;
 602
 603                if ((bio->bi_rw & REQ_FLUSH) && bio->bi_size) {
 604                        /*
 605                         * Preflush done for flush with data, reissue
 606                         * without REQ_FLUSH.
 607                         */
 608                        bio->bi_rw &= ~REQ_FLUSH;
 609                        queue_io(md, bio);
 610                } else {
 611                        /* done with normal IO or empty flush */
 612                        trace_block_bio_complete(md->queue, bio, io_error);
 613                        bio_endio(bio, io_error);
 614                }
 615        }
 616}
 617
 618static void clone_endio(struct bio *bio, int error)
 619{
 620        int r = 0;
 621        struct dm_target_io *tio = bio->bi_private;
 622        struct dm_io *io = tio->io;
 623        struct mapped_device *md = tio->io->md;
 624        dm_endio_fn endio = tio->ti->type->end_io;
 625
 626        if (!bio_flagged(bio, BIO_UPTODATE) && !error)
 627                error = -EIO;
 628
 629        if (endio) {
 630                r = endio(tio->ti, bio, error);
 631                if (r < 0 || r == DM_ENDIO_REQUEUE)
 632                        /*
 633                         * error and requeue request are handled
 634                         * in dec_pending().
 635                         */
 636                        error = r;
 637                else if (r == DM_ENDIO_INCOMPLETE)
 638                        /* The target will handle the io */
 639                        return;
 640                else if (r) {
 641                        DMWARN("unimplemented target endio return value: %d", r);
 642                        BUG();
 643                }
 644        }
 645
 646        free_tio(md, tio);
 647        dec_pending(io, error);
 648}
 649
 650/*
 651 * Partial completion handling for request-based dm
 652 */
 653static void end_clone_bio(struct bio *clone, int error)
 654{
 655        struct dm_rq_clone_bio_info *info = clone->bi_private;
 656        struct dm_rq_target_io *tio = info->tio;
 657        struct bio *bio = info->orig;
 658        unsigned int nr_bytes = info->orig->bi_size;
 659
 660        bio_put(clone);
 661
 662        if (tio->error)
 663                /*
 664                 * An error has already been detected on the request.
 665                 * Once error occurred, just let clone->end_io() handle
 666                 * the remainder.
 667                 */
 668                return;
 669        else if (error) {
 670                /*
 671                 * Don't notice the error to the upper layer yet.
 672                 * The error handling decision is made by the target driver,
 673                 * when the request is completed.
 674                 */
 675                tio->error = error;
 676                return;
 677        }
 678
 679        /*
 680         * I/O for the bio successfully completed.
 681         * Notice the data completion to the upper layer.
 682         */
 683
 684        /*
 685         * bios are processed from the head of the list.
 686         * So the completing bio should always be rq->bio.
 687         * If it's not, something wrong is happening.
 688         */
 689        if (tio->orig->bio != bio)
 690                DMERR("bio completion is going in the middle of the request");
 691
 692        /*
 693         * Update the original request.
 694         * Do not use blk_end_request() here, because it may complete
 695         * the original request before the clone, and break the ordering.
 696         */
 697        blk_update_request(tio->orig, 0, nr_bytes);
 698}
 699
 700/*
 701 * Don't touch any member of the md after calling this function because
 702 * the md may be freed in dm_put() at the end of this function.
 703 * Or do dm_get() before calling this function and dm_put() later.
 704 */
 705static void rq_completed(struct mapped_device *md, int rw, int run_queue)
 706{
 707        atomic_dec(&md->pending[rw]);
 708
 709        /* nudge anyone waiting on suspend queue */
 710        if (!md_in_flight(md))
 711                wake_up(&md->wait);
 712
 713        /*
 714         * Run this off this callpath, as drivers could invoke end_io while
 715         * inside their request_fn (and holding the queue lock). Calling
 716         * back into ->request_fn() could deadlock attempting to grab the
 717         * queue lock again.
 718         */
 719        if (run_queue)
 720                blk_run_queue_async(md->queue);
 721
 722        /*
 723         * dm_put() must be at the end of this function. See the comment above
 724         */
 725        dm_put(md);
 726}
 727
 728static void free_rq_clone(struct request *clone)
 729{
 730        struct dm_rq_target_io *tio = clone->end_io_data;
 731
 732        blk_rq_unprep_clone(clone);
 733        free_rq_tio(tio);
 734}
 735
 736/*
 737 * Complete the clone and the original request.
 738 * Must be called without queue lock.
 739 */
 740static void dm_end_request(struct request *clone, int error)
 741{
 742        int rw = rq_data_dir(clone);
 743        struct dm_rq_target_io *tio = clone->end_io_data;
 744        struct mapped_device *md = tio->md;
 745        struct request *rq = tio->orig;
 746
 747        if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
 748                rq->errors = clone->errors;
 749                rq->resid_len = clone->resid_len;
 750
 751                if (rq->sense)
 752                        /*
 753                         * We are using the sense buffer of the original
 754                         * request.
 755                         * So setting the length of the sense data is enough.
 756                         */
 757                        rq->sense_len = clone->sense_len;
 758        }
 759
 760        free_rq_clone(clone);
 761        blk_end_request_all(rq, error);
 762        rq_completed(md, rw, true);
 763}
 764
 765static void dm_unprep_request(struct request *rq)
 766{
 767        struct request *clone = rq->special;
 768
 769        rq->special = NULL;
 770        rq->cmd_flags &= ~REQ_DONTPREP;
 771
 772        free_rq_clone(clone);
 773}
 774
 775/*
 776 * Requeue the original request of a clone.
 777 */
 778void dm_requeue_unmapped_request(struct request *clone)
 779{
 780        int rw = rq_data_dir(clone);
 781        struct dm_rq_target_io *tio = clone->end_io_data;
 782        struct mapped_device *md = tio->md;
 783        struct request *rq = tio->orig;
 784        struct request_queue *q = rq->q;
 785        unsigned long flags;
 786
 787        dm_unprep_request(rq);
 788
 789        spin_lock_irqsave(q->queue_lock, flags);
 790        blk_requeue_request(q, rq);
 791        spin_unlock_irqrestore(q->queue_lock, flags);
 792
 793        rq_completed(md, rw, 0);
 794}
 795EXPORT_SYMBOL_GPL(dm_requeue_unmapped_request);
 796
 797static void __stop_queue(struct request_queue *q)
 798{
 799        blk_stop_queue(q);
 800}
 801
 802static void stop_queue(struct request_queue *q)
 803{
 804        unsigned long flags;
 805
 806        spin_lock_irqsave(q->queue_lock, flags);
 807        __stop_queue(q);
 808        spin_unlock_irqrestore(q->queue_lock, flags);
 809}
 810
 811static void __start_queue(struct request_queue *q)
 812{
 813        if (blk_queue_stopped(q))
 814                blk_start_queue(q);
 815}
 816
 817static void start_queue(struct request_queue *q)
 818{
 819        unsigned long flags;
 820
 821        spin_lock_irqsave(q->queue_lock, flags);
 822        __start_queue(q);
 823        spin_unlock_irqrestore(q->queue_lock, flags);
 824}
 825
 826static void dm_done(struct request *clone, int error, bool mapped)
 827{
 828        int r = error;
 829        struct dm_rq_target_io *tio = clone->end_io_data;
 830        dm_request_endio_fn rq_end_io = NULL;
 831
 832        if (tio->ti) {
 833                rq_end_io = tio->ti->type->rq_end_io;
 834
 835                if (mapped && rq_end_io)
 836                        r = rq_end_io(tio->ti, clone, error, &tio->info);
 837        }
 838
 839        if (r <= 0)
 840                /* The target wants to complete the I/O */
 841                dm_end_request(clone, r);
 842        else if (r == DM_ENDIO_INCOMPLETE)
 843                /* The target will handle the I/O */
 844                return;
 845        else if (r == DM_ENDIO_REQUEUE)
 846                /* The target wants to requeue the I/O */
 847                dm_requeue_unmapped_request(clone);
 848        else {
 849                DMWARN("unimplemented target endio return value: %d", r);
 850                BUG();
 851        }
 852}
 853
 854/*
 855 * Request completion handler for request-based dm
 856 */
 857static void dm_softirq_done(struct request *rq)
 858{
 859        bool mapped = true;
 860        struct request *clone = rq->completion_data;
 861        struct dm_rq_target_io *tio = clone->end_io_data;
 862
 863        if (rq->cmd_flags & REQ_FAILED)
 864                mapped = false;
 865
 866        dm_done(clone, tio->error, mapped);
 867}
 868
 869/*
 870 * Complete the clone and the original request with the error status
 871 * through softirq context.
 872 */
 873static void dm_complete_request(struct request *clone, int error)
 874{
 875        struct dm_rq_target_io *tio = clone->end_io_data;
 876        struct request *rq = tio->orig;
 877
 878        tio->error = error;
 879        rq->completion_data = clone;
 880        blk_complete_request(rq);
 881}
 882
 883/*
 884 * Complete the not-mapped clone and the original request with the error status
 885 * through softirq context.
 886 * Target's rq_end_io() function isn't called.
 887 * This may be used when the target's map_rq() function fails.
 888 */
 889void dm_kill_unmapped_request(struct request *clone, int error)
 890{
 891        struct dm_rq_target_io *tio = clone->end_io_data;
 892        struct request *rq = tio->orig;
 893
 894        rq->cmd_flags |= REQ_FAILED;
 895        dm_complete_request(clone, error);
 896}
 897EXPORT_SYMBOL_GPL(dm_kill_unmapped_request);
 898
 899/*
 900 * Called with the queue lock held
 901 */
 902static void end_clone_request(struct request *clone, int error)
 903{
 904        /*
 905         * For just cleaning up the information of the queue in which
 906         * the clone was dispatched.
 907         * The clone is *NOT* freed actually here because it is alloced from
 908         * dm own mempool and REQ_ALLOCED isn't set in clone->cmd_flags.
 909         */
 910        __blk_put_request(clone->q, clone);
 911
 912        /*
 913         * Actual request completion is done in a softirq context which doesn't
 914         * hold the queue lock.  Otherwise, deadlock could occur because:
 915         *     - another request may be submitted by the upper level driver
 916         *       of the stacking during the completion
 917         *     - the submission which requires queue lock may be done
 918         *       against this queue
 919         */
 920        dm_complete_request(clone, error);
 921}
 922
 923/*
 924 * Return maximum size of I/O possible at the supplied sector up to the current
 925 * target boundary.
 926 */
 927static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
 928{
 929        sector_t target_offset = dm_target_offset(ti, sector);
 930
 931        return ti->len - target_offset;
 932}
 933
 934static sector_t max_io_len(sector_t sector, struct dm_target *ti)
 935{
 936        sector_t len = max_io_len_target_boundary(sector, ti);
 937        sector_t offset, max_len;
 938
 939        /*
 940         * Does the target need to split even further?
 941         */
 942        if (ti->max_io_len) {
 943                offset = dm_target_offset(ti, sector);
 944                if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
 945                        max_len = sector_div(offset, ti->max_io_len);
 946                else
 947                        max_len = offset & (ti->max_io_len - 1);
 948                max_len = ti->max_io_len - max_len;
 949
 950                if (len > max_len)
 951                        len = max_len;
 952        }
 953
 954        return len;
 955}
 956
 957int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
 958{
 959        if (len > UINT_MAX) {
 960                DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
 961                      (unsigned long long)len, UINT_MAX);
 962                ti->error = "Maximum size of target IO is too large";
 963                return -EINVAL;
 964        }
 965
 966        ti->max_io_len = (uint32_t) len;
 967
 968        return 0;
 969}
 970EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
 971
 972static void __map_bio(struct dm_target_io *tio)
 973{
 974        int r;
 975        sector_t sector;
 976        struct mapped_device *md;
 977        struct bio *clone = &tio->clone;
 978        struct dm_target *ti = tio->ti;
 979
 980        clone->bi_end_io = clone_endio;
 981        clone->bi_private = tio;
 982
 983        /*
 984         * Map the clone.  If r == 0 we don't need to do
 985         * anything, the target has assumed ownership of
 986         * this io.
 987         */
 988        atomic_inc(&tio->io->io_count);
 989        sector = clone->bi_sector;
 990        r = ti->type->map(ti, clone);
 991        if (r == DM_MAPIO_REMAPPED) {
 992                /* the bio has been remapped so dispatch it */
 993
 994                trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
 995                                      tio->io->bio->bi_bdev->bd_dev, sector);
 996
 997                generic_make_request(clone);
 998        } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
 999                /* error the io and bail out, or requeue it if needed */
1000                md = tio->io->md;
1001                dec_pending(tio->io, r);
1002                free_tio(md, tio);
1003        } else if (r) {
1004                DMWARN("unimplemented target map return value: %d", r);
1005                BUG();
1006        }
1007}
1008
1009struct clone_info {
1010        struct mapped_device *md;
1011        struct dm_table *map;
1012        struct bio *bio;
1013        struct dm_io *io;
1014        sector_t sector;
1015        sector_t sector_count;
1016        unsigned short idx;
1017};
1018
1019static void bio_setup_sector(struct bio *bio, sector_t sector, sector_t len)
1020{
1021        bio->bi_sector = sector;
1022        bio->bi_size = to_bytes(len);
1023}
1024
1025static void bio_setup_bv(struct bio *bio, unsigned short idx, unsigned short bv_count)
1026{
1027        bio->bi_idx = idx;
1028        bio->bi_vcnt = idx + bv_count;
1029        bio->bi_flags &= ~(1 << BIO_SEG_VALID);
1030}
1031
1032static void clone_bio_integrity(struct bio *bio, struct bio *clone,
1033                                unsigned short idx, unsigned len, unsigned offset,
1034                                unsigned trim)
1035{
1036        if (!bio_integrity(bio))
1037                return;
1038
1039        bio_integrity_clone(clone, bio, GFP_NOIO);
1040
1041        if (trim)
1042                bio_integrity_trim(clone, bio_sector_offset(bio, idx, offset), len);
1043}
1044
1045/*
1046 * Creates a little bio that just does part of a bvec.
1047 */
1048static void clone_split_bio(struct dm_target_io *tio, struct bio *bio,
1049                            sector_t sector, unsigned short idx,
1050                            unsigned offset, unsigned len)
1051{
1052        struct bio *clone = &tio->clone;
1053        struct bio_vec *bv = bio->bi_io_vec + idx;
1054
1055        *clone->bi_io_vec = *bv;
1056
1057        bio_setup_sector(clone, sector, len);
1058
1059        clone->bi_bdev = bio->bi_bdev;
1060        clone->bi_rw = bio->bi_rw;
1061        clone->bi_vcnt = 1;
1062        clone->bi_io_vec->bv_offset = offset;
1063        clone->bi_io_vec->bv_len = clone->bi_size;
1064        clone->bi_flags |= 1 << BIO_CLONED;
1065
1066        clone_bio_integrity(bio, clone, idx, len, offset, 1);
1067}
1068
1069/*
1070 * Creates a bio that consists of range of complete bvecs.
1071 */
1072static void clone_bio(struct dm_target_io *tio, struct bio *bio,
1073                      sector_t sector, unsigned short idx,
1074                      unsigned short bv_count, unsigned len)
1075{
1076        struct bio *clone = &tio->clone;
1077        unsigned trim = 0;
1078
1079        __bio_clone(clone, bio);
1080        bio_setup_sector(clone, sector, len);
1081        bio_setup_bv(clone, idx, bv_count);
1082
1083        if (idx != bio->bi_idx || clone->bi_size < bio->bi_size)
1084                trim = 1;
1085        clone_bio_integrity(bio, clone, idx, len, 0, trim);
1086}
1087
1088static struct dm_target_io *alloc_tio(struct clone_info *ci,
1089                                      struct dm_target *ti, int nr_iovecs,
1090                                      unsigned target_bio_nr)
1091{
1092        struct dm_target_io *tio;
1093        struct bio *clone;
1094
1095        clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, ci->md->bs);
1096        tio = container_of(clone, struct dm_target_io, clone);
1097
1098        tio->io = ci->io;
1099        tio->ti = ti;
1100        memset(&tio->info, 0, sizeof(tio->info));
1101        tio->target_bio_nr = target_bio_nr;
1102
1103        return tio;
1104}
1105
1106static void __clone_and_map_simple_bio(struct clone_info *ci,
1107                                       struct dm_target *ti,
1108                                       unsigned target_bio_nr, sector_t len)
1109{
1110        struct dm_target_io *tio = alloc_tio(ci, ti, ci->bio->bi_max_vecs, target_bio_nr);
1111        struct bio *clone = &tio->clone;
1112
1113        /*
1114         * Discard requests require the bio's inline iovecs be initialized.
1115         * ci->bio->bi_max_vecs is BIO_INLINE_VECS anyway, for both flush
1116         * and discard, so no need for concern about wasted bvec allocations.
1117         */
1118         __bio_clone(clone, ci->bio);
1119        if (len)
1120                bio_setup_sector(clone, ci->sector, len);
1121
1122        __map_bio(tio);
1123}
1124
1125static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1126                                  unsigned num_bios, sector_t len)
1127{
1128        unsigned target_bio_nr;
1129
1130        for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
1131                __clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
1132}
1133
1134static int __send_empty_flush(struct clone_info *ci)
1135{
1136        unsigned target_nr = 0;
1137        struct dm_target *ti;
1138
1139        BUG_ON(bio_has_data(ci->bio));
1140        while ((ti = dm_table_get_target(ci->map, target_nr++)))
1141                __send_duplicate_bios(ci, ti, ti->num_flush_bios, 0);
1142
1143        return 0;
1144}
1145
1146static void __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1147                                     sector_t sector, int nr_iovecs,
1148                                     unsigned short idx, unsigned short bv_count,
1149                                     unsigned offset, unsigned len,
1150                                     unsigned split_bvec)
1151{
1152        struct bio *bio = ci->bio;
1153        struct dm_target_io *tio;
1154        unsigned target_bio_nr;
1155        unsigned num_target_bios = 1;
1156
1157        /*
1158         * Does the target want to receive duplicate copies of the bio?
1159         */
1160        if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
1161                num_target_bios = ti->num_write_bios(ti, bio);
1162
1163        for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
1164                tio = alloc_tio(ci, ti, nr_iovecs, target_bio_nr);
1165                if (split_bvec)
1166                        clone_split_bio(tio, bio, sector, idx, offset, len);
1167                else
1168                        clone_bio(tio, bio, sector, idx, bv_count, len);
1169                __map_bio(tio);
1170        }
1171}
1172
1173typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
1174
1175static unsigned get_num_discard_bios(struct dm_target *ti)
1176{
1177        return ti->num_discard_bios;
1178}
1179
1180static unsigned get_num_write_same_bios(struct dm_target *ti)
1181{
1182        return ti->num_write_same_bios;
1183}
1184
1185typedef bool (*is_split_required_fn)(struct dm_target *ti);
1186
1187static bool is_split_required_for_discard(struct dm_target *ti)
1188{
1189        return ti->split_discard_bios;
1190}
1191
1192static int __send_changing_extent_only(struct clone_info *ci,
1193                                       get_num_bios_fn get_num_bios,
1194                                       is_split_required_fn is_split_required)
1195{
1196        struct dm_target *ti;
1197        sector_t len;
1198        unsigned num_bios;
1199
1200        do {
1201                ti = dm_table_find_target(ci->map, ci->sector);
1202                if (!dm_target_is_valid(ti))
1203                        return -EIO;
1204
1205                /*
1206                 * Even though the device advertised support for this type of
1207                 * request, that does not mean every target supports it, and
1208                 * reconfiguration might also have changed that since the
1209                 * check was performed.
1210                 */
1211                num_bios = get_num_bios ? get_num_bios(ti) : 0;
1212                if (!num_bios)
1213                        return -EOPNOTSUPP;
1214
1215                if (is_split_required && !is_split_required(ti))
1216                        len = min(ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1217                else
1218                        len = min(ci->sector_count, max_io_len(ci->sector, ti));
1219
1220                __send_duplicate_bios(ci, ti, num_bios, len);
1221
1222                ci->sector += len;
1223        } while (ci->sector_count -= len);
1224
1225        return 0;
1226}
1227
1228static int __send_discard(struct clone_info *ci)
1229{
1230        return __send_changing_extent_only(ci, get_num_discard_bios,
1231                                           is_split_required_for_discard);
1232}
1233
1234static int __send_write_same(struct clone_info *ci)
1235{
1236        return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
1237}
1238
1239/*
1240 * Find maximum number of sectors / bvecs we can process with a single bio.
1241 */
1242static sector_t __len_within_target(struct clone_info *ci, sector_t max, int *idx)
1243{
1244        struct bio *bio = ci->bio;
1245        sector_t bv_len, total_len = 0;
1246
1247        for (*idx = ci->idx; max && (*idx < bio->bi_vcnt); (*idx)++) {
1248                bv_len = to_sector(bio->bi_io_vec[*idx].bv_len);
1249
1250                if (bv_len > max)
1251                        break;
1252
1253                max -= bv_len;
1254                total_len += bv_len;
1255        }
1256
1257        return total_len;
1258}
1259
1260static int __split_bvec_across_targets(struct clone_info *ci,
1261                                       struct dm_target *ti, sector_t max)
1262{
1263        struct bio *bio = ci->bio;
1264        struct bio_vec *bv = bio->bi_io_vec + ci->idx;
1265        sector_t remaining = to_sector(bv->bv_len);
1266        unsigned offset = 0;
1267        sector_t len;
1268
1269        do {
1270                if (offset) {
1271                        ti = dm_table_find_target(ci->map, ci->sector);
1272                        if (!dm_target_is_valid(ti))
1273                                return -EIO;
1274
1275                        max = max_io_len(ci->sector, ti);
1276                }
1277
1278                len = min(remaining, max);
1279
1280                __clone_and_map_data_bio(ci, ti, ci->sector, 1, ci->idx, 0,
1281                                         bv->bv_offset + offset, len, 1);
1282
1283                ci->sector += len;
1284                ci->sector_count -= len;
1285                offset += to_bytes(len);
1286        } while (remaining -= len);
1287
1288        ci->idx++;
1289
1290        return 0;
1291}
1292
1293/*
1294 * Select the correct strategy for processing a non-flush bio.
1295 */
1296static int __split_and_process_non_flush(struct clone_info *ci)
1297{
1298        struct bio *bio = ci->bio;
1299        struct dm_target *ti;
1300        sector_t len, max;
1301        int idx;
1302
1303        if (unlikely(bio->bi_rw & REQ_DISCARD))
1304                return __send_discard(ci);
1305        else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
1306                return __send_write_same(ci);
1307
1308        ti = dm_table_find_target(ci->map, ci->sector);
1309        if (!dm_target_is_valid(ti))
1310                return -EIO;
1311
1312        max = max_io_len(ci->sector, ti);
1313
1314        /*
1315         * Optimise for the simple case where we can do all of
1316         * the remaining io with a single clone.
1317         */
1318        if (ci->sector_count <= max) {
1319                __clone_and_map_data_bio(ci, ti, ci->sector, bio->bi_max_vecs,
1320                                         ci->idx, bio->bi_vcnt - ci->idx, 0,
1321                                         ci->sector_count, 0);
1322                ci->sector_count = 0;
1323                return 0;
1324        }
1325
1326        /*
1327         * There are some bvecs that don't span targets.
1328         * Do as many of these as possible.
1329         */
1330        if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
1331                len = __len_within_target(ci, max, &idx);
1332
1333                __clone_and_map_data_bio(ci, ti, ci->sector, bio->bi_max_vecs,
1334                                         ci->idx, idx - ci->idx, 0, len, 0);
1335
1336                ci->sector += len;
1337                ci->sector_count -= len;
1338                ci->idx = idx;
1339
1340                return 0;
1341        }
1342
1343        /*
1344         * Handle a bvec that must be split between two or more targets.
1345         */
1346        return __split_bvec_across_targets(ci, ti, max);
1347}
1348
1349/*
1350 * Entry point to split a bio into clones and submit them to the targets.
1351 */
1352static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
1353{
1354        struct clone_info ci;
1355        int error = 0;
1356
1357        ci.map = dm_get_live_table(md);
1358        if (unlikely(!ci.map)) {
1359                bio_io_error(bio);
1360                return;
1361        }
1362
1363        ci.md = md;
1364        ci.io = alloc_io(md);
1365        ci.io->error = 0;
1366        atomic_set(&ci.io->io_count, 1);
1367        ci.io->bio = bio;
1368        ci.io->md = md;
1369        spin_lock_init(&ci.io->endio_lock);
1370        ci.sector = bio->bi_sector;
1371        ci.idx = bio->bi_idx;
1372
1373        start_io_acct(ci.io);
1374
1375        if (bio->bi_rw & REQ_FLUSH) {
1376                ci.bio = &ci.md->flush_bio;
1377                ci.sector_count = 0;
1378                error = __send_empty_flush(&ci);
1379                /* dec_pending submits any data associated with flush */
1380        } else {
1381                ci.bio = bio;
1382                ci.sector_count = bio_sectors(bio);
1383                while (ci.sector_count && !error)
1384                        error = __split_and_process_non_flush(&ci);
1385        }
1386
1387        /* drop the extra reference count */
1388        dec_pending(ci.io, error);
1389        dm_table_put(ci.map);
1390}
1391/*-----------------------------------------------------------------
1392 * CRUD END
1393 *---------------------------------------------------------------*/
1394
1395static int dm_merge_bvec(struct request_queue *q,
1396                         struct bvec_merge_data *bvm,
1397                         struct bio_vec *biovec)
1398{
1399        struct mapped_device *md = q->queuedata;
1400        struct dm_table *map = dm_get_live_table(md);
1401        struct dm_target *ti;
1402        sector_t max_sectors;
1403        int max_size = 0;
1404
1405        if (unlikely(!map))
1406                goto out;
1407
1408        ti = dm_table_find_target(map, bvm->bi_sector);
1409        if (!dm_target_is_valid(ti))
1410                goto out_table;
1411
1412        /*
1413         * Find maximum amount of I/O that won't need splitting
1414         */
1415        max_sectors = min(max_io_len(bvm->bi_sector, ti),
1416                          (sector_t) BIO_MAX_SECTORS);
1417        max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1418        if (max_size < 0)
1419                max_size = 0;
1420
1421        /*
1422         * merge_bvec_fn() returns number of bytes
1423         * it can accept at this offset
1424         * max is precomputed maximal io size
1425         */
1426        if (max_size && ti->type->merge)
1427                max_size = ti->type->merge(ti, bvm, biovec, max_size);
1428        /*
1429         * If the target doesn't support merge method and some of the devices
1430         * provided their merge_bvec method (we know this by looking at
1431         * queue_max_hw_sectors), then we can't allow bios with multiple vector
1432         * entries.  So always set max_size to 0, and the code below allows
1433         * just one page.
1434         */
1435        else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
1436
1437                max_size = 0;
1438
1439out_table:
1440        dm_table_put(map);
1441
1442out:
1443        /*
1444         * Always allow an entire first page
1445         */
1446        if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1447                max_size = biovec->bv_len;
1448
1449        return max_size;
1450}
1451
1452/*
1453 * The request function that just remaps the bio built up by
1454 * dm_merge_bvec.
1455 */
1456static void _dm_request(struct request_queue *q, struct bio *bio)
1457{
1458        int rw = bio_data_dir(bio);
1459        struct mapped_device *md = q->queuedata;
1460        int cpu;
1461
1462        down_read(&md->io_lock);
1463
1464        cpu = part_stat_lock();
1465        part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
1466        part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
1467        part_stat_unlock();
1468
1469        /* if we're suspended, we have to queue this io for later */
1470        if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1471                up_read(&md->io_lock);
1472
1473                if (bio_rw(bio) != READA)
1474                        queue_io(md, bio);
1475                else
1476                        bio_io_error(bio);
1477                return;
1478        }
1479
1480        __split_and_process_bio(md, bio);
1481        up_read(&md->io_lock);
1482        return;
1483}
1484
1485static int dm_request_based(struct mapped_device *md)
1486{
1487        return blk_queue_stackable(md->queue);
1488}
1489
1490static void dm_request(struct request_queue *q, struct bio *bio)
1491{
1492        struct mapped_device *md = q->queuedata;
1493
1494        if (dm_request_based(md))
1495                blk_queue_bio(q, bio);
1496        else
1497                _dm_request(q, bio);
1498}
1499
1500void dm_dispatch_request(struct request *rq)
1501{
1502        int r;
1503
1504        if (blk_queue_io_stat(rq->q))
1505                rq->cmd_flags |= REQ_IO_STAT;
1506
1507        rq->start_time = jiffies;
1508        r = blk_insert_cloned_request(rq->q, rq);
1509        if (r)
1510                dm_complete_request(rq, r);
1511}
1512EXPORT_SYMBOL_GPL(dm_dispatch_request);
1513
1514static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
1515                                 void *data)
1516{
1517        struct dm_rq_target_io *tio = data;
1518        struct dm_rq_clone_bio_info *info =
1519                container_of(bio, struct dm_rq_clone_bio_info, clone);
1520
1521        info->orig = bio_orig;
1522        info->tio = tio;
1523        bio->bi_end_io = end_clone_bio;
1524        bio->bi_private = info;
1525
1526        return 0;
1527}
1528
1529static int setup_clone(struct request *clone, struct request *rq,
1530                       struct dm_rq_target_io *tio)
1531{
1532        int r;
1533
1534        r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,
1535                              dm_rq_bio_constructor, tio);
1536        if (r)
1537                return r;
1538
1539        clone->cmd = rq->cmd;
1540        clone->cmd_len = rq->cmd_len;
1541        clone->sense = rq->sense;
1542        clone->buffer = rq->buffer;
1543        clone->end_io = end_clone_request;
1544        clone->end_io_data = tio;
1545
1546        return 0;
1547}
1548
1549static struct request *clone_rq(struct request *rq, struct mapped_device *md,
1550                                gfp_t gfp_mask)
1551{
1552        struct request *clone;
1553        struct dm_rq_target_io *tio;
1554
1555        tio = alloc_rq_tio(md, gfp_mask);
1556        if (!tio)
1557                return NULL;
1558
1559        tio->md = md;
1560        tio->ti = NULL;
1561        tio->orig = rq;
1562        tio->error = 0;
1563        memset(&tio->info, 0, sizeof(tio->info));
1564
1565        clone = &tio->clone;
1566        if (setup_clone(clone, rq, tio)) {
1567                /* -ENOMEM */
1568                free_rq_tio(tio);
1569                return NULL;
1570        }
1571
1572        return clone;
1573}
1574
1575/*
1576 * Called with the queue lock held.
1577 */
1578static int dm_prep_fn(struct request_queue *q, struct request *rq)
1579{
1580        struct mapped_device *md = q->queuedata;
1581        struct request *clone;
1582
1583        if (unlikely(rq->special)) {
1584                DMWARN("Already has something in rq->special.");
1585                return BLKPREP_KILL;
1586        }
1587
1588        clone = clone_rq(rq, md, GFP_ATOMIC);
1589        if (!clone)
1590                return BLKPREP_DEFER;
1591
1592        rq->special = clone;
1593        rq->cmd_flags |= REQ_DONTPREP;
1594
1595        return BLKPREP_OK;
1596}
1597
1598/*
1599 * Returns:
1600 * 0  : the request has been processed (not requeued)
1601 * !0 : the request has been requeued
1602 */
1603static int map_request(struct dm_target *ti, struct request *clone,
1604                       struct mapped_device *md)
1605{
1606        int r, requeued = 0;
1607        struct dm_rq_target_io *tio = clone->end_io_data;
1608
1609        tio->ti = ti;
1610        r = ti->type->map_rq(ti, clone, &tio->info);
1611        switch (r) {
1612        case DM_MAPIO_SUBMITTED:
1613                /* The target has taken the I/O to submit by itself later */
1614                break;
1615        case DM_MAPIO_REMAPPED:
1616                /* The target has remapped the I/O so dispatch it */
1617                trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
1618                                     blk_rq_pos(tio->orig));
1619                dm_dispatch_request(clone);
1620                break;
1621        case DM_MAPIO_REQUEUE:
1622                /* The target wants to requeue the I/O */
1623                dm_requeue_unmapped_request(clone);
1624                requeued = 1;
1625                break;
1626        default:
1627                if (r > 0) {
1628                        DMWARN("unimplemented target map return value: %d", r);
1629                        BUG();
1630                }
1631
1632                /* The target wants to complete the I/O */
1633                dm_kill_unmapped_request(clone, r);
1634                break;
1635        }
1636
1637        return requeued;
1638}
1639
1640static struct request *dm_start_request(struct mapped_device *md, struct request *orig)
1641{
1642        struct request *clone;
1643
1644        blk_start_request(orig);
1645        clone = orig->special;
1646        atomic_inc(&md->pending[rq_data_dir(clone)]);
1647
1648        /*
1649         * Hold the md reference here for the in-flight I/O.
1650         * We can't rely on the reference count by device opener,
1651         * because the device may be closed during the request completion
1652         * when all bios are completed.
1653         * See the comment in rq_completed() too.
1654         */
1655        dm_get(md);
1656
1657        return clone;
1658}
1659
1660/*
1661 * q->request_fn for request-based dm.
1662 * Called with the queue lock held.
1663 */
1664static void dm_request_fn(struct request_queue *q)
1665{
1666        struct mapped_device *md = q->queuedata;
1667        struct dm_table *map = dm_get_live_table(md);
1668        struct dm_target *ti;
1669        struct request *rq, *clone;
1670        sector_t pos;
1671
1672        /*
1673         * For suspend, check blk_queue_stopped() and increment
1674         * ->pending within a single queue_lock not to increment the
1675         * number of in-flight I/Os after the queue is stopped in
1676         * dm_suspend().
1677         */
1678        while (!blk_queue_stopped(q)) {
1679                rq = blk_peek_request(q);
1680                if (!rq)
1681                        goto delay_and_out;
1682
1683                /* always use block 0 to find the target for flushes for now */
1684                pos = 0;
1685                if (!(rq->cmd_flags & REQ_FLUSH))
1686                        pos = blk_rq_pos(rq);
1687
1688                ti = dm_table_find_target(map, pos);
1689                if (!dm_target_is_valid(ti)) {
1690                        /*
1691                         * Must perform setup, that dm_done() requires,
1692                         * before calling dm_kill_unmapped_request
1693                         */
1694                        DMERR_LIMIT("request attempted access beyond the end of device");
1695                        clone = dm_start_request(md, rq);
1696                        dm_kill_unmapped_request(clone, -EIO);
1697                        continue;
1698                }
1699
1700                if (ti->type->busy && ti->type->busy(ti))
1701                        goto delay_and_out;
1702
1703                clone = dm_start_request(md, rq);
1704
1705                spin_unlock(q->queue_lock);
1706                if (map_request(ti, clone, md))
1707                        goto requeued;
1708
1709                BUG_ON(!irqs_disabled());
1710                spin_lock(q->queue_lock);
1711        }
1712
1713        goto out;
1714
1715requeued:
1716        BUG_ON(!irqs_disabled());
1717        spin_lock(q->queue_lock);
1718
1719delay_and_out:
1720        blk_delay_queue(q, HZ / 10);
1721out:
1722        dm_table_put(map);
1723}
1724
1725int dm_underlying_device_busy(struct request_queue *q)
1726{
1727        return blk_lld_busy(q);
1728}
1729EXPORT_SYMBOL_GPL(dm_underlying_device_busy);
1730
1731static int dm_lld_busy(struct request_queue *q)
1732{
1733        int r;
1734        struct mapped_device *md = q->queuedata;
1735        struct dm_table *map = dm_get_live_table(md);
1736
1737        if (!map || test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))
1738                r = 1;
1739        else
1740                r = dm_table_any_busy_target(map);
1741
1742        dm_table_put(map);
1743
1744        return r;
1745}
1746
1747static int dm_any_congested(void *congested_data, int bdi_bits)
1748{
1749        int r = bdi_bits;
1750        struct mapped_device *md = congested_data;
1751        struct dm_table *map;
1752
1753        if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1754                map = dm_get_live_table(md);
1755                if (map) {
1756                        /*
1757                         * Request-based dm cares about only own queue for
1758                         * the query about congestion status of request_queue
1759                         */
1760                        if (dm_request_based(md))
1761                                r = md->queue->backing_dev_info.state &
1762                                    bdi_bits;
1763                        else
1764                                r = dm_table_any_congested(map, bdi_bits);
1765
1766                        dm_table_put(map);
1767                }
1768        }
1769
1770        return r;
1771}
1772
1773/*-----------------------------------------------------------------
1774 * An IDR is used to keep track of allocated minor numbers.
1775 *---------------------------------------------------------------*/
1776static void free_minor(int minor)
1777{
1778        spin_lock(&_minor_lock);
1779        idr_remove(&_minor_idr, minor);
1780        spin_unlock(&_minor_lock);
1781}
1782
1783/*
1784 * See if the device with a specific minor # is free.
1785 */
1786static int specific_minor(int minor)
1787{
1788        int r;
1789
1790        if (minor >= (1 << MINORBITS))
1791                return -EINVAL;
1792
1793        idr_preload(GFP_KERNEL);
1794        spin_lock(&_minor_lock);
1795
1796        r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1797
1798        spin_unlock(&_minor_lock);
1799        idr_preload_end();
1800        if (r < 0)
1801                return r == -ENOSPC ? -EBUSY : r;
1802        return 0;
1803}
1804
1805static int next_free_minor(int *minor)
1806{
1807        int r;
1808
1809        idr_preload(GFP_KERNEL);
1810        spin_lock(&_minor_lock);
1811
1812        r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1813
1814        spin_unlock(&_minor_lock);
1815        idr_preload_end();
1816        if (r < 0)
1817                return r;
1818        *minor = r;
1819        return 0;
1820}
1821
1822static const struct block_device_operations dm_blk_dops;
1823
1824static void dm_wq_work(struct work_struct *work);
1825
1826static void dm_init_md_queue(struct mapped_device *md)
1827{
1828        /*
1829         * Request-based dm devices cannot be stacked on top of bio-based dm
1830         * devices.  The type of this dm device has not been decided yet.
1831         * The type is decided at the first table loading time.
1832         * To prevent problematic device stacking, clear the queue flag
1833         * for request stacking support until then.
1834         *
1835         * This queue is new, so no concurrency on the queue_flags.
1836         */
1837        queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
1838
1839        md->queue->queuedata = md;
1840        md->queue->backing_dev_info.congested_fn = dm_any_congested;
1841        md->queue->backing_dev_info.congested_data = md;
1842        blk_queue_make_request(md->queue, dm_request);
1843        blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
1844        blk_queue_merge_bvec(md->queue, dm_merge_bvec);
1845}
1846
1847/*
1848 * Allocate and initialise a blank device with a given minor.
1849 */
1850static struct mapped_device *alloc_dev(int minor)
1851{
1852        int r;
1853        struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1854        void *old_md;
1855
1856        if (!md) {
1857                DMWARN("unable to allocate device, out of memory.");
1858                return NULL;
1859        }
1860
1861        if (!try_module_get(THIS_MODULE))
1862                goto bad_module_get;
1863
1864        /* get a minor number for the dev */
1865        if (minor == DM_ANY_MINOR)
1866                r = next_free_minor(&minor);
1867        else
1868                r = specific_minor(minor);
1869        if (r < 0)
1870                goto bad_minor;
1871
1872        md->type = DM_TYPE_NONE;
1873        init_rwsem(&md->io_lock);
1874        mutex_init(&md->suspend_lock);
1875        mutex_init(&md->type_lock);
1876        spin_lock_init(&md->deferred_lock);
1877        rwlock_init(&md->map_lock);
1878        atomic_set(&md->holders, 1);
1879        atomic_set(&md->open_count, 0);
1880        atomic_set(&md->event_nr, 0);
1881        atomic_set(&md->uevent_seq, 0);
1882        INIT_LIST_HEAD(&md->uevent_list);
1883        spin_lock_init(&md->uevent_lock);
1884
1885        md->queue = blk_alloc_queue(GFP_KERNEL);
1886        if (!md->queue)
1887                goto bad_queue;
1888
1889        dm_init_md_queue(md);
1890
1891        md->disk = alloc_disk(1);
1892        if (!md->disk)
1893                goto bad_disk;
1894
1895        atomic_set(&md->pending[0], 0);
1896        atomic_set(&md->pending[1], 0);
1897        init_waitqueue_head(&md->wait);
1898        INIT_WORK(&md->work, dm_wq_work);
1899        init_waitqueue_head(&md->eventq);
1900
1901        md->disk->major = _major;
1902        md->disk->first_minor = minor;
1903        md->disk->fops = &dm_blk_dops;
1904        md->disk->queue = md->queue;
1905        md->disk->private_data = md;
1906        sprintf(md->disk->disk_name, "dm-%d", minor);
1907        add_disk(md->disk);
1908        format_dev_t(md->name, MKDEV(_major, minor));
1909
1910        md->wq = alloc_workqueue("kdmflush",
1911                                 WQ_NON_REENTRANT | WQ_MEM_RECLAIM, 0);
1912        if (!md->wq)
1913                goto bad_thread;
1914
1915        md->bdev = bdget_disk(md->disk, 0);
1916        if (!md->bdev)
1917                goto bad_bdev;
1918
1919        bio_init(&md->flush_bio);
1920        md->flush_bio.bi_bdev = md->bdev;
1921        md->flush_bio.bi_rw = WRITE_FLUSH;
1922
1923        /* Populate the mapping, nobody knows we exist yet */
1924        spin_lock(&_minor_lock);
1925        old_md = idr_replace(&_minor_idr, md, minor);
1926        spin_unlock(&_minor_lock);
1927
1928        BUG_ON(old_md != MINOR_ALLOCED);
1929
1930        return md;
1931
1932bad_bdev:
1933        destroy_workqueue(md->wq);
1934bad_thread:
1935        del_gendisk(md->disk);
1936        put_disk(md->disk);
1937bad_disk:
1938        blk_cleanup_queue(md->queue);
1939bad_queue:
1940        free_minor(minor);
1941bad_minor:
1942        module_put(THIS_MODULE);
1943bad_module_get:
1944        kfree(md);
1945        return NULL;
1946}
1947
1948static void unlock_fs(struct mapped_device *md);
1949
1950static void free_dev(struct mapped_device *md)
1951{
1952        int minor = MINOR(disk_devt(md->disk));
1953
1954        unlock_fs(md);
1955        bdput(md->bdev);
1956        destroy_workqueue(md->wq);
1957        if (md->io_pool)
1958                mempool_destroy(md->io_pool);
1959        if (md->bs)
1960                bioset_free(md->bs);
1961        blk_integrity_unregister(md->disk);
1962        del_gendisk(md->disk);
1963        free_minor(minor);
1964
1965        spin_lock(&_minor_lock);
1966        md->disk->private_data = NULL;
1967        spin_unlock(&_minor_lock);
1968
1969        put_disk(md->disk);
1970        blk_cleanup_queue(md->queue);
1971        module_put(THIS_MODULE);
1972        kfree(md);
1973}
1974
1975static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
1976{
1977        struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1978
1979        if (md->io_pool && md->bs) {
1980                /* The md already has necessary mempools. */
1981                if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) {
1982                        /*
1983                         * Reload bioset because front_pad may have changed
1984                         * because a different table was loaded.
1985                         */
1986                        bioset_free(md->bs);
1987                        md->bs = p->bs;
1988                        p->bs = NULL;
1989                } else if (dm_table_get_type(t) == DM_TYPE_REQUEST_BASED) {
1990                        /*
1991                         * There's no need to reload with request-based dm
1992                         * because the size of front_pad doesn't change.
1993                         * Note for future: If you are to reload bioset,
1994                         * prep-ed requests in the queue may refer
1995                         * to bio from the old bioset, so you must walk
1996                         * through the queue to unprep.
1997                         */
1998                }
1999                goto out;
2000        }
2001
2002        BUG_ON(!p || md->io_pool || md->bs);
2003
2004        md->io_pool = p->io_pool;
2005        p->io_pool = NULL;
2006        md->bs = p->bs;
2007        p->bs = NULL;
2008
2009out:
2010        /* mempool bind completed, now no need any mempools in the table */
2011        dm_table_free_md_mempools(t);
2012}
2013
2014/*
2015 * Bind a table to the device.
2016 */
2017static void event_callback(void *context)
2018{
2019        unsigned long flags;
2020        LIST_HEAD(uevents);
2021        struct mapped_device *md = (struct mapped_device *) context;
2022
2023        spin_lock_irqsave(&md->uevent_lock, flags);
2024        list_splice_init(&md->uevent_list, &uevents);
2025        spin_unlock_irqrestore(&md->uevent_lock, flags);
2026
2027        dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
2028
2029        atomic_inc(&md->event_nr);
2030        wake_up(&md->eventq);
2031}
2032
2033/*
2034 * Protected by md->suspend_lock obtained by dm_swap_table().
2035 */
2036static void __set_size(struct mapped_device *md, sector_t size)
2037{
2038        set_capacity(md->disk, size);
2039
2040        i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
2041}
2042
2043/*
2044 * Return 1 if the queue has a compulsory merge_bvec_fn function.
2045 *
2046 * If this function returns 0, then the device is either a non-dm
2047 * device without a merge_bvec_fn, or it is a dm device that is
2048 * able to split any bios it receives that are too big.
2049 */
2050int dm_queue_merge_is_compulsory(struct request_queue *q)
2051{
2052        struct mapped_device *dev_md;
2053
2054        if (!q->merge_bvec_fn)
2055                return 0;
2056
2057        if (q->make_request_fn == dm_request) {
2058                dev_md = q->queuedata;
2059                if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags))
2060                        return 0;
2061        }
2062
2063        return 1;
2064}
2065
2066static int dm_device_merge_is_compulsory(struct dm_target *ti,
2067                                         struct dm_dev *dev, sector_t start,
2068                                         sector_t len, void *data)
2069{
2070        struct block_device *bdev = dev->bdev;
2071        struct request_queue *q = bdev_get_queue(bdev);
2072
2073        return dm_queue_merge_is_compulsory(q);
2074}
2075
2076/*
2077 * Return 1 if it is acceptable to ignore merge_bvec_fn based
2078 * on the properties of the underlying devices.
2079 */
2080static int dm_table_merge_is_optional(struct dm_table *table)
2081{
2082        unsigned i = 0;
2083        struct dm_target *ti;
2084
2085        while (i < dm_table_get_num_targets(table)) {
2086                ti = dm_table_get_target(table, i++);
2087
2088                if (ti->type->iterate_devices &&
2089                    ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL))
2090                        return 0;
2091        }
2092
2093        return 1;
2094}
2095
2096/*
2097 * Returns old map, which caller must destroy.
2098 */
2099static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2100                               struct queue_limits *limits)
2101{
2102        struct dm_table *old_map;
2103        struct request_queue *q = md->queue;
2104        sector_t size;
2105        unsigned long flags;
2106        int merge_is_optional;
2107
2108        size = dm_table_get_size(t);
2109
2110        /*
2111         * Wipe any geometry if the size of the table changed.
2112         */
2113        if (size != get_capacity(md->disk))
2114                memset(&md->geometry, 0, sizeof(md->geometry));
2115
2116        __set_size(md, size);
2117
2118        dm_table_event_callback(t, event_callback, md);
2119
2120        /*
2121         * The queue hasn't been stopped yet, if the old table type wasn't
2122         * for request-based during suspension.  So stop it to prevent
2123         * I/O mapping before resume.
2124         * This must be done before setting the queue restrictions,
2125         * because request-based dm may be run just after the setting.
2126         */
2127        if (dm_table_request_based(t) && !blk_queue_stopped(q))
2128                stop_queue(q);
2129
2130        __bind_mempools(md, t);
2131
2132        merge_is_optional = dm_table_merge_is_optional(t);
2133
2134        write_lock_irqsave(&md->map_lock, flags);
2135        old_map = md->map;
2136        md->map = t;
2137        md->immutable_target_type = dm_table_get_immutable_target_type(t);
2138
2139        dm_table_set_restrictions(t, q, limits);
2140        if (merge_is_optional)
2141                set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2142        else
2143                clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2144        write_unlock_irqrestore(&md->map_lock, flags);
2145
2146        return old_map;
2147}
2148
2149/*
2150 * Returns unbound table for the caller to free.
2151 */
2152static struct dm_table *__unbind(struct mapped_device *md)
2153{
2154        struct dm_table *map = md->map;
2155        unsigned long flags;
2156
2157        if (!map)
2158                return NULL;
2159
2160        dm_table_event_callback(map, NULL, NULL);
2161        write_lock_irqsave(&md->map_lock, flags);
2162        md->map = NULL;
2163        write_unlock_irqrestore(&md->map_lock, flags);
2164
2165        return map;
2166}
2167
2168/*
2169 * Constructor for a new device.
2170 */
2171int dm_create(int minor, struct mapped_device **result)
2172{
2173        struct mapped_device *md;
2174
2175        md = alloc_dev(minor);
2176        if (!md)
2177                return -ENXIO;
2178
2179        dm_sysfs_init(md);
2180
2181        *result = md;
2182        return 0;
2183}
2184
2185/*
2186 * Functions to manage md->type.
2187 * All are required to hold md->type_lock.
2188 */
2189void dm_lock_md_type(struct mapped_device *md)
2190{
2191        mutex_lock(&md->type_lock);
2192}
2193
2194void dm_unlock_md_type(struct mapped_device *md)
2195{
2196        mutex_unlock(&md->type_lock);
2197}
2198
2199void dm_set_md_type(struct mapped_device *md, unsigned type)
2200{
2201        md->type = type;
2202}
2203
2204unsigned dm_get_md_type(struct mapped_device *md)
2205{
2206        return md->type;
2207}
2208
2209struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2210{
2211        return md->immutable_target_type;
2212}
2213
2214/*
2215 * Fully initialize a request-based queue (->elevator, ->request_fn, etc).
2216 */
2217static int dm_init_request_based_queue(struct mapped_device *md)
2218{
2219        struct request_queue *q = NULL;
2220
2221        if (md->queue->elevator)
2222                return 1;
2223
2224        /* Fully initialize the queue */
2225        q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL);
2226        if (!q)
2227                return 0;
2228
2229        md->queue = q;
2230        dm_init_md_queue(md);
2231        blk_queue_softirq_done(md->queue, dm_softirq_done);
2232        blk_queue_prep_rq(md->queue, dm_prep_fn);
2233        blk_queue_lld_busy(md->queue, dm_lld_busy);
2234
2235        elv_register_queue(md->queue);
2236
2237        return 1;
2238}
2239
2240/*
2241 * Setup the DM device's queue based on md's type
2242 */
2243int dm_setup_md_queue(struct mapped_device *md)
2244{
2245        if ((dm_get_md_type(md) == DM_TYPE_REQUEST_BASED) &&
2246            !dm_init_request_based_queue(md)) {
2247                DMWARN("Cannot initialize queue for request-based mapped device");
2248                return -EINVAL;
2249        }
2250
2251        return 0;
2252}
2253
2254static struct mapped_device *dm_find_md(dev_t dev)
2255{
2256        struct mapped_device *md;
2257        unsigned minor = MINOR(dev);
2258
2259        if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2260                return NULL;
2261
2262        spin_lock(&_minor_lock);
2263
2264        md = idr_find(&_minor_idr, minor);
2265        if (md && (md == MINOR_ALLOCED ||
2266                   (MINOR(disk_devt(dm_disk(md))) != minor) ||
2267                   dm_deleting_md(md) ||
2268                   test_bit(DMF_FREEING, &md->flags))) {
2269                md = NULL;
2270                goto out;
2271        }
2272
2273out:
2274        spin_unlock(&_minor_lock);
2275
2276        return md;
2277}
2278
2279struct mapped_device *dm_get_md(dev_t dev)
2280{
2281        struct mapped_device *md = dm_find_md(dev);
2282
2283        if (md)
2284                dm_get(md);
2285
2286        return md;
2287}
2288EXPORT_SYMBOL_GPL(dm_get_md);
2289
2290void *dm_get_mdptr(struct mapped_device *md)
2291{
2292        return md->interface_ptr;
2293}
2294
2295void dm_set_mdptr(struct mapped_device *md, void *ptr)
2296{
2297        md->interface_ptr = ptr;
2298}
2299
2300void dm_get(struct mapped_device *md)
2301{
2302        atomic_inc(&md->holders);
2303        BUG_ON(test_bit(DMF_FREEING, &md->flags));
2304}
2305
2306const char *dm_device_name(struct mapped_device *md)
2307{
2308        return md->name;
2309}
2310EXPORT_SYMBOL_GPL(dm_device_name);
2311
2312static void __dm_destroy(struct mapped_device *md, bool wait)
2313{
2314        struct dm_table *map;
2315
2316        might_sleep();
2317
2318        spin_lock(&_minor_lock);
2319        map = dm_get_live_table(md);
2320        idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2321        set_bit(DMF_FREEING, &md->flags);
2322        spin_unlock(&_minor_lock);
2323
2324        if (!dm_suspended_md(md)) {
2325                dm_table_presuspend_targets(map);
2326                dm_table_postsuspend_targets(map);
2327        }
2328
2329        /*
2330         * Rare, but there may be I/O requests still going to complete,
2331         * for example.  Wait for all references to disappear.
2332         * No one should increment the reference count of the mapped_device,
2333         * after the mapped_device state becomes DMF_FREEING.
2334         */
2335        if (wait)
2336                while (atomic_read(&md->holders))
2337                        msleep(1);
2338        else if (atomic_read(&md->holders))
2339                DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2340                       dm_device_name(md), atomic_read(&md->holders));
2341
2342        dm_sysfs_exit(md);
2343        dm_table_put(map);
2344        dm_table_destroy(__unbind(md));
2345        free_dev(md);
2346}
2347
2348void dm_destroy(struct mapped_device *md)
2349{
2350        __dm_destroy(md, true);
2351}
2352
2353void dm_destroy_immediate(struct mapped_device *md)
2354{
2355        __dm_destroy(md, false);
2356}
2357
2358void dm_put(struct mapped_device *md)
2359{
2360        atomic_dec(&md->holders);
2361}
2362EXPORT_SYMBOL_GPL(dm_put);
2363
2364static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
2365{
2366        int r = 0;
2367        DECLARE_WAITQUEUE(wait, current);
2368
2369        add_wait_queue(&md->wait, &wait);
2370
2371        while (1) {
2372                set_current_state(interruptible);
2373
2374                if (!md_in_flight(md))
2375                        break;
2376
2377                if (interruptible == TASK_INTERRUPTIBLE &&
2378                    signal_pending(current)) {
2379                        r = -EINTR;
2380                        break;
2381                }
2382
2383                io_schedule();
2384        }
2385        set_current_state(TASK_RUNNING);
2386
2387        remove_wait_queue(&md->wait, &wait);
2388
2389        return r;
2390}
2391
2392/*
2393 * Process the deferred bios
2394 */
2395static void dm_wq_work(struct work_struct *work)
2396{
2397        struct mapped_device *md = container_of(work, struct mapped_device,
2398                                                work);
2399        struct bio *c;
2400
2401        down_read(&md->io_lock);
2402
2403        while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2404                spin_lock_irq(&md->deferred_lock);
2405                c = bio_list_pop(&md->deferred);
2406                spin_unlock_irq(&md->deferred_lock);
2407
2408                if (!c)
2409                        break;
2410
2411                up_read(&md->io_lock);
2412
2413                if (dm_request_based(md))
2414                        generic_make_request(c);
2415                else
2416                        __split_and_process_bio(md, c);
2417
2418                down_read(&md->io_lock);
2419        }
2420
2421        up_read(&md->io_lock);
2422}
2423
2424static void dm_queue_flush(struct mapped_device *md)
2425{
2426        clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2427        smp_mb__after_clear_bit();
2428        queue_work(md->wq, &md->work);
2429}
2430
2431/*
2432 * Swap in a new table, returning the old one for the caller to destroy.
2433 */
2434struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2435{
2436        struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2437        struct queue_limits limits;
2438        int r;
2439
2440        mutex_lock(&md->suspend_lock);
2441
2442        /* device must be suspended */
2443        if (!dm_suspended_md(md))
2444                goto out;
2445
2446        /*
2447         * If the new table has no data devices, retain the existing limits.
2448         * This helps multipath with queue_if_no_path if all paths disappear,
2449         * then new I/O is queued based on these limits, and then some paths
2450         * reappear.
2451         */
2452        if (dm_table_has_no_data_devices(table)) {
2453                live_map = dm_get_live_table(md);
2454                if (live_map)
2455                        limits = md->queue->limits;
2456                dm_table_put(live_map);
2457        }
2458
2459        if (!live_map) {
2460                r = dm_calculate_queue_limits(table, &limits);
2461                if (r) {
2462                        map = ERR_PTR(r);
2463                        goto out;
2464                }
2465        }
2466
2467        map = __bind(md, table, &limits);
2468
2469out:
2470        mutex_unlock(&md->suspend_lock);
2471        return map;
2472}
2473
2474/*
2475 * Functions to lock and unlock any filesystem running on the
2476 * device.
2477 */
2478static int lock_fs(struct mapped_device *md)
2479{
2480        int r;
2481
2482        WARN_ON(md->frozen_sb);
2483
2484        md->frozen_sb = freeze_bdev(md->bdev);
2485        if (IS_ERR(md->frozen_sb)) {
2486                r = PTR_ERR(md->frozen_sb);
2487                md->frozen_sb = NULL;
2488                return r;
2489        }
2490
2491        set_bit(DMF_FROZEN, &md->flags);
2492
2493        return 0;
2494}
2495
2496static void unlock_fs(struct mapped_device *md)
2497{
2498        if (!test_bit(DMF_FROZEN, &md->flags))
2499                return;
2500
2501        thaw_bdev(md->bdev, md->frozen_sb);
2502        md->frozen_sb = NULL;
2503        clear_bit(DMF_FROZEN, &md->flags);
2504}
2505
2506/*
2507 * We need to be able to change a mapping table under a mounted
2508 * filesystem.  For example we might want to move some data in
2509 * the background.  Before the table can be swapped with
2510 * dm_bind_table, dm_suspend must be called to flush any in
2511 * flight bios and ensure that any further io gets deferred.
2512 */
2513/*
2514 * Suspend mechanism in request-based dm.
2515 *
2516 * 1. Flush all I/Os by lock_fs() if needed.
2517 * 2. Stop dispatching any I/O by stopping the request_queue.
2518 * 3. Wait for all in-flight I/Os to be completed or requeued.
2519 *
2520 * To abort suspend, start the request_queue.
2521 */
2522int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2523{
2524        struct dm_table *map = NULL;
2525        int r = 0;
2526        int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
2527        int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
2528
2529        mutex_lock(&md->suspend_lock);
2530
2531        if (dm_suspended_md(md)) {
2532                r = -EINVAL;
2533                goto out_unlock;
2534        }
2535
2536        map = dm_get_live_table(md);
2537
2538        /*
2539         * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2540         * This flag is cleared before dm_suspend returns.
2541         */
2542        if (noflush)
2543                set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2544
2545        /* This does not get reverted if there's an error later. */
2546        dm_table_presuspend_targets(map);
2547
2548        /*
2549         * Flush I/O to the device.
2550         * Any I/O submitted after lock_fs() may not be flushed.
2551         * noflush takes precedence over do_lockfs.
2552         * (lock_fs() flushes I/Os and waits for them to complete.)
2553         */
2554        if (!noflush && do_lockfs) {
2555                r = lock_fs(md);
2556                if (r)
2557                        goto out;
2558        }
2559
2560        /*
2561         * Here we must make sure that no processes are submitting requests
2562         * to target drivers i.e. no one may be executing
2563         * __split_and_process_bio. This is called from dm_request and
2564         * dm_wq_work.
2565         *
2566         * To get all processes out of __split_and_process_bio in dm_request,
2567         * we take the write lock. To prevent any process from reentering
2568         * __split_and_process_bio from dm_request and quiesce the thread
2569         * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2570         * flush_workqueue(md->wq).
2571         */
2572        down_write(&md->io_lock);
2573        set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2574        up_write(&md->io_lock);
2575
2576        /*
2577         * Stop md->queue before flushing md->wq in case request-based
2578         * dm defers requests to md->wq from md->queue.
2579         */
2580        if (dm_request_based(md))
2581                stop_queue(md->queue);
2582
2583        flush_workqueue(md->wq);
2584
2585        /*
2586         * At this point no more requests are entering target request routines.
2587         * We call dm_wait_for_completion to wait for all existing requests
2588         * to finish.
2589         */
2590        r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
2591
2592        down_write(&md->io_lock);
2593        if (noflush)
2594                clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2595        up_write(&md->io_lock);
2596
2597        /* were we interrupted ? */
2598        if (r < 0) {
2599                dm_queue_flush(md);
2600
2601                if (dm_request_based(md))
2602                        start_queue(md->queue);
2603
2604                unlock_fs(md);
2605                goto out; /* pushback list is already flushed, so skip flush */
2606        }
2607
2608        /*
2609         * If dm_wait_for_completion returned 0, the device is completely
2610         * quiescent now. There is no request-processing activity. All new
2611         * requests are being added to md->deferred list.
2612         */
2613
2614        set_bit(DMF_SUSPENDED, &md->flags);
2615
2616        dm_table_postsuspend_targets(map);
2617
2618out:
2619        dm_table_put(map);
2620
2621out_unlock:
2622        mutex_unlock(&md->suspend_lock);
2623        return r;
2624}
2625
2626int dm_resume(struct mapped_device *md)
2627{
2628        int r = -EINVAL;
2629        struct dm_table *map = NULL;
2630
2631        mutex_lock(&md->suspend_lock);
2632        if (!dm_suspended_md(md))
2633                goto out;
2634
2635        map = dm_get_live_table(md);
2636        if (!map || !dm_table_get_size(map))
2637                goto out;
2638
2639        r = dm_table_resume_targets(map);
2640        if (r)
2641                goto out;
2642
2643        dm_queue_flush(md);
2644
2645        /*
2646         * Flushing deferred I/Os must be done after targets are resumed
2647         * so that mapping of targets can work correctly.
2648         * Request-based dm is queueing the deferred I/Os in its request_queue.
2649         */
2650        if (dm_request_based(md))
2651                start_queue(md->queue);
2652
2653        unlock_fs(md);
2654
2655        clear_bit(DMF_SUSPENDED, &md->flags);
2656
2657        r = 0;
2658out:
2659        dm_table_put(map);
2660        mutex_unlock(&md->suspend_lock);
2661
2662        return r;
2663}
2664
2665/*-----------------------------------------------------------------
2666 * Event notification.
2667 *---------------------------------------------------------------*/
2668int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2669                       unsigned cookie)
2670{
2671        char udev_cookie[DM_COOKIE_LENGTH];
2672        char *envp[] = { udev_cookie, NULL };
2673
2674        if (!cookie)
2675                return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2676        else {
2677                snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2678                         DM_COOKIE_ENV_VAR_NAME, cookie);
2679                return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2680                                          action, envp);
2681        }
2682}
2683
2684uint32_t dm_next_uevent_seq(struct mapped_device *md)
2685{
2686        return atomic_add_return(1, &md->uevent_seq);
2687}
2688
2689uint32_t dm_get_event_nr(struct mapped_device *md)
2690{
2691        return atomic_read(&md->event_nr);
2692}
2693
2694int dm_wait_event(struct mapped_device *md, int event_nr)
2695{
2696        return wait_event_interruptible(md->eventq,
2697                        (event_nr != atomic_read(&md->event_nr)));
2698}
2699
2700void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2701{
2702        unsigned long flags;
2703
2704        spin_lock_irqsave(&md->uevent_lock, flags);
2705        list_add(elist, &md->uevent_list);
2706        spin_unlock_irqrestore(&md->uevent_lock, flags);
2707}
2708
2709/*
2710 * The gendisk is only valid as long as you have a reference
2711 * count on 'md'.
2712 */
2713struct gendisk *dm_disk(struct mapped_device *md)
2714{
2715        return md->disk;
2716}
2717
2718struct kobject *dm_kobject(struct mapped_device *md)
2719{
2720        return &md->kobj;
2721}
2722
2723/*
2724 * struct mapped_device should not be exported outside of dm.c
2725 * so use this check to verify that kobj is part of md structure
2726 */
2727struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2728{
2729        struct mapped_device *md;
2730
2731        md = container_of(kobj, struct mapped_device, kobj);
2732        if (&md->kobj != kobj)
2733                return NULL;
2734
2735        if (test_bit(DMF_FREEING, &md->flags) ||
2736            dm_deleting_md(md))
2737                return NULL;
2738
2739        dm_get(md);
2740        return md;
2741}
2742
2743int dm_suspended_md(struct mapped_device *md)
2744{
2745        return test_bit(DMF_SUSPENDED, &md->flags);
2746}
2747
2748int dm_suspended(struct dm_target *ti)
2749{
2750        return dm_suspended_md(dm_table_get_md(ti->table));
2751}
2752EXPORT_SYMBOL_GPL(dm_suspended);
2753
2754int dm_noflush_suspending(struct dm_target *ti)
2755{
2756        return __noflush_suspending(dm_table_get_md(ti->table));
2757}
2758EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2759
2760struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, unsigned per_bio_data_size)
2761{
2762        struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL);
2763        struct kmem_cache *cachep;
2764        unsigned int pool_size;
2765        unsigned int front_pad;
2766
2767        if (!pools)
2768                return NULL;
2769
2770        if (type == DM_TYPE_BIO_BASED) {
2771                cachep = _io_cache;
2772                pool_size = 16;
2773                front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2774        } else if (type == DM_TYPE_REQUEST_BASED) {
2775                cachep = _rq_tio_cache;
2776                pool_size = MIN_IOS;
2777                front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2778                /* per_bio_data_size is not used. See __bind_mempools(). */
2779                WARN_ON(per_bio_data_size != 0);
2780        } else
2781                goto out;
2782
2783        pools->io_pool = mempool_create_slab_pool(MIN_IOS, cachep);
2784        if (!pools->io_pool)
2785                goto out;
2786
2787        pools->bs = bioset_create(pool_size, front_pad);
2788        if (!pools->bs)
2789                goto out;
2790
2791        if (integrity && bioset_integrity_create(pools->bs, pool_size))
2792                goto out;
2793
2794        return pools;
2795
2796out:
2797        dm_free_md_mempools(pools);
2798
2799        return NULL;
2800}
2801
2802void dm_free_md_mempools(struct dm_md_mempools *pools)
2803{
2804        if (!pools)
2805                return;
2806
2807        if (pools->io_pool)
2808                mempool_destroy(pools->io_pool);
2809
2810        if (pools->bs)
2811                bioset_free(pools->bs);
2812
2813        kfree(pools);
2814}
2815
2816static const struct block_device_operations dm_blk_dops = {
2817        .open = dm_blk_open,
2818        .release = dm_blk_close,
2819        .ioctl = dm_blk_ioctl,
2820        .getgeo = dm_blk_getgeo,
2821        .owner = THIS_MODULE
2822};
2823
2824EXPORT_SYMBOL(dm_get_mapinfo);
2825
2826/*
2827 * module hooks
2828 */
2829module_init(dm_init);
2830module_exit(dm_exit);
2831
2832module_param(major, uint, 0);
2833MODULE_PARM_DESC(major, "The major number of the device mapper");
2834MODULE_DESCRIPTION(DM_NAME " driver");
2835MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2836MODULE_LICENSE("GPL");
2837
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.