linux/drivers/md/dm.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
   3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
   4 *
   5 * This file is released under the GPL.
   6 */
   7
   8#include "dm.h"
   9#include "dm-uevent.h"
  10
  11#include <linux/init.h>
  12#include <linux/module.h>
  13#include <linux/mutex.h>
  14#include <linux/moduleparam.h>
  15#include <linux/blkpg.h>
  16#include <linux/bio.h>
  17#include <linux/mempool.h>
  18#include <linux/slab.h>
  19#include <linux/idr.h>
  20#include <linux/hdreg.h>
  21#include <linux/delay.h>
  22
  23#include <trace/events/block.h>
  24
  25#define DM_MSG_PREFIX "core"
  26
  27#ifdef CONFIG_PRINTK
  28/*
  29 * ratelimit state to be used in DMXXX_LIMIT().
  30 */
  31DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
  32                       DEFAULT_RATELIMIT_INTERVAL,
  33                       DEFAULT_RATELIMIT_BURST);
  34EXPORT_SYMBOL(dm_ratelimit_state);
  35#endif
  36
  37/*
  38 * Cookies are numeric values sent with CHANGE and REMOVE
  39 * uevents while resuming, removing or renaming the device.
  40 */
  41#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
  42#define DM_COOKIE_LENGTH 24
  43
  44static const char *_name = DM_NAME;
  45
  46static unsigned int major = 0;
  47static unsigned int _major = 0;
  48
  49static DEFINE_IDR(_minor_idr);
  50
  51static DEFINE_SPINLOCK(_minor_lock);
  52/*
  53 * For bio-based dm.
  54 * One of these is allocated per bio.
  55 */
  56struct dm_io {
  57        struct mapped_device *md;
  58        int error;
  59        atomic_t io_count;
  60        struct bio *bio;
  61        unsigned long start_time;
  62        spinlock_t endio_lock;
  63};
  64
  65/*
  66 * For request-based dm.
  67 * One of these is allocated per request.
  68 */
  69struct dm_rq_target_io {
  70        struct mapped_device *md;
  71        struct dm_target *ti;
  72        struct request *orig, clone;
  73        int error;
  74        union map_info info;
  75};
  76
  77/*
  78 * For request-based dm - the bio clones we allocate are embedded in these
  79 * structs.
  80 *
  81 * We allocate these with bio_alloc_bioset, using the front_pad parameter when
  82 * the bioset is created - this means the bio has to come at the end of the
  83 * struct.
  84 */
  85struct dm_rq_clone_bio_info {
  86        struct bio *orig;
  87        struct dm_rq_target_io *tio;
  88        struct bio clone;
  89};
  90
  91union map_info *dm_get_mapinfo(struct bio *bio)
  92{
  93        if (bio && bio->bi_private)
  94                return &((struct dm_target_io *)bio->bi_private)->info;
  95        return NULL;
  96}
  97
  98union map_info *dm_get_rq_mapinfo(struct request *rq)
  99{
 100        if (rq && rq->end_io_data)
 101                return &((struct dm_rq_target_io *)rq->end_io_data)->info;
 102        return NULL;
 103}
 104EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo);
 105
 106#define MINOR_ALLOCED ((void *)-1)
 107
 108/*
 109 * Bits for the md->flags field.
 110 */
 111#define DMF_BLOCK_IO_FOR_SUSPEND 0
 112#define DMF_SUSPENDED 1
 113#define DMF_FROZEN 2
 114#define DMF_FREEING 3
 115#define DMF_DELETING 4
 116#define DMF_NOFLUSH_SUSPENDING 5
 117#define DMF_MERGE_IS_OPTIONAL 6
 118
 119/*
 120 * Work processed by per-device workqueue.
 121 */
 122struct mapped_device {
 123        struct rw_semaphore io_lock;
 124        struct mutex suspend_lock;
 125        rwlock_t map_lock;
 126        atomic_t holders;
 127        atomic_t open_count;
 128
 129        unsigned long flags;
 130
 131        struct request_queue *queue;
 132        unsigned type;
 133        /* Protect queue and type against concurrent access. */
 134        struct mutex type_lock;
 135
 136        struct target_type *immutable_target_type;
 137
 138        struct gendisk *disk;
 139        char name[16];
 140
 141        void *interface_ptr;
 142
 143        /*
 144         * A list of ios that arrived while we were suspended.
 145         */
 146        atomic_t pending[2];
 147        wait_queue_head_t wait;
 148        struct work_struct work;
 149        struct bio_list deferred;
 150        spinlock_t deferred_lock;
 151
 152        /*
 153         * Processing queue (flush)
 154         */
 155        struct workqueue_struct *wq;
 156
 157        /*
 158         * The current mapping.
 159         */
 160        struct dm_table *map;
 161
 162        /*
 163         * io objects are allocated from here.
 164         */
 165        mempool_t *io_pool;
 166        mempool_t *tio_pool;
 167
 168        struct bio_set *bs;
 169
 170        /*
 171         * Event handling.
 172         */
 173        atomic_t event_nr;
 174        wait_queue_head_t eventq;
 175        atomic_t uevent_seq;
 176        struct list_head uevent_list;
 177        spinlock_t uevent_lock; /* Protect access to uevent_list */
 178
 179        /*
 180         * freeze/thaw support require holding onto a super block
 181         */
 182        struct super_block *frozen_sb;
 183        struct block_device *bdev;
 184
 185        /* forced geometry settings */
 186        struct hd_geometry geometry;
 187
 188        /* sysfs handle */
 189        struct kobject kobj;
 190
 191        /* zero-length flush that will be cloned and submitted to targets */
 192        struct bio flush_bio;
 193};
 194
 195/*
 196 * For mempools pre-allocation at the table loading time.
 197 */
 198struct dm_md_mempools {
 199        mempool_t *io_pool;
 200        mempool_t *tio_pool;
 201        struct bio_set *bs;
 202};
 203
 204#define MIN_IOS 256
 205static struct kmem_cache *_io_cache;
 206static struct kmem_cache *_rq_tio_cache;
 207
 208/*
 209 * Unused now, and needs to be deleted. But since io_pool is overloaded and it's
 210 * still used for _io_cache, I'm leaving this for a later cleanup
 211 */
 212static struct kmem_cache *_rq_bio_info_cache;
 213
 214static int __init local_init(void)
 215{
 216        int r = -ENOMEM;
 217
 218        /* allocate a slab for the dm_ios */
 219        _io_cache = KMEM_CACHE(dm_io, 0);
 220        if (!_io_cache)
 221                return r;
 222
 223        _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
 224        if (!_rq_tio_cache)
 225                goto out_free_io_cache;
 226
 227        _rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
 228        if (!_rq_bio_info_cache)
 229                goto out_free_rq_tio_cache;
 230
 231        r = dm_uevent_init();
 232        if (r)
 233                goto out_free_rq_bio_info_cache;
 234
 235        _major = major;
 236        r = register_blkdev(_major, _name);
 237        if (r < 0)
 238                goto out_uevent_exit;
 239
 240        if (!_major)
 241                _major = r;
 242
 243        return 0;
 244
 245out_uevent_exit:
 246        dm_uevent_exit();
 247out_free_rq_bio_info_cache:
 248        kmem_cache_destroy(_rq_bio_info_cache);
 249out_free_rq_tio_cache:
 250        kmem_cache_destroy(_rq_tio_cache);
 251out_free_io_cache:
 252        kmem_cache_destroy(_io_cache);
 253
 254        return r;
 255}
 256
 257static void local_exit(void)
 258{
 259        kmem_cache_destroy(_rq_bio_info_cache);
 260        kmem_cache_destroy(_rq_tio_cache);
 261        kmem_cache_destroy(_io_cache);
 262        unregister_blkdev(_major, _name);
 263        dm_uevent_exit();
 264
 265        _major = 0;
 266
 267        DMINFO("cleaned up");
 268}
 269
 270static int (*_inits[])(void) __initdata = {
 271        local_init,
 272        dm_target_init,
 273        dm_linear_init,
 274        dm_stripe_init,
 275        dm_io_init,
 276        dm_kcopyd_init,
 277        dm_interface_init,
 278};
 279
 280static void (*_exits[])(void) = {
 281        local_exit,
 282        dm_target_exit,
 283        dm_linear_exit,
 284        dm_stripe_exit,
 285        dm_io_exit,
 286        dm_kcopyd_exit,
 287        dm_interface_exit,
 288};
 289
 290static int __init dm_init(void)
 291{
 292        const int count = ARRAY_SIZE(_inits);
 293
 294        int r, i;
 295
 296        for (i = 0; i < count; i++) {
 297                r = _inits[i]();
 298                if (r)
 299                        goto bad;
 300        }
 301
 302        return 0;
 303
 304      bad:
 305        while (i--)
 306                _exits[i]();
 307
 308        return r;
 309}
 310
 311static void __exit dm_exit(void)
 312{
 313        int i = ARRAY_SIZE(_exits);
 314
 315        while (i--)
 316                _exits[i]();
 317
 318        /*
 319         * Should be empty by this point.
 320         */
 321        idr_remove_all(&_minor_idr);
 322        idr_destroy(&_minor_idr);
 323}
 324
 325/*
 326 * Block device functions
 327 */
 328int dm_deleting_md(struct mapped_device *md)
 329{
 330        return test_bit(DMF_DELETING, &md->flags);
 331}
 332
 333static int dm_blk_open(struct block_device *bdev, fmode_t mode)
 334{
 335        struct mapped_device *md;
 336
 337        spin_lock(&_minor_lock);
 338
 339        md = bdev->bd_disk->private_data;
 340        if (!md)
 341                goto out;
 342
 343        if (test_bit(DMF_FREEING, &md->flags) ||
 344            dm_deleting_md(md)) {
 345                md = NULL;
 346                goto out;
 347        }
 348
 349        dm_get(md);
 350        atomic_inc(&md->open_count);
 351
 352out:
 353        spin_unlock(&_minor_lock);
 354
 355        return md ? 0 : -ENXIO;
 356}
 357
 358static int dm_blk_close(struct gendisk *disk, fmode_t mode)
 359{
 360        struct mapped_device *md = disk->private_data;
 361
 362        spin_lock(&_minor_lock);
 363
 364        atomic_dec(&md->open_count);
 365        dm_put(md);
 366
 367        spin_unlock(&_minor_lock);
 368
 369        return 0;
 370}
 371
 372int dm_open_count(struct mapped_device *md)
 373{
 374        return atomic_read(&md->open_count);
 375}
 376
 377/*
 378 * Guarantees nothing is using the device before it's deleted.
 379 */
 380int dm_lock_for_deletion(struct mapped_device *md)
 381{
 382        int r = 0;
 383
 384        spin_lock(&_minor_lock);
 385
 386        if (dm_open_count(md))
 387                r = -EBUSY;
 388        else
 389                set_bit(DMF_DELETING, &md->flags);
 390
 391        spin_unlock(&_minor_lock);
 392
 393        return r;
 394}
 395
 396static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 397{
 398        struct mapped_device *md = bdev->bd_disk->private_data;
 399
 400        return dm_get_geometry(md, geo);
 401}
 402
 403static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
 404                        unsigned int cmd, unsigned long arg)
 405{
 406        struct mapped_device *md = bdev->bd_disk->private_data;
 407        struct dm_table *map = dm_get_live_table(md);
 408        struct dm_target *tgt;
 409        int r = -ENOTTY;
 410
 411        if (!map || !dm_table_get_size(map))
 412                goto out;
 413
 414        /* We only support devices that have a single target */
 415        if (dm_table_get_num_targets(map) != 1)
 416                goto out;
 417
 418        tgt = dm_table_get_target(map, 0);
 419
 420        if (dm_suspended_md(md)) {
 421                r = -EAGAIN;
 422                goto out;
 423        }
 424
 425        if (tgt->type->ioctl)
 426                r = tgt->type->ioctl(tgt, cmd, arg);
 427
 428out:
 429        dm_table_put(map);
 430
 431        return r;
 432}
 433
 434static struct dm_io *alloc_io(struct mapped_device *md)
 435{
 436        return mempool_alloc(md->io_pool, GFP_NOIO);
 437}
 438
 439static void free_io(struct mapped_device *md, struct dm_io *io)
 440{
 441        mempool_free(io, md->io_pool);
 442}
 443
 444static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
 445{
 446        bio_put(&tio->clone);
 447}
 448
 449static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md,
 450                                            gfp_t gfp_mask)
 451{
 452        return mempool_alloc(md->tio_pool, gfp_mask);
 453}
 454
 455static void free_rq_tio(struct dm_rq_target_io *tio)
 456{
 457        mempool_free(tio, tio->md->tio_pool);
 458}
 459
 460static int md_in_flight(struct mapped_device *md)
 461{
 462        return atomic_read(&md->pending[READ]) +
 463               atomic_read(&md->pending[WRITE]);
 464}
 465
 466static void start_io_acct(struct dm_io *io)
 467{
 468        struct mapped_device *md = io->md;
 469        int cpu;
 470        int rw = bio_data_dir(io->bio);
 471
 472        io->start_time = jiffies;
 473
 474        cpu = part_stat_lock();
 475        part_round_stats(cpu, &dm_disk(md)->part0);
 476        part_stat_unlock();
 477        atomic_set(&dm_disk(md)->part0.in_flight[rw],
 478                atomic_inc_return(&md->pending[rw]));
 479}
 480
 481static void end_io_acct(struct dm_io *io)
 482{
 483        struct mapped_device *md = io->md;
 484        struct bio *bio = io->bio;
 485        unsigned long duration = jiffies - io->start_time;
 486        int pending, cpu;
 487        int rw = bio_data_dir(bio);
 488
 489        cpu = part_stat_lock();
 490        part_round_stats(cpu, &dm_disk(md)->part0);
 491        part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
 492        part_stat_unlock();
 493
 494        /*
 495         * After this is decremented the bio must not be touched if it is
 496         * a flush.
 497         */
 498        pending = atomic_dec_return(&md->pending[rw]);
 499        atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
 500        pending += atomic_read(&md->pending[rw^0x1]);
 501
 502        /* nudge anyone waiting on suspend queue */
 503        if (!pending)
 504                wake_up(&md->wait);
 505}
 506
 507/*
 508 * Add the bio to the list of deferred io.
 509 */
 510static void queue_io(struct mapped_device *md, struct bio *bio)
 511{
 512        unsigned long flags;
 513
 514        spin_lock_irqsave(&md->deferred_lock, flags);
 515        bio_list_add(&md->deferred, bio);
 516        spin_unlock_irqrestore(&md->deferred_lock, flags);
 517        queue_work(md->wq, &md->work);
 518}
 519
 520/*
 521 * Everyone (including functions in this file), should use this
 522 * function to access the md->map field, and make sure they call
 523 * dm_table_put() when finished.
 524 */
 525struct dm_table *dm_get_live_table(struct mapped_device *md)
 526{
 527        struct dm_table *t;
 528        unsigned long flags;
 529
 530        read_lock_irqsave(&md->map_lock, flags);
 531        t = md->map;
 532        if (t)
 533                dm_table_get(t);
 534        read_unlock_irqrestore(&md->map_lock, flags);
 535
 536        return t;
 537}
 538
 539/*
 540 * Get the geometry associated with a dm device
 541 */
 542int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
 543{
 544        *geo = md->geometry;
 545
 546        return 0;
 547}
 548
 549/*
 550 * Set the geometry of a device.
 551 */
 552int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
 553{
 554        sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
 555
 556        if (geo->start > sz) {
 557                DMWARN("Start sector is beyond the geometry limits.");
 558                return -EINVAL;
 559        }
 560
 561        md->geometry = *geo;
 562
 563        return 0;
 564}
 565
 566/*-----------------------------------------------------------------
 567 * CRUD START:
 568 *   A more elegant soln is in the works that uses the queue
 569 *   merge fn, unfortunately there are a couple of changes to
 570 *   the block layer that I want to make for this.  So in the
 571 *   interests of getting something for people to use I give
 572 *   you this clearly demarcated crap.
 573 *---------------------------------------------------------------*/
 574
 575static int __noflush_suspending(struct mapped_device *md)
 576{
 577        return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
 578}
 579
 580/*
 581 * Decrements the number of outstanding ios that a bio has been
 582 * cloned into, completing the original io if necc.
 583 */
 584static void dec_pending(struct dm_io *io, int error)
 585{
 586        unsigned long flags;
 587        int io_error;
 588        struct bio *bio;
 589        struct mapped_device *md = io->md;
 590
 591        /* Push-back supersedes any I/O errors */
 592        if (unlikely(error)) {
 593                spin_lock_irqsave(&io->endio_lock, flags);
 594                if (!(io->error > 0 && __noflush_suspending(md)))
 595                        io->error = error;
 596                spin_unlock_irqrestore(&io->endio_lock, flags);
 597        }
 598
 599        if (atomic_dec_and_test(&io->io_count)) {
 600                if (io->error == DM_ENDIO_REQUEUE) {
 601                        /*
 602                         * Target requested pushing back the I/O.
 603                         */
 604                        spin_lock_irqsave(&md->deferred_lock, flags);
 605                        if (__noflush_suspending(md))
 606                                bio_list_add_head(&md->deferred, io->bio);
 607                        else
 608                                /* noflush suspend was interrupted. */
 609                                io->error = -EIO;
 610                        spin_unlock_irqrestore(&md->deferred_lock, flags);
 611                }
 612
 613                io_error = io->error;
 614                bio = io->bio;
 615                end_io_acct(io);
 616                free_io(md, io);
 617
 618                if (io_error == DM_ENDIO_REQUEUE)
 619                        return;
 620
 621                if ((bio->bi_rw & REQ_FLUSH) && bio->bi_size) {
 622                        /*
 623                         * Preflush done for flush with data, reissue
 624                         * without REQ_FLUSH.
 625                         */
 626                        bio->bi_rw &= ~REQ_FLUSH;
 627                        queue_io(md, bio);
 628                } else {
 629                        /* done with normal IO or empty flush */
 630                        trace_block_bio_complete(md->queue, bio, io_error);
 631                        bio_endio(bio, io_error);
 632                }
 633        }
 634}
 635
 636static void clone_endio(struct bio *bio, int error)
 637{
 638        int r = 0;
 639        struct dm_target_io *tio = bio->bi_private;
 640        struct dm_io *io = tio->io;
 641        struct mapped_device *md = tio->io->md;
 642        dm_endio_fn endio = tio->ti->type->end_io;
 643
 644        if (!bio_flagged(bio, BIO_UPTODATE) && !error)
 645                error = -EIO;
 646
 647        if (endio) {
 648                r = endio(tio->ti, bio, error);
 649                if (r < 0 || r == DM_ENDIO_REQUEUE)
 650                        /*
 651                         * error and requeue request are handled
 652                         * in dec_pending().
 653                         */
 654                        error = r;
 655                else if (r == DM_ENDIO_INCOMPLETE)
 656                        /* The target will handle the io */
 657                        return;
 658                else if (r) {
 659                        DMWARN("unimplemented target endio return value: %d", r);
 660                        BUG();
 661                }
 662        }
 663
 664        free_tio(md, tio);
 665        dec_pending(io, error);
 666}
 667
 668/*
 669 * Partial completion handling for request-based dm
 670 */
 671static void end_clone_bio(struct bio *clone, int error)
 672{
 673        struct dm_rq_clone_bio_info *info = clone->bi_private;
 674        struct dm_rq_target_io *tio = info->tio;
 675        struct bio *bio = info->orig;
 676        unsigned int nr_bytes = info->orig->bi_size;
 677
 678        bio_put(clone);
 679
 680        if (tio->error)
 681                /*
 682                 * An error has already been detected on the request.
 683                 * Once error occurred, just let clone->end_io() handle
 684                 * the remainder.
 685                 */
 686                return;
 687        else if (error) {
 688                /*
 689                 * Don't notice the error to the upper layer yet.
 690                 * The error handling decision is made by the target driver,
 691                 * when the request is completed.
 692                 */
 693                tio->error = error;
 694                return;
 695        }
 696
 697        /*
 698         * I/O for the bio successfully completed.
 699         * Notice the data completion to the upper layer.
 700         */
 701
 702        /*
 703         * bios are processed from the head of the list.
 704         * So the completing bio should always be rq->bio.
 705         * If it's not, something wrong is happening.
 706         */
 707        if (tio->orig->bio != bio)
 708                DMERR("bio completion is going in the middle of the request");
 709
 710        /*
 711         * Update the original request.
 712         * Do not use blk_end_request() here, because it may complete
 713         * the original request before the clone, and break the ordering.
 714         */
 715        blk_update_request(tio->orig, 0, nr_bytes);
 716}
 717
 718/*
 719 * Don't touch any member of the md after calling this function because
 720 * the md may be freed in dm_put() at the end of this function.
 721 * Or do dm_get() before calling this function and dm_put() later.
 722 */
 723static void rq_completed(struct mapped_device *md, int rw, int run_queue)
 724{
 725        atomic_dec(&md->pending[rw]);
 726
 727        /* nudge anyone waiting on suspend queue */
 728        if (!md_in_flight(md))
 729                wake_up(&md->wait);
 730
 731        /*
 732         * Run this off this callpath, as drivers could invoke end_io while
 733         * inside their request_fn (and holding the queue lock). Calling
 734         * back into ->request_fn() could deadlock attempting to grab the
 735         * queue lock again.
 736         */
 737        if (run_queue)
 738                blk_run_queue_async(md->queue);
 739
 740        /*
 741         * dm_put() must be at the end of this function. See the comment above
 742         */
 743        dm_put(md);
 744}
 745
 746static void free_rq_clone(struct request *clone)
 747{
 748        struct dm_rq_target_io *tio = clone->end_io_data;
 749
 750        blk_rq_unprep_clone(clone);
 751        free_rq_tio(tio);
 752}
 753
 754/*
 755 * Complete the clone and the original request.
 756 * Must be called without queue lock.
 757 */
 758static void dm_end_request(struct request *clone, int error)
 759{
 760        int rw = rq_data_dir(clone);
 761        struct dm_rq_target_io *tio = clone->end_io_data;
 762        struct mapped_device *md = tio->md;
 763        struct request *rq = tio->orig;
 764
 765        if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
 766                rq->errors = clone->errors;
 767                rq->resid_len = clone->resid_len;
 768
 769                if (rq->sense)
 770                        /*
 771                         * We are using the sense buffer of the original
 772                         * request.
 773                         * So setting the length of the sense data is enough.
 774                         */
 775                        rq->sense_len = clone->sense_len;
 776        }
 777
 778        free_rq_clone(clone);
 779        blk_end_request_all(rq, error);
 780        rq_completed(md, rw, true);
 781}
 782
 783static void dm_unprep_request(struct request *rq)
 784{
 785        struct request *clone = rq->special;
 786
 787        rq->special = NULL;
 788        rq->cmd_flags &= ~REQ_DONTPREP;
 789
 790        free_rq_clone(clone);
 791}
 792
 793/*
 794 * Requeue the original request of a clone.
 795 */
 796void dm_requeue_unmapped_request(struct request *clone)
 797{
 798        int rw = rq_data_dir(clone);
 799        struct dm_rq_target_io *tio = clone->end_io_data;
 800        struct mapped_device *md = tio->md;
 801        struct request *rq = tio->orig;
 802        struct request_queue *q = rq->q;
 803        unsigned long flags;
 804
 805        dm_unprep_request(rq);
 806
 807        spin_lock_irqsave(q->queue_lock, flags);
 808        blk_requeue_request(q, rq);
 809        spin_unlock_irqrestore(q->queue_lock, flags);
 810
 811        rq_completed(md, rw, 0);
 812}
 813EXPORT_SYMBOL_GPL(dm_requeue_unmapped_request);
 814
 815static void __stop_queue(struct request_queue *q)
 816{
 817        blk_stop_queue(q);
 818}
 819
 820static void stop_queue(struct request_queue *q)
 821{
 822        unsigned long flags;
 823
 824        spin_lock_irqsave(q->queue_lock, flags);
 825        __stop_queue(q);
 826        spin_unlock_irqrestore(q->queue_lock, flags);
 827}
 828
 829static void __start_queue(struct request_queue *q)
 830{
 831        if (blk_queue_stopped(q))
 832                blk_start_queue(q);
 833}
 834
 835static void start_queue(struct request_queue *q)
 836{
 837        unsigned long flags;
 838
 839        spin_lock_irqsave(q->queue_lock, flags);
 840        __start_queue(q);
 841        spin_unlock_irqrestore(q->queue_lock, flags);
 842}
 843
 844static void dm_done(struct request *clone, int error, bool mapped)
 845{
 846        int r = error;
 847        struct dm_rq_target_io *tio = clone->end_io_data;
 848        dm_request_endio_fn rq_end_io = NULL;
 849
 850        if (tio->ti) {
 851                rq_end_io = tio->ti->type->rq_end_io;
 852
 853                if (mapped && rq_end_io)
 854                        r = rq_end_io(tio->ti, clone, error, &tio->info);
 855        }
 856
 857        if (r <= 0)
 858                /* The target wants to complete the I/O */
 859                dm_end_request(clone, r);
 860        else if (r == DM_ENDIO_INCOMPLETE)
 861                /* The target will handle the I/O */
 862                return;
 863        else if (r == DM_ENDIO_REQUEUE)
 864                /* The target wants to requeue the I/O */
 865                dm_requeue_unmapped_request(clone);
 866        else {
 867                DMWARN("unimplemented target endio return value: %d", r);
 868                BUG();
 869        }
 870}
 871
 872/*
 873 * Request completion handler for request-based dm
 874 */
 875static void dm_softirq_done(struct request *rq)
 876{
 877        bool mapped = true;
 878        struct request *clone = rq->completion_data;
 879        struct dm_rq_target_io *tio = clone->end_io_data;
 880
 881        if (rq->cmd_flags & REQ_FAILED)
 882                mapped = false;
 883
 884        dm_done(clone, tio->error, mapped);
 885}
 886
 887/*
 888 * Complete the clone and the original request with the error status
 889 * through softirq context.
 890 */
 891static void dm_complete_request(struct request *clone, int error)
 892{
 893        struct dm_rq_target_io *tio = clone->end_io_data;
 894        struct request *rq = tio->orig;
 895
 896        tio->error = error;
 897        rq->completion_data = clone;
 898        blk_complete_request(rq);
 899}
 900
 901/*
 902 * Complete the not-mapped clone and the original request with the error status
 903 * through softirq context.
 904 * Target's rq_end_io() function isn't called.
 905 * This may be used when the target's map_rq() function fails.
 906 */
 907void dm_kill_unmapped_request(struct request *clone, int error)
 908{
 909        struct dm_rq_target_io *tio = clone->end_io_data;
 910        struct request *rq = tio->orig;
 911
 912        rq->cmd_flags |= REQ_FAILED;
 913        dm_complete_request(clone, error);
 914}
 915EXPORT_SYMBOL_GPL(dm_kill_unmapped_request);
 916
 917/*
 918 * Called with the queue lock held
 919 */
 920static void end_clone_request(struct request *clone, int error)
 921{
 922        /*
 923         * For just cleaning up the information of the queue in which
 924         * the clone was dispatched.
 925         * The clone is *NOT* freed actually here because it is alloced from
 926         * dm own mempool and REQ_ALLOCED isn't set in clone->cmd_flags.
 927         */
 928        __blk_put_request(clone->q, clone);
 929
 930        /*
 931         * Actual request completion is done in a softirq context which doesn't
 932         * hold the queue lock.  Otherwise, deadlock could occur because:
 933         *     - another request may be submitted by the upper level driver
 934         *       of the stacking during the completion
 935         *     - the submission which requires queue lock may be done
 936         *       against this queue
 937         */
 938        dm_complete_request(clone, error);
 939}
 940
 941/*
 942 * Return maximum size of I/O possible at the supplied sector up to the current
 943 * target boundary.
 944 */
 945static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
 946{
 947        sector_t target_offset = dm_target_offset(ti, sector);
 948
 949        return ti->len - target_offset;
 950}
 951
 952static sector_t max_io_len(sector_t sector, struct dm_target *ti)
 953{
 954        sector_t len = max_io_len_target_boundary(sector, ti);
 955        sector_t offset, max_len;
 956
 957        /*
 958         * Does the target need to split even further?
 959         */
 960        if (ti->max_io_len) {
 961                offset = dm_target_offset(ti, sector);
 962                if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
 963                        max_len = sector_div(offset, ti->max_io_len);
 964                else
 965                        max_len = offset & (ti->max_io_len - 1);
 966                max_len = ti->max_io_len - max_len;
 967
 968                if (len > max_len)
 969                        len = max_len;
 970        }
 971
 972        return len;
 973}
 974
 975int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
 976{
 977        if (len > UINT_MAX) {
 978                DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
 979                      (unsigned long long)len, UINT_MAX);
 980                ti->error = "Maximum size of target IO is too large";
 981                return -EINVAL;
 982        }
 983
 984        ti->max_io_len = (uint32_t) len;
 985
 986        return 0;
 987}
 988EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
 989
 990static void __map_bio(struct dm_target *ti, struct dm_target_io *tio)
 991{
 992        int r;
 993        sector_t sector;
 994        struct mapped_device *md;
 995        struct bio *clone = &tio->clone;
 996
 997        clone->bi_end_io = clone_endio;
 998        clone->bi_private = tio;
 999
1000        /*
1001         * Map the clone.  If r == 0 we don't need to do
1002         * anything, the target has assumed ownership of
1003         * this io.
1004         */
1005        atomic_inc(&tio->io->io_count);
1006        sector = clone->bi_sector;
1007        r = ti->type->map(ti, clone);
1008        if (r == DM_MAPIO_REMAPPED) {
1009                /* the bio has been remapped so dispatch it */
1010
1011                trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
1012                                      tio->io->bio->bi_bdev->bd_dev, sector);
1013
1014                generic_make_request(clone);
1015        } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
1016                /* error the io and bail out, or requeue it if needed */
1017                md = tio->io->md;
1018                dec_pending(tio->io, r);
1019                free_tio(md, tio);
1020        } else if (r) {
1021                DMWARN("unimplemented target map return value: %d", r);
1022                BUG();
1023        }
1024}
1025
1026struct clone_info {
1027        struct mapped_device *md;
1028        struct dm_table *map;
1029        struct bio *bio;
1030        struct dm_io *io;
1031        sector_t sector;
1032        sector_t sector_count;
1033        unsigned short idx;
1034};
1035
1036/*
1037 * Creates a little bio that just does part of a bvec.
1038 */
1039static void split_bvec(struct dm_target_io *tio, struct bio *bio,
1040                       sector_t sector, unsigned short idx, unsigned int offset,
1041                       unsigned int len, struct bio_set *bs)
1042{
1043        struct bio *clone = &tio->clone;
1044        struct bio_vec *bv = bio->bi_io_vec + idx;
1045
1046        *clone->bi_io_vec = *bv;
1047
1048        clone->bi_sector = sector;
1049        clone->bi_bdev = bio->bi_bdev;
1050        clone->bi_rw = bio->bi_rw;
1051        clone->bi_vcnt = 1;
1052        clone->bi_size = to_bytes(len);
1053        clone->bi_io_vec->bv_offset = offset;
1054        clone->bi_io_vec->bv_len = clone->bi_size;
1055        clone->bi_flags |= 1 << BIO_CLONED;
1056
1057        if (bio_integrity(bio)) {
1058                bio_integrity_clone(clone, bio, GFP_NOIO);
1059                bio_integrity_trim(clone,
1060                                   bio_sector_offset(bio, idx, offset), len);
1061        }
1062}
1063
1064/*
1065 * Creates a bio that consists of range of complete bvecs.
1066 */
1067static void clone_bio(struct dm_target_io *tio, struct bio *bio,
1068                      sector_t sector, unsigned short idx,
1069                      unsigned short bv_count, unsigned int len,
1070                      struct bio_set *bs)
1071{
1072        struct bio *clone = &tio->clone;
1073
1074        __bio_clone(clone, bio);
1075        clone->bi_sector = sector;
1076        clone->bi_idx = idx;
1077        clone->bi_vcnt = idx + bv_count;
1078        clone->bi_size = to_bytes(len);
1079        clone->bi_flags &= ~(1 << BIO_SEG_VALID);
1080
1081        if (bio_integrity(bio)) {
1082                bio_integrity_clone(clone, bio, GFP_NOIO);
1083
1084                if (idx != bio->bi_idx || clone->bi_size < bio->bi_size)
1085                        bio_integrity_trim(clone,
1086                                           bio_sector_offset(bio, idx, 0), len);
1087        }
1088}
1089
1090static struct dm_target_io *alloc_tio(struct clone_info *ci,
1091                                      struct dm_target *ti, int nr_iovecs)
1092{
1093        struct dm_target_io *tio;
1094        struct bio *clone;
1095
1096        clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, ci->md->bs);
1097        tio = container_of(clone, struct dm_target_io, clone);
1098
1099        tio->io = ci->io;
1100        tio->ti = ti;
1101        memset(&tio->info, 0, sizeof(tio->info));
1102        tio->target_request_nr = 0;
1103
1104        return tio;
1105}
1106
1107static void __issue_target_request(struct clone_info *ci, struct dm_target *ti,
1108                                   unsigned request_nr, sector_t len)
1109{
1110        struct dm_target_io *tio = alloc_tio(ci, ti, ci->bio->bi_max_vecs);
1111        struct bio *clone = &tio->clone;
1112
1113        tio->target_request_nr = request_nr;
1114
1115        /*
1116         * Discard requests require the bio's inline iovecs be initialized.
1117         * ci->bio->bi_max_vecs is BIO_INLINE_VECS anyway, for both flush
1118         * and discard, so no need for concern about wasted bvec allocations.
1119         */
1120
1121         __bio_clone(clone, ci->bio);
1122        if (len) {
1123                clone->bi_sector = ci->sector;
1124                clone->bi_size = to_bytes(len);
1125        }
1126
1127        __map_bio(ti, tio);
1128}
1129
1130static void __issue_target_requests(struct clone_info *ci, struct dm_target *ti,
1131                                    unsigned num_requests, sector_t len)
1132{
1133        unsigned request_nr;
1134
1135        for (request_nr = 0; request_nr < num_requests; request_nr++)
1136                __issue_target_request(ci, ti, request_nr, len);
1137}
1138
1139static int __clone_and_map_empty_flush(struct clone_info *ci)
1140{
1141        unsigned target_nr = 0;
1142        struct dm_target *ti;
1143
1144        BUG_ON(bio_has_data(ci->bio));
1145        while ((ti = dm_table_get_target(ci->map, target_nr++)))
1146                __issue_target_requests(ci, ti, ti->num_flush_requests, 0);
1147
1148        return 0;
1149}
1150
1151/*
1152 * Perform all io with a single clone.
1153 */
1154static void __clone_and_map_simple(struct clone_info *ci, struct dm_target *ti)
1155{
1156        struct bio *bio = ci->bio;
1157        struct dm_target_io *tio;
1158
1159        tio = alloc_tio(ci, ti, bio->bi_max_vecs);
1160        clone_bio(tio, bio, ci->sector, ci->idx, bio->bi_vcnt - ci->idx,
1161                  ci->sector_count, ci->md->bs);
1162        __map_bio(ti, tio);
1163        ci->sector_count = 0;
1164}
1165
1166typedef unsigned (*get_num_requests_fn)(struct dm_target *ti);
1167
1168static unsigned get_num_discard_requests(struct dm_target *ti)
1169{
1170        return ti->num_discard_requests;
1171}
1172
1173static unsigned get_num_write_same_requests(struct dm_target *ti)
1174{
1175        return ti->num_write_same_requests;
1176}
1177
1178typedef bool (*is_split_required_fn)(struct dm_target *ti);
1179
1180static bool is_split_required_for_discard(struct dm_target *ti)
1181{
1182        return ti->split_discard_requests;
1183}
1184
1185static int __clone_and_map_changing_extent_only(struct clone_info *ci,
1186                                                get_num_requests_fn get_num_requests,
1187                                                is_split_required_fn is_split_required)
1188{
1189        struct dm_target *ti;
1190        sector_t len;
1191        unsigned num_requests;
1192
1193        do {
1194                ti = dm_table_find_target(ci->map, ci->sector);
1195                if (!dm_target_is_valid(ti))
1196                        return -EIO;
1197
1198                /*
1199                 * Even though the device advertised support for this type of
1200                 * request, that does not mean every target supports it, and
1201                 * reconfiguration might also have changed that since the
1202                 * check was performed.
1203                 */
1204                num_requests = get_num_requests ? get_num_requests(ti) : 0;
1205                if (!num_requests)
1206                        return -EOPNOTSUPP;
1207
1208                if (is_split_required && !is_split_required(ti))
1209                        len = min(ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1210                else
1211                        len = min(ci->sector_count, max_io_len(ci->sector, ti));
1212
1213                __issue_target_requests(ci, ti, num_requests, len);
1214
1215                ci->sector += len;
1216        } while (ci->sector_count -= len);
1217
1218        return 0;
1219}
1220
1221static int __clone_and_map_discard(struct clone_info *ci)
1222{
1223        return __clone_and_map_changing_extent_only(ci, get_num_discard_requests,
1224                                                    is_split_required_for_discard);
1225}
1226
1227static int __clone_and_map_write_same(struct clone_info *ci)
1228{
1229        return __clone_and_map_changing_extent_only(ci, get_num_write_same_requests, NULL);
1230}
1231
1232static int __clone_and_map(struct clone_info *ci)
1233{
1234        struct bio *bio = ci->bio;
1235        struct dm_target *ti;
1236        sector_t len = 0, max;
1237        struct dm_target_io *tio;
1238
1239        if (unlikely(bio->bi_rw & REQ_DISCARD))
1240                return __clone_and_map_discard(ci);
1241        else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
1242                return __clone_and_map_write_same(ci);
1243
1244        ti = dm_table_find_target(ci->map, ci->sector);
1245        if (!dm_target_is_valid(ti))
1246                return -EIO;
1247
1248        max = max_io_len(ci->sector, ti);
1249
1250        if (ci->sector_count <= max) {
1251                /*
1252                 * Optimise for the simple case where we can do all of
1253                 * the remaining io with a single clone.
1254                 */
1255                __clone_and_map_simple(ci, ti);
1256
1257        } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
1258                /*
1259                 * There are some bvecs that don't span targets.
1260                 * Do as many of these as possible.
1261                 */
1262                int i;
1263                sector_t remaining = max;
1264                sector_t bv_len;
1265
1266                for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
1267                        bv_len = to_sector(bio->bi_io_vec[i].bv_len);
1268
1269                        if (bv_len > remaining)
1270                                break;
1271
1272                        remaining -= bv_len;
1273                        len += bv_len;
1274                }
1275
1276                tio = alloc_tio(ci, ti, bio->bi_max_vecs);
1277                clone_bio(tio, bio, ci->sector, ci->idx, i - ci->idx, len,
1278                          ci->md->bs);
1279                __map_bio(ti, tio);
1280
1281                ci->sector += len;
1282                ci->sector_count -= len;
1283                ci->idx = i;
1284
1285        } else {
1286                /*
1287                 * Handle a bvec that must be split between two or more targets.
1288                 */
1289                struct bio_vec *bv = bio->bi_io_vec + ci->idx;
1290                sector_t remaining = to_sector(bv->bv_len);
1291                unsigned int offset = 0;
1292
1293                do {
1294                        if (offset) {
1295                                ti = dm_table_find_target(ci->map, ci->sector);
1296                                if (!dm_target_is_valid(ti))
1297                                        return -EIO;
1298
1299                                max = max_io_len(ci->sector, ti);
1300                        }
1301
1302                        len = min(remaining, max);
1303
1304                        tio = alloc_tio(ci, ti, 1);
1305                        split_bvec(tio, bio, ci->sector, ci->idx,
1306                                   bv->bv_offset + offset, len, ci->md->bs);
1307
1308                        __map_bio(ti, tio);
1309
1310                        ci->sector += len;
1311                        ci->sector_count -= len;
1312                        offset += to_bytes(len);
1313                } while (remaining -= len);
1314
1315                ci->idx++;
1316        }
1317
1318        return 0;
1319}
1320
1321/*
1322 * Split the bio into several clones and submit it to targets.
1323 */
1324static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
1325{
1326        struct clone_info ci;
1327        int error = 0;
1328
1329        ci.map = dm_get_live_table(md);
1330        if (unlikely(!ci.map)) {
1331                bio_io_error(bio);
1332                return;
1333        }
1334
1335        ci.md = md;
1336        ci.io = alloc_io(md);
1337        ci.io->error = 0;
1338        atomic_set(&ci.io->io_count, 1);
1339        ci.io->bio = bio;
1340        ci.io->md = md;
1341        spin_lock_init(&ci.io->endio_lock);
1342        ci.sector = bio->bi_sector;
1343        ci.idx = bio->bi_idx;
1344
1345        start_io_acct(ci.io);
1346        if (bio->bi_rw & REQ_FLUSH) {
1347                ci.bio = &ci.md->flush_bio;
1348                ci.sector_count = 0;
1349                error = __clone_and_map_empty_flush(&ci);
1350                /* dec_pending submits any data associated with flush */
1351        } else {
1352                ci.bio = bio;
1353                ci.sector_count = bio_sectors(bio);
1354                while (ci.sector_count && !error)
1355                        error = __clone_and_map(&ci);
1356        }
1357
1358        /* drop the extra reference count */
1359        dec_pending(ci.io, error);
1360        dm_table_put(ci.map);
1361}
1362/*-----------------------------------------------------------------
1363 * CRUD END
1364 *---------------------------------------------------------------*/
1365
1366static int dm_merge_bvec(struct request_queue *q,
1367                         struct bvec_merge_data *bvm,
1368                         struct bio_vec *biovec)
1369{
1370        struct mapped_device *md = q->queuedata;
1371        struct dm_table *map = dm_get_live_table(md);
1372        struct dm_target *ti;
1373        sector_t max_sectors;
1374        int max_size = 0;
1375
1376        if (unlikely(!map))
1377                goto out;
1378
1379        ti = dm_table_find_target(map, bvm->bi_sector);
1380        if (!dm_target_is_valid(ti))
1381                goto out_table;
1382
1383        /*
1384         * Find maximum amount of I/O that won't need splitting
1385         */
1386        max_sectors = min(max_io_len(bvm->bi_sector, ti),
1387                          (sector_t) BIO_MAX_SECTORS);
1388        max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1389        if (max_size < 0)
1390                max_size = 0;
1391
1392        /*
1393         * merge_bvec_fn() returns number of bytes
1394         * it can accept at this offset
1395         * max is precomputed maximal io size
1396         */
1397        if (max_size && ti->type->merge)
1398                max_size = ti->type->merge(ti, bvm, biovec, max_size);
1399        /*
1400         * If the target doesn't support merge method and some of the devices
1401         * provided their merge_bvec method (we know this by looking at
1402         * queue_max_hw_sectors), then we can't allow bios with multiple vector
1403         * entries.  So always set max_size to 0, and the code below allows
1404         * just one page.
1405         */
1406        else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
1407
1408                max_size = 0;
1409
1410out_table:
1411        dm_table_put(map);
1412
1413out:
1414        /*
1415         * Always allow an entire first page
1416         */
1417        if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1418                max_size = biovec->bv_len;
1419
1420        return max_size;
1421}
1422
1423/*
1424 * The request function that just remaps the bio built up by
1425 * dm_merge_bvec.
1426 */
1427static void _dm_request(struct request_queue *q, struct bio *bio)
1428{
1429        int rw = bio_data_dir(bio);
1430        struct mapped_device *md = q->queuedata;
1431        int cpu;
1432
1433        down_read(&md->io_lock);
1434
1435        cpu = part_stat_lock();
1436        part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
1437        part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
1438        part_stat_unlock();
1439
1440        /* if we're suspended, we have to queue this io for later */
1441        if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1442                up_read(&md->io_lock);
1443
1444                if (bio_rw(bio) != READA)
1445                        queue_io(md, bio);
1446                else
1447                        bio_io_error(bio);
1448                return;
1449        }
1450
1451        __split_and_process_bio(md, bio);
1452        up_read(&md->io_lock);
1453        return;
1454}
1455
1456static int dm_request_based(struct mapped_device *md)
1457{
1458        return blk_queue_stackable(md->queue);
1459}
1460
1461static void dm_request(struct request_queue *q, struct bio *bio)
1462{
1463        struct mapped_device *md = q->queuedata;
1464
1465        if (dm_request_based(md))
1466                blk_queue_bio(q, bio);
1467        else
1468                _dm_request(q, bio);
1469}
1470
1471void dm_dispatch_request(struct request *rq)
1472{
1473        int r;
1474
1475        if (blk_queue_io_stat(rq->q))
1476                rq->cmd_flags |= REQ_IO_STAT;
1477
1478        rq->start_time = jiffies;
1479        r = blk_insert_cloned_request(rq->q, rq);
1480        if (r)
1481                dm_complete_request(rq, r);
1482}
1483EXPORT_SYMBOL_GPL(dm_dispatch_request);
1484
1485static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
1486                                 void *data)
1487{
1488        struct dm_rq_target_io *tio = data;
1489        struct dm_rq_clone_bio_info *info =
1490                container_of(bio, struct dm_rq_clone_bio_info, clone);
1491
1492        info->orig = bio_orig;
1493        info->tio = tio;
1494        bio->bi_end_io = end_clone_bio;
1495        bio->bi_private = info;
1496
1497        return 0;
1498}
1499
1500static int setup_clone(struct request *clone, struct request *rq,
1501                       struct dm_rq_target_io *tio)
1502{
1503        int r;
1504
1505        r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,
1506                              dm_rq_bio_constructor, tio);
1507        if (r)
1508                return r;
1509
1510        clone->cmd = rq->cmd;
1511        clone->cmd_len = rq->cmd_len;
1512        clone->sense = rq->sense;
1513        clone->buffer = rq->buffer;
1514        clone->end_io = end_clone_request;
1515        clone->end_io_data = tio;
1516
1517        return 0;
1518}
1519
1520static struct request *clone_rq(struct request *rq, struct mapped_device *md,
1521                                gfp_t gfp_mask)
1522{
1523        struct request *clone;
1524        struct dm_rq_target_io *tio;
1525
1526        tio = alloc_rq_tio(md, gfp_mask);
1527        if (!tio)
1528                return NULL;
1529
1530        tio->md = md;
1531        tio->ti = NULL;
1532        tio->orig = rq;
1533        tio->error = 0;
1534        memset(&tio->info, 0, sizeof(tio->info));
1535
1536        clone = &tio->clone;
1537        if (setup_clone(clone, rq, tio)) {
1538                /* -ENOMEM */
1539                free_rq_tio(tio);
1540                return NULL;
1541        }
1542
1543        return clone;
1544}
1545
1546/*
1547 * Called with the queue lock held.
1548 */
1549static int dm_prep_fn(struct request_queue *q, struct request *rq)
1550{
1551        struct mapped_device *md = q->queuedata;
1552        struct request *clone;
1553
1554        if (unlikely(rq->special)) {
1555                DMWARN("Already has something in rq->special.");
1556                return BLKPREP_KILL;
1557        }
1558
1559        clone = clone_rq(rq, md, GFP_ATOMIC);
1560        if (!clone)
1561                return BLKPREP_DEFER;
1562
1563        rq->special = clone;
1564        rq->cmd_flags |= REQ_DONTPREP;
1565
1566        return BLKPREP_OK;
1567}
1568
1569/*
1570 * Returns:
1571 * 0  : the request has been processed (not requeued)
1572 * !0 : the request has been requeued
1573 */
1574static int map_request(struct dm_target *ti, struct request *clone,
1575                       struct mapped_device *md)
1576{
1577        int r, requeued = 0;
1578        struct dm_rq_target_io *tio = clone->end_io_data;
1579
1580        tio->ti = ti;
1581        r = ti->type->map_rq(ti, clone, &tio->info);
1582        switch (r) {
1583        case DM_MAPIO_SUBMITTED:
1584                /* The target has taken the I/O to submit by itself later */
1585                break;
1586        case DM_MAPIO_REMAPPED:
1587                /* The target has remapped the I/O so dispatch it */
1588                trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
1589                                     blk_rq_pos(tio->orig));
1590                dm_dispatch_request(clone);
1591                break;
1592        case DM_MAPIO_REQUEUE:
1593                /* The target wants to requeue the I/O */
1594                dm_requeue_unmapped_request(clone);
1595                requeued = 1;
1596                break;
1597        default:
1598                if (r > 0) {
1599                        DMWARN("unimplemented target map return value: %d", r);
1600                        BUG();
1601                }
1602
1603                /* The target wants to complete the I/O */
1604                dm_kill_unmapped_request(clone, r);
1605                break;
1606        }
1607
1608        return requeued;
1609}
1610
1611static struct request *dm_start_request(struct mapped_device *md, struct request *orig)
1612{
1613        struct request *clone;
1614
1615        blk_start_request(orig);
1616        clone = orig->special;
1617        atomic_inc(&md->pending[rq_data_dir(clone)]);
1618
1619        /*
1620         * Hold the md reference here for the in-flight I/O.
1621         * We can't rely on the reference count by device opener,
1622         * because the device may be closed during the request completion
1623         * when all bios are completed.
1624         * See the comment in rq_completed() too.
1625         */
1626        dm_get(md);
1627
1628        return clone;
1629}
1630
1631/*
1632 * q->request_fn for request-based dm.
1633 * Called with the queue lock held.
1634 */
1635static void dm_request_fn(struct request_queue *q)
1636{
1637        struct mapped_device *md = q->queuedata;
1638        struct dm_table *map = dm_get_live_table(md);
1639        struct dm_target *ti;
1640        struct request *rq, *clone;
1641        sector_t pos;
1642
1643        /*
1644         * For suspend, check blk_queue_stopped() and increment
1645         * ->pending within a single queue_lock not to increment the
1646         * number of in-flight I/Os after the queue is stopped in
1647         * dm_suspend().
1648         */
1649        while (!blk_queue_stopped(q)) {
1650                rq = blk_peek_request(q);
1651                if (!rq)
1652                        goto delay_and_out;
1653
1654                /* always use block 0 to find the target for flushes for now */
1655                pos = 0;
1656                if (!(rq->cmd_flags & REQ_FLUSH))
1657                        pos = blk_rq_pos(rq);
1658
1659                ti = dm_table_find_target(map, pos);
1660                if (!dm_target_is_valid(ti)) {
1661                        /*
1662                         * Must perform setup, that dm_done() requires,
1663                         * before calling dm_kill_unmapped_request
1664                         */
1665                        DMERR_LIMIT("request attempted access beyond the end of device");
1666                        clone = dm_start_request(md, rq);
1667                        dm_kill_unmapped_request(clone, -EIO);
1668                        continue;
1669                }
1670
1671                if (ti->type->busy && ti->type->busy(ti))
1672                        goto delay_and_out;
1673
1674                clone = dm_start_request(md, rq);
1675
1676                spin_unlock(q->queue_lock);
1677                if (map_request(ti, clone, md))
1678                        goto requeued;
1679
1680                BUG_ON(!irqs_disabled());
1681                spin_lock(q->queue_lock);
1682        }
1683
1684        goto out;
1685
1686requeued:
1687        BUG_ON(!irqs_disabled());
1688        spin_lock(q->queue_lock);
1689
1690delay_and_out:
1691        blk_delay_queue(q, HZ / 10);
1692out:
1693        dm_table_put(map);
1694}
1695
1696int dm_underlying_device_busy(struct request_queue *q)
1697{
1698        return blk_lld_busy(q);
1699}
1700EXPORT_SYMBOL_GPL(dm_underlying_device_busy);
1701
1702static int dm_lld_busy(struct request_queue *q)
1703{
1704        int r;
1705        struct mapped_device *md = q->queuedata;
1706        struct dm_table *map = dm_get_live_table(md);
1707
1708        if (!map || test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))
1709                r = 1;
1710        else
1711                r = dm_table_any_busy_target(map);
1712
1713        dm_table_put(map);
1714
1715        return r;
1716}
1717
1718static int dm_any_congested(void *congested_data, int bdi_bits)
1719{
1720        int r = bdi_bits;
1721        struct mapped_device *md = congested_data;
1722        struct dm_table *map;
1723
1724        if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1725                map = dm_get_live_table(md);
1726                if (map) {
1727                        /*
1728                         * Request-based dm cares about only own queue for
1729                         * the query about congestion status of request_queue
1730                         */
1731                        if (dm_request_based(md))
1732                                r = md->queue->backing_dev_info.state &
1733                                    bdi_bits;
1734                        else
1735                                r = dm_table_any_congested(map, bdi_bits);
1736
1737                        dm_table_put(map);
1738                }
1739        }
1740
1741        return r;
1742}
1743
1744/*-----------------------------------------------------------------
1745 * An IDR is used to keep track of allocated minor numbers.
1746 *---------------------------------------------------------------*/
1747static void free_minor(int minor)
1748{
1749        spin_lock(&_minor_lock);
1750        idr_remove(&_minor_idr, minor);
1751        spin_unlock(&_minor_lock);
1752}
1753
1754/*
1755 * See if the device with a specific minor # is free.
1756 */
1757static int specific_minor(int minor)
1758{
1759        int r, m;
1760
1761        if (minor >= (1 << MINORBITS))
1762                return -EINVAL;
1763
1764        r = idr_pre_get(&_minor_idr, GFP_KERNEL);
1765        if (!r)
1766                return -ENOMEM;
1767
1768        spin_lock(&_minor_lock);
1769
1770        if (idr_find(&_minor_idr, minor)) {
1771                r = -EBUSY;
1772                goto out;
1773        }
1774
1775        r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
1776        if (r)
1777                goto out;
1778
1779        if (m != minor) {
1780                idr_remove(&_minor_idr, m);
1781                r = -EBUSY;
1782                goto out;
1783        }
1784
1785out:
1786        spin_unlock(&_minor_lock);
1787        return r;
1788}
1789
1790static int next_free_minor(int *minor)
1791{
1792        int r, m;
1793
1794        r = idr_pre_get(&_minor_idr, GFP_KERNEL);
1795        if (!r)
1796                return -ENOMEM;
1797
1798        spin_lock(&_minor_lock);
1799
1800        r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
1801        if (r)
1802                goto out;
1803
1804        if (m >= (1 << MINORBITS)) {
1805                idr_remove(&_minor_idr, m);
1806                r = -ENOSPC;
1807                goto out;
1808        }
1809
1810        *minor = m;
1811
1812out:
1813        spin_unlock(&_minor_lock);
1814        return r;
1815}
1816
1817static const struct block_device_operations dm_blk_dops;
1818
1819static void dm_wq_work(struct work_struct *work);
1820
1821static void dm_init_md_queue(struct mapped_device *md)
1822{
1823        /*
1824         * Request-based dm devices cannot be stacked on top of bio-based dm
1825         * devices.  The type of this dm device has not been decided yet.
1826         * The type is decided at the first table loading time.
1827         * To prevent problematic device stacking, clear the queue flag
1828         * for request stacking support until then.
1829         *
1830         * This queue is new, so no concurrency on the queue_flags.
1831         */
1832        queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
1833
1834        md->queue->queuedata = md;
1835        md->queue->backing_dev_info.congested_fn = dm_any_congested;
1836        md->queue->backing_dev_info.congested_data = md;
1837        blk_queue_make_request(md->queue, dm_request);
1838        blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
1839        blk_queue_merge_bvec(md->queue, dm_merge_bvec);
1840}
1841
1842/*
1843 * Allocate and initialise a blank device with a given minor.
1844 */
1845static struct mapped_device *alloc_dev(int minor)
1846{
1847        int r;
1848        struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1849        void *old_md;
1850
1851        if (!md) {
1852                DMWARN("unable to allocate device, out of memory.");
1853                return NULL;
1854        }
1855
1856        if (!try_module_get(THIS_MODULE))
1857                goto bad_module_get;
1858
1859        /* get a minor number for the dev */
1860        if (minor == DM_ANY_MINOR)
1861                r = next_free_minor(&minor);
1862        else
1863                r = specific_minor(minor);
1864        if (r < 0)
1865                goto bad_minor;
1866
1867        md->type = DM_TYPE_NONE;
1868        init_rwsem(&md->io_lock);
1869        mutex_init(&md->suspend_lock);
1870        mutex_init(&md->type_lock);
1871        spin_lock_init(&md->deferred_lock);
1872        rwlock_init(&md->map_lock);
1873        atomic_set(&md->holders, 1);
1874        atomic_set(&md->open_count, 0);
1875        atomic_set(&md->event_nr, 0);
1876        atomic_set(&md->uevent_seq, 0);
1877        INIT_LIST_HEAD(&md->uevent_list);
1878        spin_lock_init(&md->uevent_lock);
1879
1880        md->queue = blk_alloc_queue(GFP_KERNEL);
1881        if (!md->queue)
1882                goto bad_queue;
1883
1884        dm_init_md_queue(md);
1885
1886        md->disk = alloc_disk(1);
1887        if (!md->disk)
1888                goto bad_disk;
1889
1890        atomic_set(&md->pending[0], 0);
1891        atomic_set(&md->pending[1], 0);
1892        init_waitqueue_head(&md->wait);
1893        INIT_WORK(&md->work, dm_wq_work);
1894        init_waitqueue_head(&md->eventq);
1895
1896        md->disk->major = _major;
1897        md->disk->first_minor = minor;
1898        md->disk->fops = &dm_blk_dops;
1899        md->disk->queue = md->queue;
1900        md->disk->private_data = md;
1901        sprintf(md->disk->disk_name, "dm-%d", minor);
1902        add_disk(md->disk);
1903        format_dev_t(md->name, MKDEV(_major, minor));
1904
1905        md->wq = alloc_workqueue("kdmflush",
1906                                 WQ_NON_REENTRANT | WQ_MEM_RECLAIM, 0);
1907        if (!md->wq)
1908                goto bad_thread;
1909
1910        md->bdev = bdget_disk(md->disk, 0);
1911        if (!md->bdev)
1912                goto bad_bdev;
1913
1914        bio_init(&md->flush_bio);
1915        md->flush_bio.bi_bdev = md->bdev;
1916        md->flush_bio.bi_rw = WRITE_FLUSH;
1917
1918        /* Populate the mapping, nobody knows we exist yet */
1919        spin_lock(&_minor_lock);
1920        old_md = idr_replace(&_minor_idr, md, minor);
1921        spin_unlock(&_minor_lock);
1922
1923        BUG_ON(old_md != MINOR_ALLOCED);
1924
1925        return md;
1926
1927bad_bdev:
1928        destroy_workqueue(md->wq);
1929bad_thread:
1930        del_gendisk(md->disk);
1931        put_disk(md->disk);
1932bad_disk:
1933        blk_cleanup_queue(md->queue);
1934bad_queue:
1935        free_minor(minor);
1936bad_minor:
1937        module_put(THIS_MODULE);
1938bad_module_get:
1939        kfree(md);
1940        return NULL;
1941}
1942
1943static void unlock_fs(struct mapped_device *md);
1944
1945static void free_dev(struct mapped_device *md)
1946{
1947        int minor = MINOR(disk_devt(md->disk));
1948
1949        unlock_fs(md);
1950        bdput(md->bdev);
1951        destroy_workqueue(md->wq);
1952        if (md->tio_pool)
1953                mempool_destroy(md->tio_pool);
1954        if (md->io_pool)
1955                mempool_destroy(md->io_pool);
1956        if (md->bs)
1957                bioset_free(md->bs);
1958        blk_integrity_unregister(md->disk);
1959        del_gendisk(md->disk);
1960        free_minor(minor);
1961
1962        spin_lock(&_minor_lock);
1963        md->disk->private_data = NULL;
1964        spin_unlock(&_minor_lock);
1965
1966        put_disk(md->disk);
1967        blk_cleanup_queue(md->queue);
1968        module_put(THIS_MODULE);
1969        kfree(md);
1970}
1971
1972static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
1973{
1974        struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1975
1976        if (md->io_pool && md->bs) {
1977                /* The md already has necessary mempools. */
1978                if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) {
1979                        /*
1980                         * Reload bioset because front_pad may have changed
1981                         * because a different table was loaded.
1982                         */
1983                        bioset_free(md->bs);
1984                        md->bs = p->bs;
1985                        p->bs = NULL;
1986                } else if (dm_table_get_type(t) == DM_TYPE_REQUEST_BASED) {
1987                        BUG_ON(!md->tio_pool);
1988                        /*
1989                         * There's no need to reload with request-based dm
1990                         * because the size of front_pad doesn't change.
1991                         * Note for future: If you are to reload bioset,
1992                         * prep-ed requests in the queue may refer
1993                         * to bio from the old bioset, so you must walk
1994                         * through the queue to unprep.
1995                         */
1996                }
1997                goto out;
1998        }
1999
2000        BUG_ON(!p || md->io_pool || md->tio_pool || md->bs);
2001
2002        md->io_pool = p->io_pool;
2003        p->io_pool = NULL;
2004        md->tio_pool = p->tio_pool;
2005        p->tio_pool = NULL;
2006        md->bs = p->bs;
2007        p->bs = NULL;
2008
2009out:
2010        /* mempool bind completed, now no need any mempools in the table */
2011        dm_table_free_md_mempools(t);
2012}
2013
2014/*
2015 * Bind a table to the device.
2016 */
2017static void event_callback(void *context)
2018{
2019        unsigned long flags;
2020        LIST_HEAD(uevents);
2021        struct mapped_device *md = (struct mapped_device *) context;
2022
2023        spin_lock_irqsave(&md->uevent_lock, flags);
2024        list_splice_init(&md->uevent_list, &uevents);
2025        spin_unlock_irqrestore(&md->uevent_lock, flags);
2026
2027        dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
2028
2029        atomic_inc(&md->event_nr);
2030        wake_up(&md->eventq);
2031}
2032
2033/*
2034 * Protected by md->suspend_lock obtained by dm_swap_table().
2035 */
2036static void __set_size(struct mapped_device *md, sector_t size)
2037{
2038        set_capacity(md->disk, size);
2039
2040        i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
2041}
2042
2043/*
2044 * Return 1 if the queue has a compulsory merge_bvec_fn function.
2045 *
2046 * If this function returns 0, then the device is either a non-dm
2047 * device without a merge_bvec_fn, or it is a dm device that is
2048 * able to split any bios it receives that are too big.
2049 */
2050int dm_queue_merge_is_compulsory(struct request_queue *q)
2051{
2052        struct mapped_device *dev_md;
2053
2054        if (!q->merge_bvec_fn)
2055                return 0;
2056
2057        if (q->make_request_fn == dm_request) {
2058                dev_md = q->queuedata;
2059                if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags))
2060                        return 0;
2061        }
2062
2063        return 1;
2064}
2065
2066static int dm_device_merge_is_compulsory(struct dm_target *ti,
2067                                         struct dm_dev *dev, sector_t start,
2068                                         sector_t len, void *data)
2069{
2070        struct block_device *bdev = dev->bdev;
2071        struct request_queue *q = bdev_get_queue(bdev);
2072
2073        return dm_queue_merge_is_compulsory(q);
2074}
2075
2076/*
2077 * Return 1 if it is acceptable to ignore merge_bvec_fn based
2078 * on the properties of the underlying devices.
2079 */
2080static int dm_table_merge_is_optional(struct dm_table *table)
2081{
2082        unsigned i = 0;
2083        struct dm_target *ti;
2084
2085        while (i < dm_table_get_num_targets(table)) {
2086                ti = dm_table_get_target(table, i++);
2087
2088                if (ti->type->iterate_devices &&
2089                    ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL))
2090                        return 0;
2091        }
2092
2093        return 1;
2094}
2095
2096/*
2097 * Returns old map, which caller must destroy.
2098 */
2099static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2100                               struct queue_limits *limits)
2101{
2102        struct dm_table *old_map;
2103        struct request_queue *q = md->queue;
2104        sector_t size;
2105        unsigned long flags;
2106        int merge_is_optional;
2107
2108        size = dm_table_get_size(t);
2109
2110        /*
2111         * Wipe any geometry if the size of the table changed.
2112         */
2113        if (size != get_capacity(md->disk))
2114                memset(&md->geometry, 0, sizeof(md->geometry));
2115
2116        __set_size(md, size);
2117
2118        dm_table_event_callback(t, event_callback, md);
2119
2120        /*
2121         * The queue hasn't been stopped yet, if the old table type wasn't
2122         * for request-based during suspension.  So stop it to prevent
2123         * I/O mapping before resume.
2124         * This must be done before setting the queue restrictions,
2125         * because request-based dm may be run just after the setting.
2126         */
2127        if (dm_table_request_based(t) && !blk_queue_stopped(q))
2128                stop_queue(q);
2129
2130        __bind_mempools(md, t);
2131
2132        merge_is_optional = dm_table_merge_is_optional(t);
2133
2134        write_lock_irqsave(&md->map_lock, flags);
2135        old_map = md->map;
2136        md->map = t;
2137        md->immutable_target_type = dm_table_get_immutable_target_type(t);
2138
2139        dm_table_set_restrictions(t, q, limits);
2140        if (merge_is_optional)
2141                set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2142        else
2143                clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2144        write_unlock_irqrestore(&md->map_lock, flags);
2145
2146        return old_map;
2147}
2148
2149/*
2150 * Returns unbound table for the caller to free.
2151 */
2152static struct dm_table *__unbind(struct mapped_device *md)
2153{
2154        struct dm_table *map = md->map;
2155        unsigned long flags;
2156
2157        if (!map)
2158                return NULL;
2159
2160        dm_table_event_callback(map, NULL, NULL);
2161        write_lock_irqsave(&md->map_lock, flags);
2162        md->map = NULL;
2163        write_unlock_irqrestore(&md->map_lock, flags);
2164
2165        return map;
2166}
2167
2168/*
2169 * Constructor for a new device.
2170 */
2171int dm_create(int minor, struct mapped_device **result)
2172{
2173        struct mapped_device *md;
2174
2175        md = alloc_dev(minor);
2176        if (!md)
2177                return -ENXIO;
2178
2179        dm_sysfs_init(md);
2180
2181        *result = md;
2182        return 0;
2183}
2184
2185/*
2186 * Functions to manage md->type.
2187 * All are required to hold md->type_lock.
2188 */
2189void dm_lock_md_type(struct mapped_device *md)
2190{
2191        mutex_lock(&md->type_lock);
2192}
2193
2194void dm_unlock_md_type(struct mapped_device *md)
2195{
2196        mutex_unlock(&md->type_lock);
2197}
2198
2199void dm_set_md_type(struct mapped_device *md, unsigned type)
2200{
2201        md->type = type;
2202}
2203
2204unsigned dm_get_md_type(struct mapped_device *md)
2205{
2206        return md->type;
2207}
2208
2209struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2210{
2211        return md->immutable_target_type;
2212}
2213
2214/*
2215 * Fully initialize a request-based queue (->elevator, ->request_fn, etc).
2216 */
2217static int dm_init_request_based_queue(struct mapped_device *md)
2218{
2219        struct request_queue *q = NULL;
2220
2221        if (md->queue->elevator)
2222                return 1;
2223
2224        /* Fully initialize the queue */
2225        q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL);
2226        if (!q)
2227                return 0;
2228
2229        md->queue = q;
2230        dm_init_md_queue(md);
2231        blk_queue_softirq_done(md->queue, dm_softirq_done);
2232        blk_queue_prep_rq(md->queue, dm_prep_fn);
2233        blk_queue_lld_busy(md->queue, dm_lld_busy);
2234
2235        elv_register_queue(md->queue);
2236
2237        return 1;
2238}
2239
2240/*
2241 * Setup the DM device's queue based on md's type
2242 */
2243int dm_setup_md_queue(struct mapped_device *md)
2244{
2245        if ((dm_get_md_type(md) == DM_TYPE_REQUEST_BASED) &&
2246            !dm_init_request_based_queue(md)) {
2247                DMWARN("Cannot initialize queue for request-based mapped device");
2248                return -EINVAL;
2249        }
2250
2251        return 0;
2252}
2253
2254static struct mapped_device *dm_find_md(dev_t dev)
2255{
2256        struct mapped_device *md;
2257        unsigned minor = MINOR(dev);
2258
2259        if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2260                return NULL;
2261
2262        spin_lock(&_minor_lock);
2263
2264        md = idr_find(&_minor_idr, minor);
2265        if (md && (md == MINOR_ALLOCED ||
2266                   (MINOR(disk_devt(dm_disk(md))) != minor) ||
2267                   dm_deleting_md(md) ||
2268                   test_bit(DMF_FREEING, &md->flags))) {
2269                md = NULL;
2270                goto out;
2271        }
2272
2273out:
2274        spin_unlock(&_minor_lock);
2275
2276        return md;
2277}
2278
2279struct mapped_device *dm_get_md(dev_t dev)
2280{
2281        struct mapped_device *md = dm_find_md(dev);
2282
2283        if (md)
2284                dm_get(md);
2285
2286        return md;
2287}
2288EXPORT_SYMBOL_GPL(dm_get_md);
2289
2290void *dm_get_mdptr(struct mapped_device *md)
2291{
2292        return md->interface_ptr;
2293}
2294
2295void dm_set_mdptr(struct mapped_device *md, void *ptr)
2296{
2297        md->interface_ptr = ptr;
2298}
2299
2300void dm_get(struct mapped_device *md)
2301{
2302        atomic_inc(&md->holders);
2303        BUG_ON(test_bit(DMF_FREEING, &md->flags));
2304}
2305
2306const char *dm_device_name(struct mapped_device *md)
2307{
2308        return md->name;
2309}
2310EXPORT_SYMBOL_GPL(dm_device_name);
2311
2312static void __dm_destroy(struct mapped_device *md, bool wait)
2313{
2314        struct dm_table *map;
2315
2316        might_sleep();
2317
2318        spin_lock(&_minor_lock);
2319        map = dm_get_live_table(md);
2320        idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2321        set_bit(DMF_FREEING, &md->flags);
2322        spin_unlock(&_minor_lock);
2323
2324        if (!dm_suspended_md(md)) {
2325                dm_table_presuspend_targets(map);
2326                dm_table_postsuspend_targets(map);
2327        }
2328
2329        /*
2330         * Rare, but there may be I/O requests still going to complete,
2331         * for example.  Wait for all references to disappear.
2332         * No one should increment the reference count of the mapped_device,
2333         * after the mapped_device state becomes DMF_FREEING.
2334         */
2335        if (wait)
2336                while (atomic_read(&md->holders))
2337                        msleep(1);
2338        else if (atomic_read(&md->holders))
2339                DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2340                       dm_device_name(md), atomic_read(&md->holders));
2341
2342        dm_sysfs_exit(md);
2343        dm_table_put(map);
2344        dm_table_destroy(__unbind(md));
2345        free_dev(md);
2346}
2347
2348void dm_destroy(struct mapped_device *md)
2349{
2350        __dm_destroy(md, true);
2351}
2352
2353void dm_destroy_immediate(struct mapped_device *md)
2354{
2355        __dm_destroy(md, false);
2356}
2357
2358void dm_put(struct mapped_device *md)
2359{
2360        atomic_dec(&md->holders);
2361}
2362EXPORT_SYMBOL_GPL(dm_put);
2363
2364static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
2365{
2366        int r = 0;
2367        DECLARE_WAITQUEUE(wait, current);
2368
2369        add_wait_queue(&md->wait, &wait);
2370
2371        while (1) {
2372                set_current_state(interruptible);
2373
2374                if (!md_in_flight(md))
2375                        break;
2376
2377                if (interruptible == TASK_INTERRUPTIBLE &&
2378                    signal_pending(current)) {
2379                        r = -EINTR;
2380                        break;
2381                }
2382
2383                io_schedule();
2384        }
2385        set_current_state(TASK_RUNNING);
2386
2387        remove_wait_queue(&md->wait, &wait);
2388
2389        return r;
2390}
2391
2392/*
2393 * Process the deferred bios
2394 */
2395static void dm_wq_work(struct work_struct *work)
2396{
2397        struct mapped_device *md = container_of(work, struct mapped_device,
2398                                                work);
2399        struct bio *c;
2400
2401        down_read(&md->io_lock);
2402
2403        while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2404                spin_lock_irq(&md->deferred_lock);
2405                c = bio_list_pop(&md->deferred);
2406                spin_unlock_irq(&md->deferred_lock);
2407
2408                if (!c)
2409                        break;
2410
2411                up_read(&md->io_lock);
2412
2413                if (dm_request_based(md))
2414                        generic_make_request(c);
2415                else
2416                        __split_and_process_bio(md, c);
2417
2418                down_read(&md->io_lock);
2419        }
2420
2421        up_read(&md->io_lock);
2422}
2423
2424static void dm_queue_flush(struct mapped_device *md)
2425{
2426        clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2427        smp_mb__after_clear_bit();
2428        queue_work(md->wq, &md->work);
2429}
2430
2431/*
2432 * Swap in a new table, returning the old one for the caller to destroy.
2433 */
2434struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2435{
2436        struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2437        struct queue_limits limits;
2438        int r;
2439
2440        mutex_lock(&md->suspend_lock);
2441
2442        /* device must be suspended */
2443        if (!dm_suspended_md(md))
2444                goto out;
2445
2446        /*
2447         * If the new table has no data devices, retain the existing limits.
2448         * This helps multipath with queue_if_no_path if all paths disappear,
2449         * then new I/O is queued based on these limits, and then some paths
2450         * reappear.
2451         */
2452        if (dm_table_has_no_data_devices(table)) {
2453                live_map = dm_get_live_table(md);
2454                if (live_map)
2455                        limits = md->queue->limits;
2456                dm_table_put(live_map);
2457        }
2458
2459        if (!live_map) {
2460                r = dm_calculate_queue_limits(table, &limits);
2461                if (r) {
2462                        map = ERR_PTR(r);
2463                        goto out;
2464                }
2465        }
2466
2467        map = __bind(md, table, &limits);
2468
2469out:
2470        mutex_unlock(&md->suspend_lock);
2471        return map;
2472}
2473
2474/*
2475 * Functions to lock and unlock any filesystem running on the
2476 * device.
2477 */
2478static int lock_fs(struct mapped_device *md)
2479{
2480        int r;
2481
2482        WARN_ON(md->frozen_sb);
2483
2484        md->frozen_sb = freeze_bdev(md->bdev);
2485        if (IS_ERR(md->frozen_sb)) {
2486                r = PTR_ERR(md->frozen_sb);
2487                md->frozen_sb = NULL;
2488                return r;
2489        }
2490
2491        set_bit(DMF_FROZEN, &md->flags);
2492
2493        return 0;
2494}
2495
2496static void unlock_fs(struct mapped_device *md)
2497{
2498        if (!test_bit(DMF_FROZEN, &md->flags))
2499                return;
2500
2501        thaw_bdev(md->bdev, md->frozen_sb);
2502        md->frozen_sb = NULL;
2503        clear_bit(DMF_FROZEN, &md->flags);
2504}
2505
2506/*
2507 * We need to be able to change a mapping table under a mounted
2508 * filesystem.  For example we might want to move some data in
2509 * the background.  Before the table can be swapped with
2510 * dm_bind_table, dm_suspend must be called to flush any in
2511 * flight bios and ensure that any further io gets deferred.
2512 */
2513/*
2514 * Suspend mechanism in request-based dm.
2515 *
2516 * 1. Flush all I/Os by lock_fs() if needed.
2517 * 2. Stop dispatching any I/O by stopping the request_queue.
2518 * 3. Wait for all in-flight I/Os to be completed or requeued.
2519 *
2520 * To abort suspend, start the request_queue.
2521 */
2522int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2523{
2524        struct dm_table *map = NULL;
2525        int r = 0;
2526        int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
2527        int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
2528
2529        mutex_lock(&md->suspend_lock);
2530
2531        if (dm_suspended_md(md)) {
2532                r = -EINVAL;
2533                goto out_unlock;
2534        }
2535
2536        map = dm_get_live_table(md);
2537
2538        /*
2539         * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2540         * This flag is cleared before dm_suspend returns.
2541         */
2542        if (noflush)
2543                set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2544
2545        /* This does not get reverted if there's an error later. */
2546        dm_table_presuspend_targets(map);
2547
2548        /*
2549         * Flush I/O to the device.
2550         * Any I/O submitted after lock_fs() may not be flushed.
2551         * noflush takes precedence over do_lockfs.
2552         * (lock_fs() flushes I/Os and waits for them to complete.)
2553         */
2554        if (!noflush && do_lockfs) {
2555                r = lock_fs(md);
2556                if (r)
2557                        goto out;
2558        }
2559
2560        /*
2561         * Here we must make sure that no processes are submitting requests
2562         * to target drivers i.e. no one may be executing
2563         * __split_and_process_bio. This is called from dm_request and
2564         * dm_wq_work.
2565         *
2566         * To get all processes out of __split_and_process_bio in dm_request,
2567         * we take the write lock. To prevent any process from reentering
2568         * __split_and_process_bio from dm_request and quiesce the thread
2569         * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2570         * flush_workqueue(md->wq).
2571         */
2572        down_write(&md->io_lock);
2573        set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2574        up_write(&md->io_lock);
2575
2576        /*
2577         * Stop md->queue before flushing md->wq in case request-based
2578         * dm defers requests to md->wq from md->queue.
2579         */
2580        if (dm_request_based(md))
2581                stop_queue(md->queue);
2582
2583        flush_workqueue(md->wq);
2584
2585        /*
2586         * At this point no more requests are entering target request routines.
2587         * We call dm_wait_for_completion to wait for all existing requests
2588         * to finish.
2589         */
2590        r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
2591
2592        down_write(&md->io_lock);
2593        if (noflush)
2594                clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2595        up_write(&md->io_lock);
2596
2597        /* were we interrupted ? */
2598        if (r < 0) {
2599                dm_queue_flush(md);
2600
2601                if (dm_request_based(md))
2602                        start_queue(md->queue);
2603
2604                unlock_fs(md);
2605                goto out; /* pushback list is already flushed, so skip flush */
2606        }
2607
2608        /*
2609         * If dm_wait_for_completion returned 0, the device is completely
2610         * quiescent now. There is no request-processing activity. All new
2611         * requests are being added to md->deferred list.
2612         */
2613
2614        set_bit(DMF_SUSPENDED, &md->flags);
2615
2616        dm_table_postsuspend_targets(map);
2617
2618out:
2619        dm_table_put(map);
2620
2621out_unlock:
2622        mutex_unlock(&md->suspend_lock);
2623        return r;
2624}
2625
2626int dm_resume(struct mapped_device *md)
2627{
2628        int r = -EINVAL;
2629        struct dm_table *map = NULL;
2630
2631        mutex_lock(&md->suspend_lock);
2632        if (!dm_suspended_md(md))
2633                goto out;
2634
2635        map = dm_get_live_table(md);
2636        if (!map || !dm_table_get_size(map))
2637                goto out;
2638
2639        r = dm_table_resume_targets(map);
2640        if (r)
2641                goto out;
2642
2643        dm_queue_flush(md);
2644
2645        /*
2646         * Flushing deferred I/Os must be done after targets are resumed
2647         * so that mapping of targets can work correctly.
2648         * Request-based dm is queueing the deferred I/Os in its request_queue.
2649         */
2650        if (dm_request_based(md))
2651                start_queue(md->queue);
2652
2653        unlock_fs(md);
2654
2655        clear_bit(DMF_SUSPENDED, &md->flags);
2656
2657        r = 0;
2658out:
2659        dm_table_put(map);
2660        mutex_unlock(&md->suspend_lock);
2661
2662        return r;
2663}
2664
2665/*-----------------------------------------------------------------
2666 * Event notification.
2667 *---------------------------------------------------------------*/
2668int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2669                       unsigned cookie)
2670{
2671        char udev_cookie[DM_COOKIE_LENGTH];
2672        char *envp[] = { udev_cookie, NULL };
2673
2674        if (!cookie)
2675                return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2676        else {
2677                snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2678                         DM_COOKIE_ENV_VAR_NAME, cookie);
2679                return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2680                                          action, envp);
2681        }
2682}
2683
2684uint32_t dm_next_uevent_seq(struct mapped_device *md)
2685{
2686        return atomic_add_return(1, &md->uevent_seq);
2687}
2688
2689uint32_t dm_get_event_nr(struct mapped_device *md)
2690{
2691        return atomic_read(&md->event_nr);
2692}
2693
2694int dm_wait_event(struct mapped_device *md, int event_nr)
2695{
2696        return wait_event_interruptible(md->eventq,
2697                        (event_nr != atomic_read(&md->event_nr)));
2698}
2699
2700void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2701{
2702        unsigned long flags;
2703
2704        spin_lock_irqsave(&md->uevent_lock, flags);
2705        list_add(elist, &md->uevent_list);
2706        spin_unlock_irqrestore(&md->uevent_lock, flags);
2707}
2708
2709/*
2710 * The gendisk is only valid as long as you have a reference
2711 * count on 'md'.
2712 */
2713struct gendisk *dm_disk(struct mapped_device *md)
2714{
2715        return md->disk;
2716}
2717
2718struct kobject *dm_kobject(struct mapped_device *md)
2719{
2720        return &md->kobj;
2721}
2722
2723/*
2724 * struct mapped_device should not be exported outside of dm.c
2725 * so use this check to verify that kobj is part of md structure
2726 */
2727struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2728{
2729        struct mapped_device *md;
2730
2731        md = container_of(kobj, struct mapped_device, kobj);
2732        if (&md->kobj != kobj)
2733                return NULL;
2734
2735        if (test_bit(DMF_FREEING, &md->flags) ||
2736            dm_deleting_md(md))
2737                return NULL;
2738
2739        dm_get(md);
2740        return md;
2741}
2742
2743int dm_suspended_md(struct mapped_device *md)
2744{
2745        return test_bit(DMF_SUSPENDED, &md->flags);
2746}
2747
2748int dm_suspended(struct dm_target *ti)
2749{
2750        return dm_suspended_md(dm_table_get_md(ti->table));
2751}
2752EXPORT_SYMBOL_GPL(dm_suspended);
2753
2754int dm_noflush_suspending(struct dm_target *ti)
2755{
2756        return __noflush_suspending(dm_table_get_md(ti->table));
2757}
2758EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2759
2760struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, unsigned per_bio_data_size)
2761{
2762        struct dm_md_mempools *pools = kmalloc(sizeof(*pools), GFP_KERNEL);
2763        unsigned int pool_size = (type == DM_TYPE_BIO_BASED) ? 16 : MIN_IOS;
2764
2765        if (!pools)
2766                return NULL;
2767
2768        per_bio_data_size = roundup(per_bio_data_size, __alignof__(struct dm_target_io));
2769
2770        pools->io_pool = (type == DM_TYPE_BIO_BASED) ?
2771                         mempool_create_slab_pool(MIN_IOS, _io_cache) :
2772                         mempool_create_slab_pool(MIN_IOS, _rq_bio_info_cache);
2773        if (!pools->io_pool)
2774                goto free_pools_and_out;
2775
2776        pools->tio_pool = NULL;
2777        if (type == DM_TYPE_REQUEST_BASED) {
2778                pools->tio_pool = mempool_create_slab_pool(MIN_IOS, _rq_tio_cache);
2779                if (!pools->tio_pool)
2780                        goto free_io_pool_and_out;
2781        }
2782
2783        pools->bs = (type == DM_TYPE_BIO_BASED) ?
2784                bioset_create(pool_size,
2785                              per_bio_data_size + offsetof(struct dm_target_io, clone)) :
2786                bioset_create(pool_size,
2787                              offsetof(struct dm_rq_clone_bio_info, clone));
2788        if (!pools->bs)
2789                goto free_tio_pool_and_out;
2790
2791        if (integrity && bioset_integrity_create(pools->bs, pool_size))
2792                goto free_bioset_and_out;
2793
2794        return pools;
2795
2796free_bioset_and_out:
2797        bioset_free(pools->bs);
2798
2799free_tio_pool_and_out:
2800        if (pools->tio_pool)
2801                mempool_destroy(pools->tio_pool);
2802
2803free_io_pool_and_out:
2804        mempool_destroy(pools->io_pool);
2805
2806free_pools_and_out:
2807        kfree(pools);
2808
2809        return NULL;
2810}
2811
2812void dm_free_md_mempools(struct dm_md_mempools *pools)
2813{
2814        if (!pools)
2815                return;
2816
2817        if (pools->io_pool)
2818                mempool_destroy(pools->io_pool);
2819
2820        if (pools->tio_pool)
2821                mempool_destroy(pools->tio_pool);
2822
2823        if (pools->bs)
2824                bioset_free(pools->bs);
2825
2826        kfree(pools);
2827}
2828
2829static const struct block_device_operations dm_blk_dops = {
2830        .open = dm_blk_open,
2831        .release = dm_blk_close,
2832        .ioctl = dm_blk_ioctl,
2833        .getgeo = dm_blk_getgeo,
2834        .owner = THIS_MODULE
2835};
2836
2837EXPORT_SYMBOL(dm_get_mapinfo);
2838
2839/*
2840 * module hooks
2841 */
2842module_init(dm_init);
2843module_exit(dm_exit);
2844
2845module_param(major, uint, 0);
2846MODULE_PARM_DESC(major, "The major number of the device mapper");
2847MODULE_DESCRIPTION(DM_NAME " driver");
2848MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2849MODULE_LICENSE("GPL");
2850
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.