linux/drivers/md/dm-cache-target.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2012 Red Hat. All rights reserved.
   3 *
   4 * This file is released under the GPL.
   5 */
   6
   7#include "dm.h"
   8#include "dm-bio-prison.h"
   9#include "dm-bio-record.h"
  10#include "dm-cache-metadata.h"
  11
  12#include <linux/dm-io.h>
  13#include <linux/dm-kcopyd.h>
  14#include <linux/init.h>
  15#include <linux/mempool.h>
  16#include <linux/module.h>
  17#include <linux/slab.h>
  18#include <linux/vmalloc.h>
  19
  20#define DM_MSG_PREFIX "cache"
  21
  22DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
  23        "A percentage of time allocated for copying to and/or from cache");
  24
  25/*----------------------------------------------------------------*/
  26
  27/*
  28 * Glossary:
  29 *
  30 * oblock: index of an origin block
  31 * cblock: index of a cache block
  32 * promotion: movement of a block from origin to cache
  33 * demotion: movement of a block from cache to origin
  34 * migration: movement of a block between the origin and cache device,
  35 *            either direction
  36 */
  37
  38/*----------------------------------------------------------------*/
  39
  40static size_t bitset_size_in_bytes(unsigned nr_entries)
  41{
  42        return sizeof(unsigned long) * dm_div_up(nr_entries, BITS_PER_LONG);
  43}
  44
  45static unsigned long *alloc_bitset(unsigned nr_entries)
  46{
  47        size_t s = bitset_size_in_bytes(nr_entries);
  48        return vzalloc(s);
  49}
  50
  51static void clear_bitset(void *bitset, unsigned nr_entries)
  52{
  53        size_t s = bitset_size_in_bytes(nr_entries);
  54        memset(bitset, 0, s);
  55}
  56
  57static void free_bitset(unsigned long *bits)
  58{
  59        vfree(bits);
  60}
  61
  62/*----------------------------------------------------------------*/
  63
  64#define PRISON_CELLS 1024
  65#define MIGRATION_POOL_SIZE 128
  66#define COMMIT_PERIOD HZ
  67#define MIGRATION_COUNT_WINDOW 10
  68
  69/*
  70 * The block size of the device holding cache data must be >= 32KB
  71 */
  72#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
  73
  74/*
  75 * FIXME: the cache is read/write for the time being.
  76 */
  77enum cache_mode {
  78        CM_WRITE,               /* metadata may be changed */
  79        CM_READ_ONLY,           /* metadata may not be changed */
  80};
  81
  82struct cache_features {
  83        enum cache_mode mode;
  84        bool write_through:1;
  85};
  86
  87struct cache_stats {
  88        atomic_t read_hit;
  89        atomic_t read_miss;
  90        atomic_t write_hit;
  91        atomic_t write_miss;
  92        atomic_t demotion;
  93        atomic_t promotion;
  94        atomic_t copies_avoided;
  95        atomic_t cache_cell_clash;
  96        atomic_t commit_count;
  97        atomic_t discard_count;
  98};
  99
 100struct cache {
 101        struct dm_target *ti;
 102        struct dm_target_callbacks callbacks;
 103
 104        /*
 105         * Metadata is written to this device.
 106         */
 107        struct dm_dev *metadata_dev;
 108
 109        /*
 110         * The slower of the two data devices.  Typically a spindle.
 111         */
 112        struct dm_dev *origin_dev;
 113
 114        /*
 115         * The faster of the two data devices.  Typically an SSD.
 116         */
 117        struct dm_dev *cache_dev;
 118
 119        /*
 120         * Cache features such as write-through.
 121         */
 122        struct cache_features features;
 123
 124        /*
 125         * Size of the origin device in _complete_ blocks and native sectors.
 126         */
 127        dm_oblock_t origin_blocks;
 128        sector_t origin_sectors;
 129
 130        /*
 131         * Size of the cache device in blocks.
 132         */
 133        dm_cblock_t cache_size;
 134
 135        /*
 136         * Fields for converting from sectors to blocks.
 137         */
 138        uint32_t sectors_per_block;
 139        int sectors_per_block_shift;
 140
 141        struct dm_cache_metadata *cmd;
 142
 143        spinlock_t lock;
 144        struct bio_list deferred_bios;
 145        struct bio_list deferred_flush_bios;
 146        struct bio_list deferred_writethrough_bios;
 147        struct list_head quiesced_migrations;
 148        struct list_head completed_migrations;
 149        struct list_head need_commit_migrations;
 150        sector_t migration_threshold;
 151        atomic_t nr_migrations;
 152        wait_queue_head_t migration_wait;
 153
 154        /*
 155         * cache_size entries, dirty if set
 156         */
 157        dm_cblock_t nr_dirty;
 158        unsigned long *dirty_bitset;
 159
 160        /*
 161         * origin_blocks entries, discarded if set.
 162         */
 163        uint32_t discard_block_size; /* a power of 2 times sectors per block */
 164        dm_dblock_t discard_nr_blocks;
 165        unsigned long *discard_bitset;
 166
 167        struct dm_kcopyd_client *copier;
 168        struct workqueue_struct *wq;
 169        struct work_struct worker;
 170
 171        struct delayed_work waker;
 172        unsigned long last_commit_jiffies;
 173
 174        struct dm_bio_prison *prison;
 175        struct dm_deferred_set *all_io_ds;
 176
 177        mempool_t *migration_pool;
 178        struct dm_cache_migration *next_migration;
 179
 180        struct dm_cache_policy *policy;
 181        unsigned policy_nr_args;
 182
 183        bool need_tick_bio:1;
 184        bool sized:1;
 185        bool quiescing:1;
 186        bool commit_requested:1;
 187        bool loaded_mappings:1;
 188        bool loaded_discards:1;
 189
 190        struct cache_stats stats;
 191
 192        /*
 193         * Rather than reconstructing the table line for the status we just
 194         * save it and regurgitate.
 195         */
 196        unsigned nr_ctr_args;
 197        const char **ctr_args;
 198};
 199
 200struct per_bio_data {
 201        bool tick:1;
 202        unsigned req_nr:2;
 203        struct dm_deferred_entry *all_io_entry;
 204
 205        /*
 206         * writethrough fields.  These MUST remain at the end of this
 207         * structure and the 'cache' member must be the first as it
 208         * is used to determine the offsetof the writethrough fields.
 209         */
 210        struct cache *cache;
 211        dm_cblock_t cblock;
 212        bio_end_io_t *saved_bi_end_io;
 213        struct dm_bio_details bio_details;
 214};
 215
 216struct dm_cache_migration {
 217        struct list_head list;
 218        struct cache *cache;
 219
 220        unsigned long start_jiffies;
 221        dm_oblock_t old_oblock;
 222        dm_oblock_t new_oblock;
 223        dm_cblock_t cblock;
 224
 225        bool err:1;
 226        bool writeback:1;
 227        bool demote:1;
 228        bool promote:1;
 229
 230        struct dm_bio_prison_cell *old_ocell;
 231        struct dm_bio_prison_cell *new_ocell;
 232};
 233
 234/*
 235 * Processing a bio in the worker thread may require these memory
 236 * allocations.  We prealloc to avoid deadlocks (the same worker thread
 237 * frees them back to the mempool).
 238 */
 239struct prealloc {
 240        struct dm_cache_migration *mg;
 241        struct dm_bio_prison_cell *cell1;
 242        struct dm_bio_prison_cell *cell2;
 243};
 244
 245static void wake_worker(struct cache *cache)
 246{
 247        queue_work(cache->wq, &cache->worker);
 248}
 249
 250/*----------------------------------------------------------------*/
 251
 252static struct dm_bio_prison_cell *alloc_prison_cell(struct cache *cache)
 253{
 254        /* FIXME: change to use a local slab. */
 255        return dm_bio_prison_alloc_cell(cache->prison, GFP_NOWAIT);
 256}
 257
 258static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell *cell)
 259{
 260        dm_bio_prison_free_cell(cache->prison, cell);
 261}
 262
 263static int prealloc_data_structs(struct cache *cache, struct prealloc *p)
 264{
 265        if (!p->mg) {
 266                p->mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT);
 267                if (!p->mg)
 268                        return -ENOMEM;
 269        }
 270
 271        if (!p->cell1) {
 272                p->cell1 = alloc_prison_cell(cache);
 273                if (!p->cell1)
 274                        return -ENOMEM;
 275        }
 276
 277        if (!p->cell2) {
 278                p->cell2 = alloc_prison_cell(cache);
 279                if (!p->cell2)
 280                        return -ENOMEM;
 281        }
 282
 283        return 0;
 284}
 285
 286static void prealloc_free_structs(struct cache *cache, struct prealloc *p)
 287{
 288        if (p->cell2)
 289                free_prison_cell(cache, p->cell2);
 290
 291        if (p->cell1)
 292                free_prison_cell(cache, p->cell1);
 293
 294        if (p->mg)
 295                mempool_free(p->mg, cache->migration_pool);
 296}
 297
 298static struct dm_cache_migration *prealloc_get_migration(struct prealloc *p)
 299{
 300        struct dm_cache_migration *mg = p->mg;
 301
 302        BUG_ON(!mg);
 303        p->mg = NULL;
 304
 305        return mg;
 306}
 307
 308/*
 309 * You must have a cell within the prealloc struct to return.  If not this
 310 * function will BUG() rather than returning NULL.
 311 */
 312static struct dm_bio_prison_cell *prealloc_get_cell(struct prealloc *p)
 313{
 314        struct dm_bio_prison_cell *r = NULL;
 315
 316        if (p->cell1) {
 317                r = p->cell1;
 318                p->cell1 = NULL;
 319
 320        } else if (p->cell2) {
 321                r = p->cell2;
 322                p->cell2 = NULL;
 323        } else
 324                BUG();
 325
 326        return r;
 327}
 328
 329/*
 330 * You can't have more than two cells in a prealloc struct.  BUG() will be
 331 * called if you try and overfill.
 332 */
 333static void prealloc_put_cell(struct prealloc *p, struct dm_bio_prison_cell *cell)
 334{
 335        if (!p->cell2)
 336                p->cell2 = cell;
 337
 338        else if (!p->cell1)
 339                p->cell1 = cell;
 340
 341        else
 342                BUG();
 343}
 344
 345/*----------------------------------------------------------------*/
 346
 347static void build_key(dm_oblock_t oblock, struct dm_cell_key *key)
 348{
 349        key->virtual = 0;
 350        key->dev = 0;
 351        key->block = from_oblock(oblock);
 352}
 353
 354/*
 355 * The caller hands in a preallocated cell, and a free function for it.
 356 * The cell will be freed if there's an error, or if it wasn't used because
 357 * a cell with that key already exists.
 358 */
 359typedef void (*cell_free_fn)(void *context, struct dm_bio_prison_cell *cell);
 360
 361static int bio_detain(struct cache *cache, dm_oblock_t oblock,
 362                      struct bio *bio, struct dm_bio_prison_cell *cell_prealloc,
 363                      cell_free_fn free_fn, void *free_context,
 364                      struct dm_bio_prison_cell **cell_result)
 365{
 366        int r;
 367        struct dm_cell_key key;
 368
 369        build_key(oblock, &key);
 370        r = dm_bio_detain(cache->prison, &key, bio, cell_prealloc, cell_result);
 371        if (r)
 372                free_fn(free_context, cell_prealloc);
 373
 374        return r;
 375}
 376
 377static int get_cell(struct cache *cache,
 378                    dm_oblock_t oblock,
 379                    struct prealloc *structs,
 380                    struct dm_bio_prison_cell **cell_result)
 381{
 382        int r;
 383        struct dm_cell_key key;
 384        struct dm_bio_prison_cell *cell_prealloc;
 385
 386        cell_prealloc = prealloc_get_cell(structs);
 387
 388        build_key(oblock, &key);
 389        r = dm_get_cell(cache->prison, &key, cell_prealloc, cell_result);
 390        if (r)
 391                prealloc_put_cell(structs, cell_prealloc);
 392
 393        return r;
 394}
 395
 396 /*----------------------------------------------------------------*/
 397
 398static bool is_dirty(struct cache *cache, dm_cblock_t b)
 399{
 400        return test_bit(from_cblock(b), cache->dirty_bitset);
 401}
 402
 403static void set_dirty(struct cache *cache, dm_oblock_t oblock, dm_cblock_t cblock)
 404{
 405        if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
 406                cache->nr_dirty = to_cblock(from_cblock(cache->nr_dirty) + 1);
 407                policy_set_dirty(cache->policy, oblock);
 408        }
 409}
 410
 411static void clear_dirty(struct cache *cache, dm_oblock_t oblock, dm_cblock_t cblock)
 412{
 413        if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
 414                policy_clear_dirty(cache->policy, oblock);
 415                cache->nr_dirty = to_cblock(from_cblock(cache->nr_dirty) - 1);
 416                if (!from_cblock(cache->nr_dirty))
 417                        dm_table_event(cache->ti->table);
 418        }
 419}
 420
 421/*----------------------------------------------------------------*/
 422static bool block_size_is_power_of_two(struct cache *cache)
 423{
 424        return cache->sectors_per_block_shift >= 0;
 425}
 426
 427static dm_block_t block_div(dm_block_t b, uint32_t n)
 428{
 429        do_div(b, n);
 430
 431        return b;
 432}
 433
 434static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
 435{
 436        uint32_t discard_blocks = cache->discard_block_size;
 437        dm_block_t b = from_oblock(oblock);
 438
 439        if (!block_size_is_power_of_two(cache))
 440                discard_blocks = discard_blocks / cache->sectors_per_block;
 441        else
 442                discard_blocks >>= cache->sectors_per_block_shift;
 443
 444        b = block_div(b, discard_blocks);
 445
 446        return to_dblock(b);
 447}
 448
 449static void set_discard(struct cache *cache, dm_dblock_t b)
 450{
 451        unsigned long flags;
 452
 453        atomic_inc(&cache->stats.discard_count);
 454
 455        spin_lock_irqsave(&cache->lock, flags);
 456        set_bit(from_dblock(b), cache->discard_bitset);
 457        spin_unlock_irqrestore(&cache->lock, flags);
 458}
 459
 460static void clear_discard(struct cache *cache, dm_dblock_t b)
 461{
 462        unsigned long flags;
 463
 464        spin_lock_irqsave(&cache->lock, flags);
 465        clear_bit(from_dblock(b), cache->discard_bitset);
 466        spin_unlock_irqrestore(&cache->lock, flags);
 467}
 468
 469static bool is_discarded(struct cache *cache, dm_dblock_t b)
 470{
 471        int r;
 472        unsigned long flags;
 473
 474        spin_lock_irqsave(&cache->lock, flags);
 475        r = test_bit(from_dblock(b), cache->discard_bitset);
 476        spin_unlock_irqrestore(&cache->lock, flags);
 477
 478        return r;
 479}
 480
 481static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
 482{
 483        int r;
 484        unsigned long flags;
 485
 486        spin_lock_irqsave(&cache->lock, flags);
 487        r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
 488                     cache->discard_bitset);
 489        spin_unlock_irqrestore(&cache->lock, flags);
 490
 491        return r;
 492}
 493
 494/*----------------------------------------------------------------*/
 495
 496static void load_stats(struct cache *cache)
 497{
 498        struct dm_cache_statistics stats;
 499
 500        dm_cache_metadata_get_stats(cache->cmd, &stats);
 501        atomic_set(&cache->stats.read_hit, stats.read_hits);
 502        atomic_set(&cache->stats.read_miss, stats.read_misses);
 503        atomic_set(&cache->stats.write_hit, stats.write_hits);
 504        atomic_set(&cache->stats.write_miss, stats.write_misses);
 505}
 506
 507static void save_stats(struct cache *cache)
 508{
 509        struct dm_cache_statistics stats;
 510
 511        stats.read_hits = atomic_read(&cache->stats.read_hit);
 512        stats.read_misses = atomic_read(&cache->stats.read_miss);
 513        stats.write_hits = atomic_read(&cache->stats.write_hit);
 514        stats.write_misses = atomic_read(&cache->stats.write_miss);
 515
 516        dm_cache_metadata_set_stats(cache->cmd, &stats);
 517}
 518
 519/*----------------------------------------------------------------
 520 * Per bio data
 521 *--------------------------------------------------------------*/
 522
 523/*
 524 * If using writeback, leave out struct per_bio_data's writethrough fields.
 525 */
 526#define PB_DATA_SIZE_WB (offsetof(struct per_bio_data, cache))
 527#define PB_DATA_SIZE_WT (sizeof(struct per_bio_data))
 528
 529static size_t get_per_bio_data_size(struct cache *cache)
 530{
 531        return cache->features.write_through ? PB_DATA_SIZE_WT : PB_DATA_SIZE_WB;
 532}
 533
 534static struct per_bio_data *get_per_bio_data(struct bio *bio, size_t data_size)
 535{
 536        struct per_bio_data *pb = dm_per_bio_data(bio, data_size);
 537        BUG_ON(!pb);
 538        return pb;
 539}
 540
 541static struct per_bio_data *init_per_bio_data(struct bio *bio, size_t data_size)
 542{
 543        struct per_bio_data *pb = get_per_bio_data(bio, data_size);
 544
 545        pb->tick = false;
 546        pb->req_nr = dm_bio_get_target_bio_nr(bio);
 547        pb->all_io_entry = NULL;
 548
 549        return pb;
 550}
 551
 552/*----------------------------------------------------------------
 553 * Remapping
 554 *--------------------------------------------------------------*/
 555static void remap_to_origin(struct cache *cache, struct bio *bio)
 556{
 557        bio->bi_bdev = cache->origin_dev->bdev;
 558}
 559
 560static void remap_to_cache(struct cache *cache, struct bio *bio,
 561                           dm_cblock_t cblock)
 562{
 563        sector_t bi_sector = bio->bi_sector;
 564
 565        bio->bi_bdev = cache->cache_dev->bdev;
 566        if (!block_size_is_power_of_two(cache))
 567                bio->bi_sector = (from_cblock(cblock) * cache->sectors_per_block) +
 568                                sector_div(bi_sector, cache->sectors_per_block);
 569        else
 570                bio->bi_sector = (from_cblock(cblock) << cache->sectors_per_block_shift) |
 571                                (bi_sector & (cache->sectors_per_block - 1));
 572}
 573
 574static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
 575{
 576        unsigned long flags;
 577        size_t pb_data_size = get_per_bio_data_size(cache);
 578        struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
 579
 580        spin_lock_irqsave(&cache->lock, flags);
 581        if (cache->need_tick_bio &&
 582            !(bio->bi_rw & (REQ_FUA | REQ_FLUSH | REQ_DISCARD))) {
 583                pb->tick = true;
 584                cache->need_tick_bio = false;
 585        }
 586        spin_unlock_irqrestore(&cache->lock, flags);
 587}
 588
 589static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
 590                                  dm_oblock_t oblock)
 591{
 592        check_if_tick_bio_needed(cache, bio);
 593        remap_to_origin(cache, bio);
 594        if (bio_data_dir(bio) == WRITE)
 595                clear_discard(cache, oblock_to_dblock(cache, oblock));
 596}
 597
 598static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
 599                                 dm_oblock_t oblock, dm_cblock_t cblock)
 600{
 601        remap_to_cache(cache, bio, cblock);
 602        if (bio_data_dir(bio) == WRITE) {
 603                set_dirty(cache, oblock, cblock);
 604                clear_discard(cache, oblock_to_dblock(cache, oblock));
 605        }
 606}
 607
 608static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
 609{
 610        sector_t block_nr = bio->bi_sector;
 611
 612        if (!block_size_is_power_of_two(cache))
 613                (void) sector_div(block_nr, cache->sectors_per_block);
 614        else
 615                block_nr >>= cache->sectors_per_block_shift;
 616
 617        return to_oblock(block_nr);
 618}
 619
 620static int bio_triggers_commit(struct cache *cache, struct bio *bio)
 621{
 622        return bio->bi_rw & (REQ_FLUSH | REQ_FUA);
 623}
 624
 625static void issue(struct cache *cache, struct bio *bio)
 626{
 627        unsigned long flags;
 628
 629        if (!bio_triggers_commit(cache, bio)) {
 630                generic_make_request(bio);
 631                return;
 632        }
 633
 634        /*
 635         * Batch together any bios that trigger commits and then issue a
 636         * single commit for them in do_worker().
 637         */
 638        spin_lock_irqsave(&cache->lock, flags);
 639        cache->commit_requested = true;
 640        bio_list_add(&cache->deferred_flush_bios, bio);
 641        spin_unlock_irqrestore(&cache->lock, flags);
 642}
 643
 644static void defer_writethrough_bio(struct cache *cache, struct bio *bio)
 645{
 646        unsigned long flags;
 647
 648        spin_lock_irqsave(&cache->lock, flags);
 649        bio_list_add(&cache->deferred_writethrough_bios, bio);
 650        spin_unlock_irqrestore(&cache->lock, flags);
 651
 652        wake_worker(cache);
 653}
 654
 655static void writethrough_endio(struct bio *bio, int err)
 656{
 657        struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT);
 658        bio->bi_end_io = pb->saved_bi_end_io;
 659
 660        if (err) {
 661                bio_endio(bio, err);
 662                return;
 663        }
 664
 665        dm_bio_restore(&pb->bio_details, bio);
 666        remap_to_cache(pb->cache, bio, pb->cblock);
 667
 668        /*
 669         * We can't issue this bio directly, since we're in interrupt
 670         * context.  So it get's put on a bio list for processing by the
 671         * worker thread.
 672         */
 673        defer_writethrough_bio(pb->cache, bio);
 674}
 675
 676/*
 677 * When running in writethrough mode we need to send writes to clean blocks
 678 * to both the cache and origin devices.  In future we'd like to clone the
 679 * bio and send them in parallel, but for now we're doing them in
 680 * series as this is easier.
 681 */
 682static void remap_to_origin_then_cache(struct cache *cache, struct bio *bio,
 683                                       dm_oblock_t oblock, dm_cblock_t cblock)
 684{
 685        struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT);
 686
 687        pb->cache = cache;
 688        pb->cblock = cblock;
 689        pb->saved_bi_end_io = bio->bi_end_io;
 690        dm_bio_record(&pb->bio_details, bio);
 691        bio->bi_end_io = writethrough_endio;
 692
 693        remap_to_origin_clear_discard(pb->cache, bio, oblock);
 694}
 695
 696/*----------------------------------------------------------------
 697 * Migration processing
 698 *
 699 * Migration covers moving data from the origin device to the cache, or
 700 * vice versa.
 701 *--------------------------------------------------------------*/
 702static void free_migration(struct dm_cache_migration *mg)
 703{
 704        mempool_free(mg, mg->cache->migration_pool);
 705}
 706
 707static void inc_nr_migrations(struct cache *cache)
 708{
 709        atomic_inc(&cache->nr_migrations);
 710}
 711
 712static void dec_nr_migrations(struct cache *cache)
 713{
 714        atomic_dec(&cache->nr_migrations);
 715
 716        /*
 717         * Wake the worker in case we're suspending the target.
 718         */
 719        wake_up(&cache->migration_wait);
 720}
 721
 722static void __cell_defer(struct cache *cache, struct dm_bio_prison_cell *cell,
 723                         bool holder)
 724{
 725        (holder ? dm_cell_release : dm_cell_release_no_holder)
 726                (cache->prison, cell, &cache->deferred_bios);
 727        free_prison_cell(cache, cell);
 728}
 729
 730static void cell_defer(struct cache *cache, struct dm_bio_prison_cell *cell,
 731                       bool holder)
 732{
 733        unsigned long flags;
 734
 735        spin_lock_irqsave(&cache->lock, flags);
 736        __cell_defer(cache, cell, holder);
 737        spin_unlock_irqrestore(&cache->lock, flags);
 738
 739        wake_worker(cache);
 740}
 741
 742static void cleanup_migration(struct dm_cache_migration *mg)
 743{
 744        dec_nr_migrations(mg->cache);
 745        free_migration(mg);
 746}
 747
 748static void migration_failure(struct dm_cache_migration *mg)
 749{
 750        struct cache *cache = mg->cache;
 751
 752        if (mg->writeback) {
 753                DMWARN_LIMIT("writeback failed; couldn't copy block");
 754                set_dirty(cache, mg->old_oblock, mg->cblock);
 755                cell_defer(cache, mg->old_ocell, false);
 756
 757        } else if (mg->demote) {
 758                DMWARN_LIMIT("demotion failed; couldn't copy block");
 759                policy_force_mapping(cache->policy, mg->new_oblock, mg->old_oblock);
 760
 761                cell_defer(cache, mg->old_ocell, mg->promote ? 0 : 1);
 762                if (mg->promote)
 763                        cell_defer(cache, mg->new_ocell, 1);
 764        } else {
 765                DMWARN_LIMIT("promotion failed; couldn't copy block");
 766                policy_remove_mapping(cache->policy, mg->new_oblock);
 767                cell_defer(cache, mg->new_ocell, 1);
 768        }
 769
 770        cleanup_migration(mg);
 771}
 772
 773static void migration_success_pre_commit(struct dm_cache_migration *mg)
 774{
 775        unsigned long flags;
 776        struct cache *cache = mg->cache;
 777
 778        if (mg->writeback) {
 779                cell_defer(cache, mg->old_ocell, false);
 780                clear_dirty(cache, mg->old_oblock, mg->cblock);
 781                cleanup_migration(mg);
 782                return;
 783
 784        } else if (mg->demote) {
 785                if (dm_cache_remove_mapping(cache->cmd, mg->cblock)) {
 786                        DMWARN_LIMIT("demotion failed; couldn't update on disk metadata");
 787                        policy_force_mapping(cache->policy, mg->new_oblock,
 788                                             mg->old_oblock);
 789                        if (mg->promote)
 790                                cell_defer(cache, mg->new_ocell, true);
 791                        cleanup_migration(mg);
 792                        return;
 793                }
 794        } else {
 795                if (dm_cache_insert_mapping(cache->cmd, mg->cblock, mg->new_oblock)) {
 796                        DMWARN_LIMIT("promotion failed; couldn't update on disk metadata");
 797                        policy_remove_mapping(cache->policy, mg->new_oblock);
 798                        cleanup_migration(mg);
 799                        return;
 800                }
 801        }
 802
 803        spin_lock_irqsave(&cache->lock, flags);
 804        list_add_tail(&mg->list, &cache->need_commit_migrations);
 805        cache->commit_requested = true;
 806        spin_unlock_irqrestore(&cache->lock, flags);
 807}
 808
 809static void migration_success_post_commit(struct dm_cache_migration *mg)
 810{
 811        unsigned long flags;
 812        struct cache *cache = mg->cache;
 813
 814        if (mg->writeback) {
 815                DMWARN("writeback unexpectedly triggered commit");
 816                return;
 817
 818        } else if (mg->demote) {
 819                cell_defer(cache, mg->old_ocell, mg->promote ? 0 : 1);
 820
 821                if (mg->promote) {
 822                        mg->demote = false;
 823
 824                        spin_lock_irqsave(&cache->lock, flags);
 825                        list_add_tail(&mg->list, &cache->quiesced_migrations);
 826                        spin_unlock_irqrestore(&cache->lock, flags);
 827
 828                } else
 829                        cleanup_migration(mg);
 830
 831        } else {
 832                cell_defer(cache, mg->new_ocell, true);
 833                clear_dirty(cache, mg->new_oblock, mg->cblock);
 834                cleanup_migration(mg);
 835        }
 836}
 837
 838static void copy_complete(int read_err, unsigned long write_err, void *context)
 839{
 840        unsigned long flags;
 841        struct dm_cache_migration *mg = (struct dm_cache_migration *) context;
 842        struct cache *cache = mg->cache;
 843
 844        if (read_err || write_err)
 845                mg->err = true;
 846
 847        spin_lock_irqsave(&cache->lock, flags);
 848        list_add_tail(&mg->list, &cache->completed_migrations);
 849        spin_unlock_irqrestore(&cache->lock, flags);
 850
 851        wake_worker(cache);
 852}
 853
 854static void issue_copy_real(struct dm_cache_migration *mg)
 855{
 856        int r;
 857        struct dm_io_region o_region, c_region;
 858        struct cache *cache = mg->cache;
 859
 860        o_region.bdev = cache->origin_dev->bdev;
 861        o_region.count = cache->sectors_per_block;
 862
 863        c_region.bdev = cache->cache_dev->bdev;
 864        c_region.sector = from_cblock(mg->cblock) * cache->sectors_per_block;
 865        c_region.count = cache->sectors_per_block;
 866
 867        if (mg->writeback || mg->demote) {
 868                /* demote */
 869                o_region.sector = from_oblock(mg->old_oblock) * cache->sectors_per_block;
 870                r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, mg);
 871        } else {
 872                /* promote */
 873                o_region.sector = from_oblock(mg->new_oblock) * cache->sectors_per_block;
 874                r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, mg);
 875        }
 876
 877        if (r < 0)
 878                migration_failure(mg);
 879}
 880
 881static void avoid_copy(struct dm_cache_migration *mg)
 882{
 883        atomic_inc(&mg->cache->stats.copies_avoided);
 884        migration_success_pre_commit(mg);
 885}
 886
 887static void issue_copy(struct dm_cache_migration *mg)
 888{
 889        bool avoid;
 890        struct cache *cache = mg->cache;
 891
 892        if (mg->writeback || mg->demote)
 893                avoid = !is_dirty(cache, mg->cblock) ||
 894                        is_discarded_oblock(cache, mg->old_oblock);
 895        else
 896                avoid = is_discarded_oblock(cache, mg->new_oblock);
 897
 898        avoid ? avoid_copy(mg) : issue_copy_real(mg);
 899}
 900
 901static void complete_migration(struct dm_cache_migration *mg)
 902{
 903        if (mg->err)
 904                migration_failure(mg);
 905        else
 906                migration_success_pre_commit(mg);
 907}
 908
 909static void process_migrations(struct cache *cache, struct list_head *head,
 910                               void (*fn)(struct dm_cache_migration *))
 911{
 912        unsigned long flags;
 913        struct list_head list;
 914        struct dm_cache_migration *mg, *tmp;
 915
 916        INIT_LIST_HEAD(&list);
 917        spin_lock_irqsave(&cache->lock, flags);
 918        list_splice_init(head, &list);
 919        spin_unlock_irqrestore(&cache->lock, flags);
 920
 921        list_for_each_entry_safe(mg, tmp, &list, list)
 922                fn(mg);
 923}
 924
 925static void __queue_quiesced_migration(struct dm_cache_migration *mg)
 926{
 927        list_add_tail(&mg->list, &mg->cache->quiesced_migrations);
 928}
 929
 930static void queue_quiesced_migration(struct dm_cache_migration *mg)
 931{
 932        unsigned long flags;
 933        struct cache *cache = mg->cache;
 934
 935        spin_lock_irqsave(&cache->lock, flags);
 936        __queue_quiesced_migration(mg);
 937        spin_unlock_irqrestore(&cache->lock, flags);
 938
 939        wake_worker(cache);
 940}
 941
 942static void queue_quiesced_migrations(struct cache *cache, struct list_head *work)
 943{
 944        unsigned long flags;
 945        struct dm_cache_migration *mg, *tmp;
 946
 947        spin_lock_irqsave(&cache->lock, flags);
 948        list_for_each_entry_safe(mg, tmp, work, list)
 949                __queue_quiesced_migration(mg);
 950        spin_unlock_irqrestore(&cache->lock, flags);
 951
 952        wake_worker(cache);
 953}
 954
 955static void check_for_quiesced_migrations(struct cache *cache,
 956                                          struct per_bio_data *pb)
 957{
 958        struct list_head work;
 959
 960        if (!pb->all_io_entry)
 961                return;
 962
 963        INIT_LIST_HEAD(&work);
 964        if (pb->all_io_entry)
 965                dm_deferred_entry_dec(pb->all_io_entry, &work);
 966
 967        if (!list_empty(&work))
 968                queue_quiesced_migrations(cache, &work);
 969}
 970
 971static void quiesce_migration(struct dm_cache_migration *mg)
 972{
 973        if (!dm_deferred_set_add_work(mg->cache->all_io_ds, &mg->list))
 974                queue_quiesced_migration(mg);
 975}
 976
 977static void promote(struct cache *cache, struct prealloc *structs,
 978                    dm_oblock_t oblock, dm_cblock_t cblock,
 979                    struct dm_bio_prison_cell *cell)
 980{
 981        struct dm_cache_migration *mg = prealloc_get_migration(structs);
 982
 983        mg->err = false;
 984        mg->writeback = false;
 985        mg->demote = false;
 986        mg->promote = true;
 987        mg->cache = cache;
 988        mg->new_oblock = oblock;
 989        mg->cblock = cblock;
 990        mg->old_ocell = NULL;
 991        mg->new_ocell = cell;
 992        mg->start_jiffies = jiffies;
 993
 994        inc_nr_migrations(cache);
 995        quiesce_migration(mg);
 996}
 997
 998static void writeback(struct cache *cache, struct prealloc *structs,
 999                      dm_oblock_t oblock, dm_cblock_t cblock,
1000                      struct dm_bio_prison_cell *cell)
1001{
1002        struct dm_cache_migration *mg = prealloc_get_migration(structs);
1003
1004        mg->err = false;
1005        mg->writeback = true;
1006        mg->demote = false;
1007        mg->promote = false;
1008        mg->cache = cache;
1009        mg->old_oblock = oblock;
1010        mg->cblock = cblock;
1011        mg->old_ocell = cell;
1012        mg->new_ocell = NULL;
1013        mg->start_jiffies = jiffies;
1014
1015        inc_nr_migrations(cache);
1016        quiesce_migration(mg);
1017}
1018
1019static void demote_then_promote(struct cache *cache, struct prealloc *structs,
1020                                dm_oblock_t old_oblock, dm_oblock_t new_oblock,
1021                                dm_cblock_t cblock,
1022                                struct dm_bio_prison_cell *old_ocell,
1023                                struct dm_bio_prison_cell *new_ocell)
1024{
1025        struct dm_cache_migration *mg = prealloc_get_migration(structs);
1026
1027        mg->err = false;
1028        mg->writeback = false;
1029        mg->demote = true;
1030        mg->promote = true;
1031        mg->cache = cache;
1032        mg->old_oblock = old_oblock;
1033        mg->new_oblock = new_oblock;
1034        mg->cblock = cblock;
1035        mg->old_ocell = old_ocell;
1036        mg->new_ocell = new_ocell;
1037        mg->start_jiffies = jiffies;
1038
1039        inc_nr_migrations(cache);
1040        quiesce_migration(mg);
1041}
1042
1043/*----------------------------------------------------------------
1044 * bio processing
1045 *--------------------------------------------------------------*/
1046static void defer_bio(struct cache *cache, struct bio *bio)
1047{
1048        unsigned long flags;
1049
1050        spin_lock_irqsave(&cache->lock, flags);
1051        bio_list_add(&cache->deferred_bios, bio);
1052        spin_unlock_irqrestore(&cache->lock, flags);
1053
1054        wake_worker(cache);
1055}
1056
1057static void process_flush_bio(struct cache *cache, struct bio *bio)
1058{
1059        size_t pb_data_size = get_per_bio_data_size(cache);
1060        struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1061
1062        BUG_ON(bio->bi_size);
1063        if (!pb->req_nr)
1064                remap_to_origin(cache, bio);
1065        else
1066                remap_to_cache(cache, bio, 0);
1067
1068        issue(cache, bio);
1069}
1070
1071/*
1072 * People generally discard large parts of a device, eg, the whole device
1073 * when formatting.  Splitting these large discards up into cache block
1074 * sized ios and then quiescing (always neccessary for discard) takes too
1075 * long.
1076 *
1077 * We keep it simple, and allow any size of discard to come in, and just
1078 * mark off blocks on the discard bitset.  No passdown occurs!
1079 *
1080 * To implement passdown we need to change the bio_prison such that a cell
1081 * can have a key that spans many blocks.
1082 */
1083static void process_discard_bio(struct cache *cache, struct bio *bio)
1084{
1085        dm_block_t start_block = dm_sector_div_up(bio->bi_sector,
1086                                                  cache->discard_block_size);
1087        dm_block_t end_block = bio->bi_sector + bio_sectors(bio);
1088        dm_block_t b;
1089
1090        end_block = block_div(end_block, cache->discard_block_size);
1091
1092        for (b = start_block; b < end_block; b++)
1093                set_discard(cache, to_dblock(b));
1094
1095        bio_endio(bio, 0);
1096}
1097
1098static bool spare_migration_bandwidth(struct cache *cache)
1099{
1100        sector_t current_volume = (atomic_read(&cache->nr_migrations) + 1) *
1101                cache->sectors_per_block;
1102        return current_volume < cache->migration_threshold;
1103}
1104
1105static bool is_writethrough_io(struct cache *cache, struct bio *bio,
1106                               dm_cblock_t cblock)
1107{
1108        return bio_data_dir(bio) == WRITE &&
1109                cache->features.write_through && !is_dirty(cache, cblock);
1110}
1111
1112static void inc_hit_counter(struct cache *cache, struct bio *bio)
1113{
1114        atomic_inc(bio_data_dir(bio) == READ ?
1115                   &cache->stats.read_hit : &cache->stats.write_hit);
1116}
1117
1118static void inc_miss_counter(struct cache *cache, struct bio *bio)
1119{
1120        atomic_inc(bio_data_dir(bio) == READ ?
1121                   &cache->stats.read_miss : &cache->stats.write_miss);
1122}
1123
1124static void process_bio(struct cache *cache, struct prealloc *structs,
1125                        struct bio *bio)
1126{
1127        int r;
1128        bool release_cell = true;
1129        dm_oblock_t block = get_bio_block(cache, bio);
1130        struct dm_bio_prison_cell *cell_prealloc, *old_ocell, *new_ocell;
1131        struct policy_result lookup_result;
1132        size_t pb_data_size = get_per_bio_data_size(cache);
1133        struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1134        bool discarded_block = is_discarded_oblock(cache, block);
1135        bool can_migrate = discarded_block || spare_migration_bandwidth(cache);
1136
1137        /*
1138         * Check to see if that block is currently migrating.
1139         */
1140        cell_prealloc = prealloc_get_cell(structs);
1141        r = bio_detain(cache, block, bio, cell_prealloc,
1142                       (cell_free_fn) prealloc_put_cell,
1143                       structs, &new_ocell);
1144        if (r > 0)
1145                return;
1146
1147        r = policy_map(cache->policy, block, true, can_migrate, discarded_block,
1148                       bio, &lookup_result);
1149
1150        if (r == -EWOULDBLOCK)
1151                /* migration has been denied */
1152                lookup_result.op = POLICY_MISS;
1153
1154        switch (lookup_result.op) {
1155        case POLICY_HIT:
1156                inc_hit_counter(cache, bio);
1157                pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
1158
1159                if (is_writethrough_io(cache, bio, lookup_result.cblock))
1160                        remap_to_origin_then_cache(cache, bio, block, lookup_result.cblock);
1161                else
1162                        remap_to_cache_dirty(cache, bio, block, lookup_result.cblock);
1163
1164                issue(cache, bio);
1165                break;
1166
1167        case POLICY_MISS:
1168                inc_miss_counter(cache, bio);
1169                pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
1170                remap_to_origin_clear_discard(cache, bio, block);
1171                issue(cache, bio);
1172                break;
1173
1174        case POLICY_NEW:
1175                atomic_inc(&cache->stats.promotion);
1176                promote(cache, structs, block, lookup_result.cblock, new_ocell);
1177                release_cell = false;
1178                break;
1179
1180        case POLICY_REPLACE:
1181                cell_prealloc = prealloc_get_cell(structs);
1182                r = bio_detain(cache, lookup_result.old_oblock, bio, cell_prealloc,
1183                               (cell_free_fn) prealloc_put_cell,
1184                               structs, &old_ocell);
1185                if (r > 0) {
1186                        /*
1187                         * We have to be careful to avoid lock inversion of
1188                         * the cells.  So we back off, and wait for the
1189                         * old_ocell to become free.
1190                         */
1191                        policy_force_mapping(cache->policy, block,
1192                                             lookup_result.old_oblock);
1193                        atomic_inc(&cache->stats.cache_cell_clash);
1194                        break;
1195                }
1196                atomic_inc(&cache->stats.demotion);
1197                atomic_inc(&cache->stats.promotion);
1198
1199                demote_then_promote(cache, structs, lookup_result.old_oblock,
1200                                    block, lookup_result.cblock,
1201                                    old_ocell, new_ocell);
1202                release_cell = false;
1203                break;
1204
1205        default:
1206                DMERR_LIMIT("%s: erroring bio, unknown policy op: %u", __func__,
1207                            (unsigned) lookup_result.op);
1208                bio_io_error(bio);
1209        }
1210
1211        if (release_cell)
1212                cell_defer(cache, new_ocell, false);
1213}
1214
1215static int need_commit_due_to_time(struct cache *cache)
1216{
1217        return jiffies < cache->last_commit_jiffies ||
1218               jiffies > cache->last_commit_jiffies + COMMIT_PERIOD;
1219}
1220
1221static int commit_if_needed(struct cache *cache)
1222{
1223        if (dm_cache_changed_this_transaction(cache->cmd) &&
1224            (cache->commit_requested || need_commit_due_to_time(cache))) {
1225                atomic_inc(&cache->stats.commit_count);
1226                cache->last_commit_jiffies = jiffies;
1227                cache->commit_requested = false;
1228                return dm_cache_commit(cache->cmd, false);
1229        }
1230
1231        return 0;
1232}
1233
1234static void process_deferred_bios(struct cache *cache)
1235{
1236        unsigned long flags;
1237        struct bio_list bios;
1238        struct bio *bio;
1239        struct prealloc structs;
1240
1241        memset(&structs, 0, sizeof(structs));
1242        bio_list_init(&bios);
1243
1244        spin_lock_irqsave(&cache->lock, flags);
1245        bio_list_merge(&bios, &cache->deferred_bios);
1246        bio_list_init(&cache->deferred_bios);
1247        spin_unlock_irqrestore(&cache->lock, flags);
1248
1249        while (!bio_list_empty(&bios)) {
1250                /*
1251                 * If we've got no free migration structs, and processing
1252                 * this bio might require one, we pause until there are some
1253                 * prepared mappings to process.
1254                 */
1255                if (prealloc_data_structs(cache, &structs)) {
1256                        spin_lock_irqsave(&cache->lock, flags);
1257                        bio_list_merge(&cache->deferred_bios, &bios);
1258                        spin_unlock_irqrestore(&cache->lock, flags);
1259                        break;
1260                }
1261
1262                bio = bio_list_pop(&bios);
1263
1264                if (bio->bi_rw & REQ_FLUSH)
1265                        process_flush_bio(cache, bio);
1266                else if (bio->bi_rw & REQ_DISCARD)
1267                        process_discard_bio(cache, bio);
1268                else
1269                        process_bio(cache, &structs, bio);
1270        }
1271
1272        prealloc_free_structs(cache, &structs);
1273}
1274
1275static void process_deferred_flush_bios(struct cache *cache, bool submit_bios)
1276{
1277        unsigned long flags;
1278        struct bio_list bios;
1279        struct bio *bio;
1280
1281        bio_list_init(&bios);
1282
1283        spin_lock_irqsave(&cache->lock, flags);
1284        bio_list_merge(&bios, &cache->deferred_flush_bios);
1285        bio_list_init(&cache->deferred_flush_bios);
1286        spin_unlock_irqrestore(&cache->lock, flags);
1287
1288        while ((bio = bio_list_pop(&bios)))
1289                submit_bios ? generic_make_request(bio) : bio_io_error(bio);
1290}
1291
1292static void process_deferred_writethrough_bios(struct cache *cache)
1293{
1294        unsigned long flags;
1295        struct bio_list bios;
1296        struct bio *bio;
1297
1298        bio_list_init(&bios);
1299
1300        spin_lock_irqsave(&cache->lock, flags);
1301        bio_list_merge(&bios, &cache->deferred_writethrough_bios);
1302        bio_list_init(&cache->deferred_writethrough_bios);
1303        spin_unlock_irqrestore(&cache->lock, flags);
1304
1305        while ((bio = bio_list_pop(&bios)))
1306                generic_make_request(bio);
1307}
1308
1309static void writeback_some_dirty_blocks(struct cache *cache)
1310{
1311        int r = 0;
1312        dm_oblock_t oblock;
1313        dm_cblock_t cblock;
1314        struct prealloc structs;
1315        struct dm_bio_prison_cell *old_ocell;
1316
1317        memset(&structs, 0, sizeof(structs));
1318
1319        while (spare_migration_bandwidth(cache)) {
1320                if (prealloc_data_structs(cache, &structs))
1321                        break;
1322
1323                r = policy_writeback_work(cache->policy, &oblock, &cblock);
1324                if (r)
1325                        break;
1326
1327                r = get_cell(cache, oblock, &structs, &old_ocell);
1328                if (r) {
1329                        policy_set_dirty(cache->policy, oblock);
1330                        break;
1331                }
1332
1333                writeback(cache, &structs, oblock, cblock, old_ocell);
1334        }
1335
1336        prealloc_free_structs(cache, &structs);
1337}
1338
1339/*----------------------------------------------------------------
1340 * Main worker loop
1341 *--------------------------------------------------------------*/
1342static void start_quiescing(struct cache *cache)
1343{
1344        unsigned long flags;
1345
1346        spin_lock_irqsave(&cache->lock, flags);
1347        cache->quiescing = 1;
1348        spin_unlock_irqrestore(&cache->lock, flags);
1349}
1350
1351static void stop_quiescing(struct cache *cache)
1352{
1353        unsigned long flags;
1354
1355        spin_lock_irqsave(&cache->lock, flags);
1356        cache->quiescing = 0;
1357        spin_unlock_irqrestore(&cache->lock, flags);
1358}
1359
1360static bool is_quiescing(struct cache *cache)
1361{
1362        int r;
1363        unsigned long flags;
1364
1365        spin_lock_irqsave(&cache->lock, flags);
1366        r = cache->quiescing;
1367        spin_unlock_irqrestore(&cache->lock, flags);
1368
1369        return r;
1370}
1371
1372static void wait_for_migrations(struct cache *cache)
1373{
1374        wait_event(cache->migration_wait, !atomic_read(&cache->nr_migrations));
1375}
1376
1377static void stop_worker(struct cache *cache)
1378{
1379        cancel_delayed_work(&cache->waker);
1380        flush_workqueue(cache->wq);
1381}
1382
1383static void requeue_deferred_io(struct cache *cache)
1384{
1385        struct bio *bio;
1386        struct bio_list bios;
1387
1388        bio_list_init(&bios);
1389        bio_list_merge(&bios, &cache->deferred_bios);
1390        bio_list_init(&cache->deferred_bios);
1391
1392        while ((bio = bio_list_pop(&bios)))
1393                bio_endio(bio, DM_ENDIO_REQUEUE);
1394}
1395
1396static int more_work(struct cache *cache)
1397{
1398        if (is_quiescing(cache))
1399                return !list_empty(&cache->quiesced_migrations) ||
1400                        !list_empty(&cache->completed_migrations) ||
1401                        !list_empty(&cache->need_commit_migrations);
1402        else
1403                return !bio_list_empty(&cache->deferred_bios) ||
1404                        !bio_list_empty(&cache->deferred_flush_bios) ||
1405                        !bio_list_empty(&cache->deferred_writethrough_bios) ||
1406                        !list_empty(&cache->quiesced_migrations) ||
1407                        !list_empty(&cache->completed_migrations) ||
1408                        !list_empty(&cache->need_commit_migrations);
1409}
1410
1411static void do_worker(struct work_struct *ws)
1412{
1413        struct cache *cache = container_of(ws, struct cache, worker);
1414
1415        do {
1416                if (!is_quiescing(cache))
1417                        process_deferred_bios(cache);
1418
1419                process_migrations(cache, &cache->quiesced_migrations, issue_copy);
1420                process_migrations(cache, &cache->completed_migrations, complete_migration);
1421
1422                writeback_some_dirty_blocks(cache);
1423
1424                process_deferred_writethrough_bios(cache);
1425
1426                if (commit_if_needed(cache)) {
1427                        process_deferred_flush_bios(cache, false);
1428
1429                        /*
1430                         * FIXME: rollback metadata or just go into a
1431                         * failure mode and error everything
1432                         */
1433                } else {
1434                        process_deferred_flush_bios(cache, true);
1435                        process_migrations(cache, &cache->need_commit_migrations,
1436                                           migration_success_post_commit);
1437                }
1438        } while (more_work(cache));
1439}
1440
1441/*
1442 * We want to commit periodically so that not too much
1443 * unwritten metadata builds up.
1444 */
1445static void do_waker(struct work_struct *ws)
1446{
1447        struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
1448        wake_worker(cache);
1449        queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
1450}
1451
1452/*----------------------------------------------------------------*/
1453
1454static int is_congested(struct dm_dev *dev, int bdi_bits)
1455{
1456        struct request_queue *q = bdev_get_queue(dev->bdev);
1457        return bdi_congested(&q->backing_dev_info, bdi_bits);
1458}
1459
1460static int cache_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
1461{
1462        struct cache *cache = container_of(cb, struct cache, callbacks);
1463
1464        return is_congested(cache->origin_dev, bdi_bits) ||
1465                is_congested(cache->cache_dev, bdi_bits);
1466}
1467
1468/*----------------------------------------------------------------
1469 * Target methods
1470 *--------------------------------------------------------------*/
1471
1472/*
1473 * This function gets called on the error paths of the constructor, so we
1474 * have to cope with a partially initialised struct.
1475 */
1476static void destroy(struct cache *cache)
1477{
1478        unsigned i;
1479
1480        if (cache->next_migration)
1481                mempool_free(cache->next_migration, cache->migration_pool);
1482
1483        if (cache->migration_pool)
1484                mempool_destroy(cache->migration_pool);
1485
1486        if (cache->all_io_ds)
1487                dm_deferred_set_destroy(cache->all_io_ds);
1488
1489        if (cache->prison)
1490                dm_bio_prison_destroy(cache->prison);
1491
1492        if (cache->wq)
1493                destroy_workqueue(cache->wq);
1494
1495        if (cache->dirty_bitset)
1496                free_bitset(cache->dirty_bitset);
1497
1498        if (cache->discard_bitset)
1499                free_bitset(cache->discard_bitset);
1500
1501        if (cache->copier)
1502                dm_kcopyd_client_destroy(cache->copier);
1503
1504        if (cache->cmd)
1505                dm_cache_metadata_close(cache->cmd);
1506
1507        if (cache->metadata_dev)
1508                dm_put_device(cache->ti, cache->metadata_dev);
1509
1510        if (cache->origin_dev)
1511                dm_put_device(cache->ti, cache->origin_dev);
1512
1513        if (cache->cache_dev)
1514                dm_put_device(cache->ti, cache->cache_dev);
1515
1516        if (cache->policy)
1517                dm_cache_policy_destroy(cache->policy);
1518
1519        for (i = 0; i < cache->nr_ctr_args ; i++)
1520                kfree(cache->ctr_args[i]);
1521        kfree(cache->ctr_args);
1522
1523        kfree(cache);
1524}
1525
1526static void cache_dtr(struct dm_target *ti)
1527{
1528        struct cache *cache = ti->private;
1529
1530        destroy(cache);
1531}
1532
1533static sector_t get_dev_size(struct dm_dev *dev)
1534{
1535        return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
1536}
1537
1538/*----------------------------------------------------------------*/
1539
1540/*
1541 * Construct a cache device mapping.
1542 *
1543 * cache <metadata dev> <cache dev> <origin dev> <block size>
1544 *       <#feature args> [<feature arg>]*
1545 *       <policy> <#policy args> [<policy arg>]*
1546 *
1547 * metadata dev    : fast device holding the persistent metadata
1548 * cache dev       : fast device holding cached data blocks
1549 * origin dev      : slow device holding original data blocks
1550 * block size      : cache unit size in sectors
1551 *
1552 * #feature args   : number of feature arguments passed
1553 * feature args    : writethrough.  (The default is writeback.)
1554 *
1555 * policy          : the replacement policy to use
1556 * #policy args    : an even number of policy arguments corresponding
1557 *                   to key/value pairs passed to the policy
1558 * policy args     : key/value pairs passed to the policy
1559 *                   E.g. 'sequential_threshold 1024'
1560 *                   See cache-policies.txt for details.
1561 *
1562 * Optional feature arguments are:
1563 *   writethrough  : write through caching that prohibits cache block
1564 *                   content from being different from origin block content.
1565 *                   Without this argument, the default behaviour is to write
1566 *                   back cache block contents later for performance reasons,
1567 *                   so they may differ from the corresponding origin blocks.
1568 */
1569struct cache_args {
1570        struct dm_target *ti;
1571
1572        struct dm_dev *metadata_dev;
1573
1574        struct dm_dev *cache_dev;
1575        sector_t cache_sectors;
1576
1577        struct dm_dev *origin_dev;
1578        sector_t origin_sectors;
1579
1580        uint32_t block_size;
1581
1582        const char *policy_name;
1583        int policy_argc;
1584        const char **policy_argv;
1585
1586        struct cache_features features;
1587};
1588
1589static void destroy_cache_args(struct cache_args *ca)
1590{
1591        if (ca->metadata_dev)
1592                dm_put_device(ca->ti, ca->metadata_dev);
1593
1594        if (ca->cache_dev)
1595                dm_put_device(ca->ti, ca->cache_dev);
1596
1597        if (ca->origin_dev)
1598                dm_put_device(ca->ti, ca->origin_dev);
1599
1600        kfree(ca);
1601}
1602
1603static bool at_least_one_arg(struct dm_arg_set *as, char **error)
1604{
1605        if (!as->argc) {
1606                *error = "Insufficient args";
1607                return false;
1608        }
1609
1610        return true;
1611}
1612
1613static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
1614                              char **error)
1615{
1616        int r;
1617        sector_t metadata_dev_size;
1618        char b[BDEVNAME_SIZE];
1619
1620        if (!at_least_one_arg(as, error))
1621                return -EINVAL;
1622
1623        r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1624                          &ca->metadata_dev);
1625        if (r) {
1626                *error = "Error opening metadata device";
1627                return r;
1628        }
1629
1630        metadata_dev_size = get_dev_size(ca->metadata_dev);
1631        if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
1632                DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
1633                       bdevname(ca->metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS);
1634
1635        return 0;
1636}
1637
1638static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
1639                           char **error)
1640{
1641        int r;
1642
1643        if (!at_least_one_arg(as, error))
1644                return -EINVAL;
1645
1646        r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1647                          &ca->cache_dev);
1648        if (r) {
1649                *error = "Error opening cache device";
1650                return r;
1651        }
1652        ca->cache_sectors = get_dev_size(ca->cache_dev);
1653
1654        return 0;
1655}
1656
1657static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
1658                            char **error)
1659{
1660        int r;
1661
1662        if (!at_least_one_arg(as, error))
1663                return -EINVAL;
1664
1665        r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1666                          &ca->origin_dev);
1667        if (r) {
1668                *error = "Error opening origin device";
1669                return r;
1670        }
1671
1672        ca->origin_sectors = get_dev_size(ca->origin_dev);
1673        if (ca->ti->len > ca->origin_sectors) {
1674                *error = "Device size larger than cached device";
1675                return -EINVAL;
1676        }
1677
1678        return 0;
1679}
1680
1681static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
1682                            char **error)
1683{
1684        unsigned long tmp;
1685
1686        if (!at_least_one_arg(as, error))
1687                return -EINVAL;
1688
1689        if (kstrtoul(dm_shift_arg(as), 10, &tmp) || !tmp ||
1690            tmp < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
1691            tmp & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
1692                *error = "Invalid data block size";
1693                return -EINVAL;
1694        }
1695
1696        if (tmp > ca->cache_sectors) {
1697                *error = "Data block size is larger than the cache device";
1698                return -EINVAL;
1699        }
1700
1701        ca->block_size = tmp;
1702
1703        return 0;
1704}
1705
1706static void init_features(struct cache_features *cf)
1707{
1708        cf->mode = CM_WRITE;
1709        cf->write_through = false;
1710}
1711
1712static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
1713                          char **error)
1714{
1715        static struct dm_arg _args[] = {
1716                {0, 1, "Invalid number of cache feature arguments"},
1717        };
1718
1719        int r;
1720        unsigned argc;
1721        const char *arg;
1722        struct cache_features *cf = &ca->features;
1723
1724        init_features(cf);
1725
1726        r = dm_read_arg_group(_args, as, &argc, error);
1727        if (r)
1728                return -EINVAL;
1729
1730        while (argc--) {
1731                arg = dm_shift_arg(as);
1732
1733                if (!strcasecmp(arg, "writeback"))
1734                        cf->write_through = false;
1735
1736                else if (!strcasecmp(arg, "writethrough"))
1737                        cf->write_through = true;
1738
1739                else {
1740                        *error = "Unrecognised cache feature requested";
1741                        return -EINVAL;
1742                }
1743        }
1744
1745        return 0;
1746}
1747
1748static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
1749                        char **error)
1750{
1751        static struct dm_arg _args[] = {
1752                {0, 1024, "Invalid number of policy arguments"},
1753        };
1754
1755        int r;
1756
1757        if (!at_least_one_arg(as, error))
1758                return -EINVAL;
1759
1760        ca->policy_name = dm_shift_arg(as);
1761
1762        r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
1763        if (r)
1764                return -EINVAL;
1765
1766        ca->policy_argv = (const char **)as->argv;
1767        dm_consume_args(as, ca->policy_argc);
1768
1769        return 0;
1770}
1771
1772static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
1773                            char **error)
1774{
1775        int r;
1776        struct dm_arg_set as;
1777
1778        as.argc = argc;
1779        as.argv = argv;
1780
1781        r = parse_metadata_dev(ca, &as, error);
1782        if (r)
1783                return r;
1784
1785        r = parse_cache_dev(ca, &as, error);
1786        if (r)
1787                return r;
1788
1789        r = parse_origin_dev(ca, &as, error);
1790        if (r)
1791                return r;
1792
1793        r = parse_block_size(ca, &as, error);
1794        if (r)
1795                return r;
1796
1797        r = parse_features(ca, &as, error);
1798        if (r)
1799                return r;
1800
1801        r = parse_policy(ca, &as, error);
1802        if (r)
1803                return r;
1804
1805        return 0;
1806}
1807
1808/*----------------------------------------------------------------*/
1809
1810static struct kmem_cache *migration_cache;
1811
1812static int set_config_values(struct dm_cache_policy *p, int argc, const char **argv)
1813{
1814        int r = 0;
1815
1816        if (argc & 1) {
1817                DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
1818                return -EINVAL;
1819        }
1820
1821        while (argc) {
1822                r = policy_set_config_value(p, argv[0], argv[1]);
1823                if (r) {
1824                        DMWARN("policy_set_config_value failed: key = '%s', value = '%s'",
1825                               argv[0], argv[1]);
1826                        return r;
1827                }
1828
1829                argc -= 2;
1830                argv += 2;
1831        }
1832
1833        return r;
1834}
1835
1836static int create_cache_policy(struct cache *cache, struct cache_args *ca,
1837                               char **error)
1838{
1839        int r;
1840
1841        cache->policy = dm_cache_policy_create(ca->policy_name,
1842                                               cache->cache_size,
1843                                               cache->origin_sectors,
1844                                               cache->sectors_per_block);
1845        if (!cache->policy) {
1846                *error = "Error creating cache's policy";
1847                return -ENOMEM;
1848        }
1849
1850        r = set_config_values(cache->policy, ca->policy_argc, ca->policy_argv);
1851        if (r) {
1852                *error = "Error setting cache policy's config values";
1853                dm_cache_policy_destroy(cache->policy);
1854                cache->policy = NULL;
1855        }
1856
1857        return r;
1858}
1859
1860/*
1861 * We want the discard block size to be a power of two, at least the size
1862 * of the cache block size, and have no more than 2^14 discard blocks
1863 * across the origin.
1864 */
1865#define MAX_DISCARD_BLOCKS (1 << 14)
1866
1867static bool too_many_discard_blocks(sector_t discard_block_size,
1868                                    sector_t origin_size)
1869{
1870        (void) sector_div(origin_size, discard_block_size);
1871
1872        return origin_size > MAX_DISCARD_BLOCKS;
1873}
1874
1875static sector_t calculate_discard_block_size(sector_t cache_block_size,
1876                                             sector_t origin_size)
1877{
1878        sector_t discard_block_size;
1879
1880        discard_block_size = roundup_pow_of_two(cache_block_size);
1881
1882        if (origin_size)
1883                while (too_many_discard_blocks(discard_block_size, origin_size))
1884                        discard_block_size *= 2;
1885
1886        return discard_block_size;
1887}
1888
1889#define DEFAULT_MIGRATION_THRESHOLD (2048 * 100)
1890
1891static int cache_create(struct cache_args *ca, struct cache **result)
1892{
1893        int r = 0;
1894        char **error = &ca->ti->error;
1895        struct cache *cache;
1896        struct dm_target *ti = ca->ti;
1897        dm_block_t origin_blocks;
1898        struct dm_cache_metadata *cmd;
1899        bool may_format = ca->features.mode == CM_WRITE;
1900
1901        cache = kzalloc(sizeof(*cache), GFP_KERNEL);
1902        if (!cache)
1903                return -ENOMEM;
1904
1905        cache->ti = ca->ti;
1906        ti->private = cache;
1907        ti->num_flush_bios = 2;
1908        ti->flush_supported = true;
1909
1910        ti->num_discard_bios = 1;
1911        ti->discards_supported = true;
1912        ti->discard_zeroes_data_unsupported = true;
1913
1914        memcpy(&cache->features, &ca->features, sizeof(cache->features));
1915        ti->per_bio_data_size = get_per_bio_data_size(cache);
1916
1917        cache->callbacks.congested_fn = cache_is_congested;
1918        dm_table_add_target_callbacks(ti->table, &cache->callbacks);
1919
1920        cache->metadata_dev = ca->metadata_dev;
1921        cache->origin_dev = ca->origin_dev;
1922        cache->cache_dev = ca->cache_dev;
1923
1924        ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
1925
1926        /* FIXME: factor out this whole section */
1927        origin_blocks = cache->origin_sectors = ca->origin_sectors;
1928        origin_blocks = block_div(origin_blocks, ca->block_size);
1929        cache->origin_blocks = to_oblock(origin_blocks);
1930
1931        cache->sectors_per_block = ca->block_size;
1932        if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
1933                r = -EINVAL;
1934                goto bad;
1935        }
1936
1937        if (ca->block_size & (ca->block_size - 1)) {
1938                dm_block_t cache_size = ca->cache_sectors;
1939
1940                cache->sectors_per_block_shift = -1;
1941                cache_size = block_div(cache_size, ca->block_size);
1942                cache->cache_size = to_cblock(cache_size);
1943        } else {
1944                cache->sectors_per_block_shift = __ffs(ca->block_size);
1945                cache->cache_size = to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift);
1946        }
1947
1948        r = create_cache_policy(cache, ca, error);
1949        if (r)
1950                goto bad;
1951        cache->policy_nr_args = ca->policy_argc;
1952
1953        cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
1954                                     ca->block_size, may_format,
1955                                     dm_cache_policy_get_hint_size(cache->policy));
1956        if (IS_ERR(cmd)) {
1957                *error = "Error creating metadata object";
1958                r = PTR_ERR(cmd);
1959                goto bad;
1960        }
1961        cache->cmd = cmd;
1962
1963        spin_lock_init(&cache->lock);
1964        bio_list_init(&cache->deferred_bios);
1965        bio_list_init(&cache->deferred_flush_bios);
1966        bio_list_init(&cache->deferred_writethrough_bios);
1967        INIT_LIST_HEAD(&cache->quiesced_migrations);
1968        INIT_LIST_HEAD(&cache->completed_migrations);
1969        INIT_LIST_HEAD(&cache->need_commit_migrations);
1970        cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
1971        atomic_set(&cache->nr_migrations, 0);
1972        init_waitqueue_head(&cache->migration_wait);
1973
1974        r = -ENOMEM;
1975        cache->nr_dirty = 0;
1976        cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
1977        if (!cache->dirty_bitset) {
1978                *error = "could not allocate dirty bitset";
1979                goto bad;
1980        }
1981        clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
1982
1983        cache->discard_block_size =
1984                calculate_discard_block_size(cache->sectors_per_block,
1985                                             cache->origin_sectors);
1986        cache->discard_nr_blocks = oblock_to_dblock(cache, cache->origin_blocks);
1987        cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
1988        if (!cache->discard_bitset) {
1989                *error = "could not allocate discard bitset";
1990                goto bad;
1991        }
1992        clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
1993
1994        cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
1995        if (IS_ERR(cache->copier)) {
1996                *error = "could not create kcopyd client";
1997                r = PTR_ERR(cache->copier);
1998                goto bad;
1999        }
2000
2001        cache->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
2002        if (!cache->wq) {
2003                *error = "could not create workqueue for metadata object";
2004                goto bad;
2005        }
2006        INIT_WORK(&cache->worker, do_worker);
2007        INIT_DELAYED_WORK(&cache->waker, do_waker);
2008        cache->last_commit_jiffies = jiffies;
2009
2010        cache->prison = dm_bio_prison_create(PRISON_CELLS);
2011        if (!cache->prison) {
2012                *error = "could not create bio prison";
2013                goto bad;
2014        }
2015
2016        cache->all_io_ds = dm_deferred_set_create();
2017        if (!cache->all_io_ds) {
2018                *error = "could not create all_io deferred set";
2019                goto bad;
2020        }
2021
2022        cache->migration_pool = mempool_create_slab_pool(MIGRATION_POOL_SIZE,
2023                                                         migration_cache);
2024        if (!cache->migration_pool) {
2025                *error = "Error creating cache's migration mempool";
2026                goto bad;
2027        }
2028
2029        cache->next_migration = NULL;
2030
2031        cache->need_tick_bio = true;
2032        cache->sized = false;
2033        cache->quiescing = false;
2034        cache->commit_requested = false;
2035        cache->loaded_mappings = false;
2036        cache->loaded_discards = false;
2037
2038        load_stats(cache);
2039
2040        atomic_set(&cache->stats.demotion, 0);
2041        atomic_set(&cache->stats.promotion, 0);
2042        atomic_set(&cache->stats.copies_avoided, 0);
2043        atomic_set(&cache->stats.cache_cell_clash, 0);
2044        atomic_set(&cache->stats.commit_count, 0);
2045        atomic_set(&cache->stats.discard_count, 0);
2046
2047        *result = cache;
2048        return 0;
2049
2050bad:
2051        destroy(cache);
2052        return r;
2053}
2054
2055static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
2056{
2057        unsigned i;
2058        const char **copy;
2059
2060        copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
2061        if (!copy)
2062                return -ENOMEM;
2063        for (i = 0; i < argc; i++) {
2064                copy[i] = kstrdup(argv[i], GFP_KERNEL);
2065                if (!copy[i]) {
2066                        while (i--)
2067                                kfree(copy[i]);
2068                        kfree(copy);
2069                        return -ENOMEM;
2070                }
2071        }
2072
2073        cache->nr_ctr_args = argc;
2074        cache->ctr_args = copy;
2075
2076        return 0;
2077}
2078
2079static int cache_ctr(struct dm_target *ti, unsigned argc, char **argv)
2080{
2081        int r = -EINVAL;
2082        struct cache_args *ca;
2083        struct cache *cache = NULL;
2084
2085        ca = kzalloc(sizeof(*ca), GFP_KERNEL);
2086        if (!ca) {
2087                ti->error = "Error allocating memory for cache";
2088                return -ENOMEM;
2089        }
2090        ca->ti = ti;
2091
2092        r = parse_cache_args(ca, argc, argv, &ti->error);
2093        if (r)
2094                goto out;
2095
2096        r = cache_create(ca, &cache);
2097        if (r)
2098                goto out;
2099
2100        r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
2101        if (r) {
2102                destroy(cache);
2103                goto out;
2104        }
2105
2106        ti->private = cache;
2107
2108out:
2109        destroy_cache_args(ca);
2110        return r;
2111}
2112
2113static int cache_map(struct dm_target *ti, struct bio *bio)
2114{
2115        struct cache *cache = ti->private;
2116
2117        int r;
2118        dm_oblock_t block = get_bio_block(cache, bio);
2119        size_t pb_data_size = get_per_bio_data_size(cache);
2120        bool can_migrate = false;
2121        bool discarded_block;
2122        struct dm_bio_prison_cell *cell;
2123        struct policy_result lookup_result;
2124        struct per_bio_data *pb;
2125
2126        if (from_oblock(block) > from_oblock(cache->origin_blocks)) {
2127                /*
2128                 * This can only occur if the io goes to a partial block at
2129                 * the end of the origin device.  We don't cache these.
2130                 * Just remap to the origin and carry on.
2131                 */
2132                remap_to_origin_clear_discard(cache, bio, block);
2133                return DM_MAPIO_REMAPPED;
2134        }
2135
2136        pb = init_per_bio_data(bio, pb_data_size);
2137
2138        if (bio->bi_rw & (REQ_FLUSH | REQ_FUA | REQ_DISCARD)) {
2139                defer_bio(cache, bio);
2140                return DM_MAPIO_SUBMITTED;
2141        }
2142
2143        /*
2144         * Check to see if that block is currently migrating.
2145         */
2146        cell = alloc_prison_cell(cache);
2147        if (!cell) {
2148                defer_bio(cache, bio);
2149                return DM_MAPIO_SUBMITTED;
2150        }
2151
2152        r = bio_detain(cache, block, bio, cell,
2153                       (cell_free_fn) free_prison_cell,
2154                       cache, &cell);
2155        if (r) {
2156                if (r < 0)
2157                        defer_bio(cache, bio);
2158
2159                return DM_MAPIO_SUBMITTED;
2160        }
2161
2162        discarded_block = is_discarded_oblock(cache, block);
2163
2164        r = policy_map(cache->policy, block, false, can_migrate, discarded_block,
2165                       bio, &lookup_result);
2166        if (r == -EWOULDBLOCK) {
2167                cell_defer(cache, cell, true);
2168                return DM_MAPIO_SUBMITTED;
2169
2170        } else if (r) {
2171                DMERR_LIMIT("Unexpected return from cache replacement policy: %d", r);
2172                bio_io_error(bio);
2173                return DM_MAPIO_SUBMITTED;
2174        }
2175
2176        switch (lookup_result.op) {
2177        case POLICY_HIT:
2178                inc_hit_counter(cache, bio);
2179                pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
2180
2181                if (is_writethrough_io(cache, bio, lookup_result.cblock))
2182                        remap_to_origin_then_cache(cache, bio, block, lookup_result.cblock);
2183                else
2184                        remap_to_cache_dirty(cache, bio, block, lookup_result.cblock);
2185
2186                cell_defer(cache, cell, false);
2187                break;
2188
2189        case POLICY_MISS:
2190                inc_miss_counter(cache, bio);
2191                pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
2192
2193                if (pb->req_nr != 0) {
2194                        /*
2195                         * This is a duplicate writethrough io that is no
2196                         * longer needed because the block has been demoted.
2197                         */
2198                        bio_endio(bio, 0);
2199                        cell_defer(cache, cell, false);
2200                        return DM_MAPIO_SUBMITTED;
2201                } else {
2202                        remap_to_origin_clear_discard(cache, bio, block);
2203                        cell_defer(cache, cell, false);
2204                }
2205                break;
2206
2207        default:
2208                DMERR_LIMIT("%s: erroring bio: unknown policy op: %u", __func__,
2209                            (unsigned) lookup_result.op);
2210                bio_io_error(bio);
2211                return DM_MAPIO_SUBMITTED;
2212        }
2213
2214        return DM_MAPIO_REMAPPED;
2215}
2216
2217static int cache_end_io(struct dm_target *ti, struct bio *bio, int error)
2218{
2219        struct cache *cache = ti->private;
2220        unsigned long flags;
2221        size_t pb_data_size = get_per_bio_data_size(cache);
2222        struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
2223
2224        if (pb->tick) {
2225                policy_tick(cache->policy);
2226
2227                spin_lock_irqsave(&cache->lock, flags);
2228                cache->need_tick_bio = true;
2229                spin_unlock_irqrestore(&cache->lock, flags);
2230        }
2231
2232        check_for_quiesced_migrations(cache, pb);
2233
2234        return 0;
2235}
2236
2237static int write_dirty_bitset(struct cache *cache)
2238{
2239        unsigned i, r;
2240
2241        for (i = 0; i < from_cblock(cache->cache_size); i++) {
2242                r = dm_cache_set_dirty(cache->cmd, to_cblock(i),
2243                                       is_dirty(cache, to_cblock(i)));
2244                if (r)
2245                        return r;
2246        }
2247
2248        return 0;
2249}
2250
2251static int write_discard_bitset(struct cache *cache)
2252{
2253        unsigned i, r;
2254
2255        r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
2256                                           cache->discard_nr_blocks);
2257        if (r) {
2258                DMERR("could not resize on-disk discard bitset");
2259                return r;
2260        }
2261
2262        for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
2263                r = dm_cache_set_discard(cache->cmd, to_dblock(i),
2264                                         is_discarded(cache, to_dblock(i)));
2265                if (r)
2266                        return r;
2267        }
2268
2269        return 0;
2270}
2271
2272static int save_hint(void *context, dm_cblock_t cblock, dm_oblock_t oblock,
2273                     uint32_t hint)
2274{
2275        struct cache *cache = context;
2276        return dm_cache_save_hint(cache->cmd, cblock, hint);
2277}
2278
2279static int write_hints(struct cache *cache)
2280{
2281        int r;
2282
2283        r = dm_cache_begin_hints(cache->cmd, cache->policy);
2284        if (r) {
2285                DMERR("dm_cache_begin_hints failed");
2286                return r;
2287        }
2288
2289        r = policy_walk_mappings(cache->policy, save_hint, cache);
2290        if (r)
2291                DMERR("policy_walk_mappings failed");
2292
2293        return r;
2294}
2295
2296/*
2297 * returns true on success
2298 */
2299static bool sync_metadata(struct cache *cache)
2300{
2301        int r1, r2, r3, r4;
2302
2303        r1 = write_dirty_bitset(cache);
2304        if (r1)
2305                DMERR("could not write dirty bitset");
2306
2307        r2 = write_discard_bitset(cache);
2308        if (r2)
2309                DMERR("could not write discard bitset");
2310
2311        save_stats(cache);
2312
2313        r3 = write_hints(cache);
2314        if (r3)
2315                DMERR("could not write hints");
2316
2317        /*
2318         * If writing the above metadata failed, we still commit, but don't
2319         * set the clean shutdown flag.  This will effectively force every
2320         * dirty bit to be set on reload.
2321         */
2322        r4 = dm_cache_commit(cache->cmd, !r1 && !r2 && !r3);
2323        if (r4)
2324                DMERR("could not write cache metadata.  Data loss may occur.");
2325
2326        return !r1 && !r2 && !r3 && !r4;
2327}
2328
2329static void cache_postsuspend(struct dm_target *ti)
2330{
2331        struct cache *cache = ti->private;
2332
2333        start_quiescing(cache);
2334        wait_for_migrations(cache);
2335        stop_worker(cache);
2336        requeue_deferred_io(cache);
2337        stop_quiescing(cache);
2338
2339        (void) sync_metadata(cache);
2340}
2341
2342static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2343                        bool dirty, uint32_t hint, bool hint_valid)
2344{
2345        int r;
2346        struct cache *cache = context;
2347
2348        r = policy_load_mapping(cache->policy, oblock, cblock, hint, hint_valid);
2349        if (r)
2350                return r;
2351
2352        if (dirty)
2353                set_dirty(cache, oblock, cblock);
2354        else
2355                clear_dirty(cache, oblock, cblock);
2356
2357        return 0;
2358}
2359
2360static int load_discard(void *context, sector_t discard_block_size,
2361                        dm_dblock_t dblock, bool discard)
2362{
2363        struct cache *cache = context;
2364
2365        /* FIXME: handle mis-matched block size */
2366
2367        if (discard)
2368                set_discard(cache, dblock);
2369        else
2370                clear_discard(cache, dblock);
2371
2372        return 0;
2373}
2374
2375static int cache_preresume(struct dm_target *ti)
2376{
2377        int r = 0;
2378        struct cache *cache = ti->private;
2379        sector_t actual_cache_size = get_dev_size(cache->cache_dev);
2380        (void) sector_div(actual_cache_size, cache->sectors_per_block);
2381
2382        /*
2383         * Check to see if the cache has resized.
2384         */
2385        if (from_cblock(cache->cache_size) != actual_cache_size || !cache->sized) {
2386                cache->cache_size = to_cblock(actual_cache_size);
2387
2388                r = dm_cache_resize(cache->cmd, cache->cache_size);
2389                if (r) {
2390                        DMERR("could not resize cache metadata");
2391                        return r;
2392                }
2393
2394                cache->sized = true;
2395        }
2396
2397        if (!cache->loaded_mappings) {
2398                r = dm_cache_load_mappings(cache->cmd, cache->policy,
2399                                           load_mapping, cache);
2400                if (r) {
2401                        DMERR("could not load cache mappings");
2402                        return r;
2403                }
2404
2405                cache->loaded_mappings = true;
2406        }
2407
2408        if (!cache->loaded_discards) {
2409                r = dm_cache_load_discards(cache->cmd, load_discard, cache);
2410                if (r) {
2411                        DMERR("could not load origin discards");
2412                        return r;
2413                }
2414
2415                cache->loaded_discards = true;
2416        }
2417
2418        return r;
2419}
2420
2421static void cache_resume(struct dm_target *ti)
2422{
2423        struct cache *cache = ti->private;
2424
2425        cache->need_tick_bio = true;
2426        do_waker(&cache->waker.work);
2427}
2428
2429/*
2430 * Status format:
2431 *
2432 * <#used metadata blocks>/<#total metadata blocks>
2433 * <#read hits> <#read misses> <#write hits> <#write misses>
2434 * <#demotions> <#promotions> <#blocks in cache> <#dirty>
2435 * <#features> <features>*
2436 * <#core args> <core args>
2437 * <#policy args> <policy args>*
2438 */
2439static void cache_status(struct dm_target *ti, status_type_t type,
2440                         unsigned status_flags, char *result, unsigned maxlen)
2441{
2442        int r = 0;
2443        unsigned i;
2444        ssize_t sz = 0;
2445        dm_block_t nr_free_blocks_metadata = 0;
2446        dm_block_t nr_blocks_metadata = 0;
2447        char buf[BDEVNAME_SIZE];
2448        struct cache *cache = ti->private;
2449        dm_cblock_t residency;
2450
2451        switch (type) {
2452        case STATUSTYPE_INFO:
2453                /* Commit to ensure statistics aren't out-of-date */
2454                if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti)) {
2455                        r = dm_cache_commit(cache->cmd, false);
2456                        if (r)
2457                                DMERR("could not commit metadata for accurate status");
2458                }
2459
2460                r = dm_cache_get_free_metadata_block_count(cache->cmd,
2461                                                           &nr_free_blocks_metadata);
2462                if (r) {
2463                        DMERR("could not get metadata free block count");
2464                        goto err;
2465                }
2466
2467                r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
2468                if (r) {
2469                        DMERR("could not get metadata device size");
2470                        goto err;
2471                }
2472
2473                residency = policy_residency(cache->policy);
2474
2475                DMEMIT("%llu/%llu %u %u %u %u %u %u %llu %u ",
2476                       (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
2477                       (unsigned long long)nr_blocks_metadata,
2478                       (unsigned) atomic_read(&cache->stats.read_hit),
2479                       (unsigned) atomic_read(&cache->stats.read_miss),
2480                       (unsigned) atomic_read(&cache->stats.write_hit),
2481                       (unsigned) atomic_read(&cache->stats.write_miss),
2482                       (unsigned) atomic_read(&cache->stats.demotion),
2483                       (unsigned) atomic_read(&cache->stats.promotion),
2484                       (unsigned long long) from_cblock(residency),
2485                       cache->nr_dirty);
2486
2487                if (cache->features.write_through)
2488                        DMEMIT("1 writethrough ");
2489                else
2490                        DMEMIT("0 ");
2491
2492                DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
2493                if (sz < maxlen) {
2494                        r = policy_emit_config_values(cache->policy, result + sz, maxlen - sz);
2495                        if (r)
2496                                DMERR("policy_emit_config_values returned %d", r);
2497                }
2498
2499                break;
2500
2501        case STATUSTYPE_TABLE:
2502                format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
2503                DMEMIT("%s ", buf);
2504                format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
2505                DMEMIT("%s ", buf);
2506                format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
2507                DMEMIT("%s", buf);
2508
2509                for (i = 0; i < cache->nr_ctr_args - 1; i++)
2510                        DMEMIT(" %s", cache->ctr_args[i]);
2511                if (cache->nr_ctr_args)
2512                        DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
2513        }
2514
2515        return;
2516
2517err:
2518        DMEMIT("Error");
2519}
2520
2521#define NOT_CORE_OPTION 1
2522
2523static int process_config_option(struct cache *cache, char **argv)
2524{
2525        unsigned long tmp;
2526
2527        if (!strcasecmp(argv[0], "migration_threshold")) {
2528                if (kstrtoul(argv[1], 10, &tmp))
2529                        return -EINVAL;
2530
2531                cache->migration_threshold = tmp;
2532                return 0;
2533        }
2534
2535        return NOT_CORE_OPTION;
2536}
2537
2538/*
2539 * Supports <key> <value>.
2540 *
2541 * The key migration_threshold is supported by the cache target core.
2542 */
2543static int cache_message(struct dm_target *ti, unsigned argc, char **argv)
2544{
2545        int r;
2546        struct cache *cache = ti->private;
2547
2548        if (argc != 2)
2549                return -EINVAL;
2550
2551        r = process_config_option(cache, argv);
2552        if (r == NOT_CORE_OPTION)
2553                return policy_set_config_value(cache->policy, argv[0], argv[1]);
2554
2555        return r;
2556}
2557
2558static int cache_iterate_devices(struct dm_target *ti,
2559                                 iterate_devices_callout_fn fn, void *data)
2560{
2561        int r = 0;
2562        struct cache *cache = ti->private;
2563
2564        r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
2565        if (!r)
2566                r = fn(ti, cache->origin_dev, 0, ti->len, data);
2567
2568        return r;
2569}
2570
2571/*
2572 * We assume I/O is going to the origin (which is the volume
2573 * more likely to have restrictions e.g. by being striped).
2574 * (Looking up the exact location of the data would be expensive
2575 * and could always be out of date by the time the bio is submitted.)
2576 */
2577static int cache_bvec_merge(struct dm_target *ti,
2578                            struct bvec_merge_data *bvm,
2579                            struct bio_vec *biovec, int max_size)
2580{
2581        struct cache *cache = ti->private;
2582        struct request_queue *q = bdev_get_queue(cache->origin_dev->bdev);
2583
2584        if (!q->merge_bvec_fn)
2585                return max_size;
2586
2587        bvm->bi_bdev = cache->origin_dev->bdev;
2588        return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2589}
2590
2591static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
2592{
2593        /*
2594         * FIXME: these limits may be incompatible with the cache device
2595         */
2596        limits->max_discard_sectors = cache->discard_block_size * 1024;
2597        limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
2598}
2599
2600static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
2601{
2602        struct cache *cache = ti->private;
2603
2604        blk_limits_io_min(limits, 0);
2605        blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT);
2606        set_discard_limits(cache, limits);
2607}
2608
2609/*----------------------------------------------------------------*/
2610
2611static struct target_type cache_target = {
2612        .name = "cache",
2613        .version = {1, 1, 0},
2614        .module = THIS_MODULE,
2615        .ctr = cache_ctr,
2616        .dtr = cache_dtr,
2617        .map = cache_map,
2618        .end_io = cache_end_io,
2619        .postsuspend = cache_postsuspend,
2620        .preresume = cache_preresume,
2621        .resume = cache_resume,
2622        .status = cache_status,
2623        .message = cache_message,
2624        .iterate_devices = cache_iterate_devices,
2625        .merge = cache_bvec_merge,
2626        .io_hints = cache_io_hints,
2627};
2628
2629static int __init dm_cache_init(void)
2630{
2631        int r;
2632
2633        r = dm_register_target(&cache_target);
2634        if (r) {
2635                DMERR("cache target registration failed: %d", r);
2636                return r;
2637        }
2638
2639        migration_cache = KMEM_CACHE(dm_cache_migration, 0);
2640        if (!migration_cache) {
2641                dm_unregister_target(&cache_target);
2642                return -ENOMEM;
2643        }
2644
2645        return 0;
2646}
2647
2648static void __exit dm_cache_exit(void)
2649{
2650        dm_unregister_target(&cache_target);
2651        kmem_cache_destroy(migration_cache);
2652}
2653
2654module_init(dm_cache_init);
2655module_exit(dm_cache_exit);
2656
2657MODULE_DESCRIPTION(DM_NAME " cache target");
2658MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
2659MODULE_LICENSE("GPL");
2660
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.