linux/drivers/md/raid5.c
<<
>>
Prefs
   1/*
   2 * raid5.c : Multiple Devices driver for Linux
   3 *         Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
   4 *         Copyright (C) 1999, 2000 Ingo Molnar
   5 *         Copyright (C) 2002, 2003 H. Peter Anvin
   6 *
   7 * RAID-4/5/6 management functions.
   8 * Thanks to Penguin Computing for making the RAID-6 development possible
   9 * by donating a test server!
  10 *
  11 * This program is free software; you can redistribute it and/or modify
  12 * it under the terms of the GNU General Public License as published by
  13 * the Free Software Foundation; either version 2, or (at your option)
  14 * any later version.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * (for example /usr/src/linux/COPYING); if not, write to the Free
  18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20
  21/*
  22 * BITMAP UNPLUGGING:
  23 *
  24 * The sequencing for updating the bitmap reliably is a little
  25 * subtle (and I got it wrong the first time) so it deserves some
  26 * explanation.
  27 *
  28 * We group bitmap updates into batches.  Each batch has a number.
  29 * We may write out several batches at once, but that isn't very important.
  30 * conf->seq_write is the number of the last batch successfully written.
  31 * conf->seq_flush is the number of the last batch that was closed to
  32 *    new additions.
  33 * When we discover that we will need to write to any block in a stripe
  34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
  35 * the number of the batch it will be in. This is seq_flush+1.
  36 * When we are ready to do a write, if that batch hasn't been written yet,
  37 *   we plug the array and queue the stripe for later.
  38 * When an unplug happens, we increment bm_flush, thus closing the current
  39 *   batch.
  40 * When we notice that bm_flush > bm_write, we write out all pending updates
  41 * to the bitmap, and advance bm_write to where bm_flush was.
  42 * This may occasionally write a bit out twice, but is sure never to
  43 * miss any bits.
  44 */
  45
  46#include <linux/blkdev.h>
  47#include <linux/kthread.h>
  48#include <linux/raid/pq.h>
  49#include <linux/async_tx.h>
  50#include <linux/module.h>
  51#include <linux/async.h>
  52#include <linux/seq_file.h>
  53#include <linux/cpu.h>
  54#include <linux/slab.h>
  55#include <linux/ratelimit.h>
  56#include <linux/nodemask.h>
  57#include <trace/events/block.h>
  58
  59#include "md.h"
  60#include "raid5.h"
  61#include "raid0.h"
  62#include "bitmap.h"
  63
  64#define cpu_to_group(cpu) cpu_to_node(cpu)
  65#define ANY_GROUP NUMA_NO_NODE
  66
  67static struct workqueue_struct *raid5_wq;
  68/*
  69 * Stripe cache
  70 */
  71
  72#define NR_STRIPES              256
  73#define STRIPE_SIZE             PAGE_SIZE
  74#define STRIPE_SHIFT            (PAGE_SHIFT - 9)
  75#define STRIPE_SECTORS          (STRIPE_SIZE>>9)
  76#define IO_THRESHOLD            1
  77#define BYPASS_THRESHOLD        1
  78#define NR_HASH                 (PAGE_SIZE / sizeof(struct hlist_head))
  79#define HASH_MASK               (NR_HASH - 1)
  80#define MAX_STRIPE_BATCH        8
  81
  82static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
  83{
  84        int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
  85        return &conf->stripe_hashtbl[hash];
  86}
  87
  88/* bio's attached to a stripe+device for I/O are linked together in bi_sector
  89 * order without overlap.  There may be several bio's per stripe+device, and
  90 * a bio could span several devices.
  91 * When walking this list for a particular stripe+device, we must never proceed
  92 * beyond a bio that extends past this device, as the next bio might no longer
  93 * be valid.
  94 * This function is used to determine the 'next' bio in the list, given the sector
  95 * of the current stripe+device
  96 */
  97static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
  98{
  99        int sectors = bio_sectors(bio);
 100        if (bio->bi_sector + sectors < sector + STRIPE_SECTORS)
 101                return bio->bi_next;
 102        else
 103                return NULL;
 104}
 105
 106/*
 107 * We maintain a biased count of active stripes in the bottom 16 bits of
 108 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
 109 */
 110static inline int raid5_bi_processed_stripes(struct bio *bio)
 111{
 112        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 113        return (atomic_read(segments) >> 16) & 0xffff;
 114}
 115
 116static inline int raid5_dec_bi_active_stripes(struct bio *bio)
 117{
 118        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 119        return atomic_sub_return(1, segments) & 0xffff;
 120}
 121
 122static inline void raid5_inc_bi_active_stripes(struct bio *bio)
 123{
 124        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 125        atomic_inc(segments);
 126}
 127
 128static inline void raid5_set_bi_processed_stripes(struct bio *bio,
 129        unsigned int cnt)
 130{
 131        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 132        int old, new;
 133
 134        do {
 135                old = atomic_read(segments);
 136                new = (old & 0xffff) | (cnt << 16);
 137        } while (atomic_cmpxchg(segments, old, new) != old);
 138}
 139
 140static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
 141{
 142        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
 143        atomic_set(segments, cnt);
 144}
 145
 146/* Find first data disk in a raid6 stripe */
 147static inline int raid6_d0(struct stripe_head *sh)
 148{
 149        if (sh->ddf_layout)
 150                /* ddf always start from first device */
 151                return 0;
 152        /* md starts just after Q block */
 153        if (sh->qd_idx == sh->disks - 1)
 154                return 0;
 155        else
 156                return sh->qd_idx + 1;
 157}
 158static inline int raid6_next_disk(int disk, int raid_disks)
 159{
 160        disk++;
 161        return (disk < raid_disks) ? disk : 0;
 162}
 163
 164/* When walking through the disks in a raid5, starting at raid6_d0,
 165 * We need to map each disk to a 'slot', where the data disks are slot
 166 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
 167 * is raid_disks-1.  This help does that mapping.
 168 */
 169static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
 170                             int *count, int syndrome_disks)
 171{
 172        int slot = *count;
 173
 174        if (sh->ddf_layout)
 175                (*count)++;
 176        if (idx == sh->pd_idx)
 177                return syndrome_disks;
 178        if (idx == sh->qd_idx)
 179                return syndrome_disks + 1;
 180        if (!sh->ddf_layout)
 181                (*count)++;
 182        return slot;
 183}
 184
 185static void return_io(struct bio *return_bi)
 186{
 187        struct bio *bi = return_bi;
 188        while (bi) {
 189
 190                return_bi = bi->bi_next;
 191                bi->bi_next = NULL;
 192                bi->bi_size = 0;
 193                trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
 194                                         bi, 0);
 195                bio_endio(bi, 0);
 196                bi = return_bi;
 197        }
 198}
 199
 200static void print_raid5_conf (struct r5conf *conf);
 201
 202static int stripe_operations_active(struct stripe_head *sh)
 203{
 204        return sh->check_state || sh->reconstruct_state ||
 205               test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
 206               test_bit(STRIPE_COMPUTE_RUN, &sh->state);
 207}
 208
 209static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
 210{
 211        struct r5conf *conf = sh->raid_conf;
 212        struct r5worker_group *group;
 213        int thread_cnt;
 214        int i, cpu = sh->cpu;
 215
 216        if (!cpu_online(cpu)) {
 217                cpu = cpumask_any(cpu_online_mask);
 218                sh->cpu = cpu;
 219        }
 220
 221        if (list_empty(&sh->lru)) {
 222                struct r5worker_group *group;
 223                group = conf->worker_groups + cpu_to_group(cpu);
 224                list_add_tail(&sh->lru, &group->handle_list);
 225                group->stripes_cnt++;
 226                sh->group = group;
 227        }
 228
 229        if (conf->worker_cnt_per_group == 0) {
 230                md_wakeup_thread(conf->mddev->thread);
 231                return;
 232        }
 233
 234        group = conf->worker_groups + cpu_to_group(sh->cpu);
 235
 236        group->workers[0].working = true;
 237        /* at least one worker should run to avoid race */
 238        queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
 239
 240        thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
 241        /* wakeup more workers */
 242        for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
 243                if (group->workers[i].working == false) {
 244                        group->workers[i].working = true;
 245                        queue_work_on(sh->cpu, raid5_wq,
 246                                      &group->workers[i].work);
 247                        thread_cnt--;
 248                }
 249        }
 250}
 251
 252static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh)
 253{
 254        BUG_ON(!list_empty(&sh->lru));
 255        BUG_ON(atomic_read(&conf->active_stripes)==0);
 256        if (test_bit(STRIPE_HANDLE, &sh->state)) {
 257                if (test_bit(STRIPE_DELAYED, &sh->state) &&
 258                    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
 259                        list_add_tail(&sh->lru, &conf->delayed_list);
 260                else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
 261                           sh->bm_seq - conf->seq_write > 0)
 262                        list_add_tail(&sh->lru, &conf->bitmap_list);
 263                else {
 264                        clear_bit(STRIPE_DELAYED, &sh->state);
 265                        clear_bit(STRIPE_BIT_DELAY, &sh->state);
 266                        if (conf->worker_cnt_per_group == 0) {
 267                                list_add_tail(&sh->lru, &conf->handle_list);
 268                        } else {
 269                                raid5_wakeup_stripe_thread(sh);
 270                                return;
 271                        }
 272                }
 273                md_wakeup_thread(conf->mddev->thread);
 274        } else {
 275                BUG_ON(stripe_operations_active(sh));
 276                if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
 277                        if (atomic_dec_return(&conf->preread_active_stripes)
 278                            < IO_THRESHOLD)
 279                                md_wakeup_thread(conf->mddev->thread);
 280                atomic_dec(&conf->active_stripes);
 281                if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
 282                        list_add_tail(&sh->lru, &conf->inactive_list);
 283                        wake_up(&conf->wait_for_stripe);
 284                        if (conf->retry_read_aligned)
 285                                md_wakeup_thread(conf->mddev->thread);
 286                }
 287        }
 288}
 289
 290static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
 291{
 292        if (atomic_dec_and_test(&sh->count))
 293                do_release_stripe(conf, sh);
 294}
 295
 296static struct llist_node *llist_reverse_order(struct llist_node *head)
 297{
 298        struct llist_node *new_head = NULL;
 299
 300        while (head) {
 301                struct llist_node *tmp = head;
 302                head = head->next;
 303                tmp->next = new_head;
 304                new_head = tmp;
 305        }
 306
 307        return new_head;
 308}
 309
 310/* should hold conf->device_lock already */
 311static int release_stripe_list(struct r5conf *conf)
 312{
 313        struct stripe_head *sh;
 314        int count = 0;
 315        struct llist_node *head;
 316
 317        head = llist_del_all(&conf->released_stripes);
 318        head = llist_reverse_order(head);
 319        while (head) {
 320                sh = llist_entry(head, struct stripe_head, release_list);
 321                head = llist_next(head);
 322                /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
 323                smp_mb();
 324                clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
 325                /*
 326                 * Don't worry the bit is set here, because if the bit is set
 327                 * again, the count is always > 1. This is true for
 328                 * STRIPE_ON_UNPLUG_LIST bit too.
 329                 */
 330                __release_stripe(conf, sh);
 331                count++;
 332        }
 333
 334        return count;
 335}
 336
 337static void release_stripe(struct stripe_head *sh)
 338{
 339        struct r5conf *conf = sh->raid_conf;
 340        unsigned long flags;
 341        bool wakeup;
 342
 343        if (test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
 344                goto slow_path;
 345        wakeup = llist_add(&sh->release_list, &conf->released_stripes);
 346        if (wakeup)
 347                md_wakeup_thread(conf->mddev->thread);
 348        return;
 349slow_path:
 350        local_irq_save(flags);
 351        /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
 352        if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
 353                do_release_stripe(conf, sh);
 354                spin_unlock(&conf->device_lock);
 355        }
 356        local_irq_restore(flags);
 357}
 358
 359static inline void remove_hash(struct stripe_head *sh)
 360{
 361        pr_debug("remove_hash(), stripe %llu\n",
 362                (unsigned long long)sh->sector);
 363
 364        hlist_del_init(&sh->hash);
 365}
 366
 367static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
 368{
 369        struct hlist_head *hp = stripe_hash(conf, sh->sector);
 370
 371        pr_debug("insert_hash(), stripe %llu\n",
 372                (unsigned long long)sh->sector);
 373
 374        hlist_add_head(&sh->hash, hp);
 375}
 376
 377
 378/* find an idle stripe, make sure it is unhashed, and return it. */
 379static struct stripe_head *get_free_stripe(struct r5conf *conf)
 380{
 381        struct stripe_head *sh = NULL;
 382        struct list_head *first;
 383
 384        if (list_empty(&conf->inactive_list))
 385                goto out;
 386        first = conf->inactive_list.next;
 387        sh = list_entry(first, struct stripe_head, lru);
 388        list_del_init(first);
 389        remove_hash(sh);
 390        atomic_inc(&conf->active_stripes);
 391out:
 392        return sh;
 393}
 394
 395static void shrink_buffers(struct stripe_head *sh)
 396{
 397        struct page *p;
 398        int i;
 399        int num = sh->raid_conf->pool_size;
 400
 401        for (i = 0; i < num ; i++) {
 402                p = sh->dev[i].page;
 403                if (!p)
 404                        continue;
 405                sh->dev[i].page = NULL;
 406                put_page(p);
 407        }
 408}
 409
 410static int grow_buffers(struct stripe_head *sh)
 411{
 412        int i;
 413        int num = sh->raid_conf->pool_size;
 414
 415        for (i = 0; i < num; i++) {
 416                struct page *page;
 417
 418                if (!(page = alloc_page(GFP_KERNEL))) {
 419                        return 1;
 420                }
 421                sh->dev[i].page = page;
 422        }
 423        return 0;
 424}
 425
 426static void raid5_build_block(struct stripe_head *sh, int i, int previous);
 427static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
 428                            struct stripe_head *sh);
 429
 430static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
 431{
 432        struct r5conf *conf = sh->raid_conf;
 433        int i;
 434
 435        BUG_ON(atomic_read(&sh->count) != 0);
 436        BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
 437        BUG_ON(stripe_operations_active(sh));
 438
 439        pr_debug("init_stripe called, stripe %llu\n",
 440                (unsigned long long)sh->sector);
 441
 442        remove_hash(sh);
 443
 444        sh->generation = conf->generation - previous;
 445        sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
 446        sh->sector = sector;
 447        stripe_set_idx(sector, conf, previous, sh);
 448        sh->state = 0;
 449
 450
 451        for (i = sh->disks; i--; ) {
 452                struct r5dev *dev = &sh->dev[i];
 453
 454                if (dev->toread || dev->read || dev->towrite || dev->written ||
 455                    test_bit(R5_LOCKED, &dev->flags)) {
 456                        printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
 457                               (unsigned long long)sh->sector, i, dev->toread,
 458                               dev->read, dev->towrite, dev->written,
 459                               test_bit(R5_LOCKED, &dev->flags));
 460                        WARN_ON(1);
 461                }
 462                dev->flags = 0;
 463                raid5_build_block(sh, i, previous);
 464        }
 465        insert_hash(conf, sh);
 466        sh->cpu = smp_processor_id();
 467}
 468
 469static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
 470                                         short generation)
 471{
 472        struct stripe_head *sh;
 473
 474        pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
 475        hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
 476                if (sh->sector == sector && sh->generation == generation)
 477                        return sh;
 478        pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
 479        return NULL;
 480}
 481
 482/*
 483 * Need to check if array has failed when deciding whether to:
 484 *  - start an array
 485 *  - remove non-faulty devices
 486 *  - add a spare
 487 *  - allow a reshape
 488 * This determination is simple when no reshape is happening.
 489 * However if there is a reshape, we need to carefully check
 490 * both the before and after sections.
 491 * This is because some failed devices may only affect one
 492 * of the two sections, and some non-in_sync devices may
 493 * be insync in the section most affected by failed devices.
 494 */
 495static int calc_degraded(struct r5conf *conf)
 496{
 497        int degraded, degraded2;
 498        int i;
 499
 500        rcu_read_lock();
 501        degraded = 0;
 502        for (i = 0; i < conf->previous_raid_disks; i++) {
 503                struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
 504                if (rdev && test_bit(Faulty, &rdev->flags))
 505                        rdev = rcu_dereference(conf->disks[i].replacement);
 506                if (!rdev || test_bit(Faulty, &rdev->flags))
 507                        degraded++;
 508                else if (test_bit(In_sync, &rdev->flags))
 509                        ;
 510                else
 511                        /* not in-sync or faulty.
 512                         * If the reshape increases the number of devices,
 513                         * this is being recovered by the reshape, so
 514                         * this 'previous' section is not in_sync.
 515                         * If the number of devices is being reduced however,
 516                         * the device can only be part of the array if
 517                         * we are reverting a reshape, so this section will
 518                         * be in-sync.
 519                         */
 520                        if (conf->raid_disks >= conf->previous_raid_disks)
 521                                degraded++;
 522        }
 523        rcu_read_unlock();
 524        if (conf->raid_disks == conf->previous_raid_disks)
 525                return degraded;
 526        rcu_read_lock();
 527        degraded2 = 0;
 528        for (i = 0; i < conf->raid_disks; i++) {
 529                struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
 530                if (rdev && test_bit(Faulty, &rdev->flags))
 531                        rdev = rcu_dereference(conf->disks[i].replacement);
 532                if (!rdev || test_bit(Faulty, &rdev->flags))
 533                        degraded2++;
 534                else if (test_bit(In_sync, &rdev->flags))
 535                        ;
 536                else
 537                        /* not in-sync or faulty.
 538                         * If reshape increases the number of devices, this
 539                         * section has already been recovered, else it
 540                         * almost certainly hasn't.
 541                         */
 542                        if (conf->raid_disks <= conf->previous_raid_disks)
 543                                degraded2++;
 544        }
 545        rcu_read_unlock();
 546        if (degraded2 > degraded)
 547                return degraded2;
 548        return degraded;
 549}
 550
 551static int has_failed(struct r5conf *conf)
 552{
 553        int degraded;
 554
 555        if (conf->mddev->reshape_position == MaxSector)
 556                return conf->mddev->degraded > conf->max_degraded;
 557
 558        degraded = calc_degraded(conf);
 559        if (degraded > conf->max_degraded)
 560                return 1;
 561        return 0;
 562}
 563
 564static struct stripe_head *
 565get_active_stripe(struct r5conf *conf, sector_t sector,
 566                  int previous, int noblock, int noquiesce)
 567{
 568        struct stripe_head *sh;
 569
 570        pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
 571
 572        spin_lock_irq(&conf->device_lock);
 573
 574        do {
 575                wait_event_lock_irq(conf->wait_for_stripe,
 576                                    conf->quiesce == 0 || noquiesce,
 577                                    conf->device_lock);
 578                sh = __find_stripe(conf, sector, conf->generation - previous);
 579                if (!sh) {
 580                        if (!conf->inactive_blocked)
 581                                sh = get_free_stripe(conf);
 582                        if (noblock && sh == NULL)
 583                                break;
 584                        if (!sh) {
 585                                conf->inactive_blocked = 1;
 586                                wait_event_lock_irq(conf->wait_for_stripe,
 587                                                    !list_empty(&conf->inactive_list) &&
 588                                                    (atomic_read(&conf->active_stripes)
 589                                                     < (conf->max_nr_stripes *3/4)
 590                                                     || !conf->inactive_blocked),
 591                                                    conf->device_lock);
 592                                conf->inactive_blocked = 0;
 593                        } else
 594                                init_stripe(sh, sector, previous);
 595                } else {
 596                        if (atomic_read(&sh->count)) {
 597                                BUG_ON(!list_empty(&sh->lru)
 598                                    && !test_bit(STRIPE_EXPANDING, &sh->state)
 599                                    && !test_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)
 600                                    && !test_bit(STRIPE_ON_RELEASE_LIST, &sh->state));
 601                        } else {
 602                                if (!test_bit(STRIPE_HANDLE, &sh->state))
 603                                        atomic_inc(&conf->active_stripes);
 604                                if (list_empty(&sh->lru) &&
 605                                    !test_bit(STRIPE_EXPANDING, &sh->state))
 606                                        BUG();
 607                                list_del_init(&sh->lru);
 608                                if (sh->group) {
 609                                        sh->group->stripes_cnt--;
 610                                        sh->group = NULL;
 611                                }
 612                        }
 613                }
 614        } while (sh == NULL);
 615
 616        if (sh)
 617                atomic_inc(&sh->count);
 618
 619        spin_unlock_irq(&conf->device_lock);
 620        return sh;
 621}
 622
 623/* Determine if 'data_offset' or 'new_data_offset' should be used
 624 * in this stripe_head.
 625 */
 626static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
 627{
 628        sector_t progress = conf->reshape_progress;
 629        /* Need a memory barrier to make sure we see the value
 630         * of conf->generation, or ->data_offset that was set before
 631         * reshape_progress was updated.
 632         */
 633        smp_rmb();
 634        if (progress == MaxSector)
 635                return 0;
 636        if (sh->generation == conf->generation - 1)
 637                return 0;
 638        /* We are in a reshape, and this is a new-generation stripe,
 639         * so use new_data_offset.
 640         */
 641        return 1;
 642}
 643
 644static void
 645raid5_end_read_request(struct bio *bi, int error);
 646static void
 647raid5_end_write_request(struct bio *bi, int error);
 648
 649static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
 650{
 651        struct r5conf *conf = sh->raid_conf;
 652        int i, disks = sh->disks;
 653
 654        might_sleep();
 655
 656        for (i = disks; i--; ) {
 657                int rw;
 658                int replace_only = 0;
 659                struct bio *bi, *rbi;
 660                struct md_rdev *rdev, *rrdev = NULL;
 661                if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
 662                        if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
 663                                rw = WRITE_FUA;
 664                        else
 665                                rw = WRITE;
 666                        if (test_bit(R5_Discard, &sh->dev[i].flags))
 667                                rw |= REQ_DISCARD;
 668                } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
 669                        rw = READ;
 670                else if (test_and_clear_bit(R5_WantReplace,
 671                                            &sh->dev[i].flags)) {
 672                        rw = WRITE;
 673                        replace_only = 1;
 674                } else
 675                        continue;
 676                if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
 677                        rw |= REQ_SYNC;
 678
 679                bi = &sh->dev[i].req;
 680                rbi = &sh->dev[i].rreq; /* For writing to replacement */
 681
 682                rcu_read_lock();
 683                rrdev = rcu_dereference(conf->disks[i].replacement);
 684                smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
 685                rdev = rcu_dereference(conf->disks[i].rdev);
 686                if (!rdev) {
 687                        rdev = rrdev;
 688                        rrdev = NULL;
 689                }
 690                if (rw & WRITE) {
 691                        if (replace_only)
 692                                rdev = NULL;
 693                        if (rdev == rrdev)
 694                                /* We raced and saw duplicates */
 695                                rrdev = NULL;
 696                } else {
 697                        if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev)
 698                                rdev = rrdev;
 699                        rrdev = NULL;
 700                }
 701
 702                if (rdev && test_bit(Faulty, &rdev->flags))
 703                        rdev = NULL;
 704                if (rdev)
 705                        atomic_inc(&rdev->nr_pending);
 706                if (rrdev && test_bit(Faulty, &rrdev->flags))
 707                        rrdev = NULL;
 708                if (rrdev)
 709                        atomic_inc(&rrdev->nr_pending);
 710                rcu_read_unlock();
 711
 712                /* We have already checked bad blocks for reads.  Now
 713                 * need to check for writes.  We never accept write errors
 714                 * on the replacement, so we don't to check rrdev.
 715                 */
 716                while ((rw & WRITE) && rdev &&
 717                       test_bit(WriteErrorSeen, &rdev->flags)) {
 718                        sector_t first_bad;
 719                        int bad_sectors;
 720                        int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
 721                                              &first_bad, &bad_sectors);
 722                        if (!bad)
 723                                break;
 724
 725                        if (bad < 0) {
 726                                set_bit(BlockedBadBlocks, &rdev->flags);
 727                                if (!conf->mddev->external &&
 728                                    conf->mddev->flags) {
 729                                        /* It is very unlikely, but we might
 730                                         * still need to write out the
 731                                         * bad block log - better give it
 732                                         * a chance*/
 733                                        md_check_recovery(conf->mddev);
 734                                }
 735                                /*
 736                                 * Because md_wait_for_blocked_rdev
 737                                 * will dec nr_pending, we must
 738                                 * increment it first.
 739                                 */
 740                                atomic_inc(&rdev->nr_pending);
 741                                md_wait_for_blocked_rdev(rdev, conf->mddev);
 742                        } else {
 743                                /* Acknowledged bad block - skip the write */
 744                                rdev_dec_pending(rdev, conf->mddev);
 745                                rdev = NULL;
 746                        }
 747                }
 748
 749                if (rdev) {
 750                        if (s->syncing || s->expanding || s->expanded
 751                            || s->replacing)
 752                                md_sync_acct(rdev->bdev, STRIPE_SECTORS);
 753
 754                        set_bit(STRIPE_IO_STARTED, &sh->state);
 755
 756                        bio_reset(bi);
 757                        bi->bi_bdev = rdev->bdev;
 758                        bi->bi_rw = rw;
 759                        bi->bi_end_io = (rw & WRITE)
 760                                ? raid5_end_write_request
 761                                : raid5_end_read_request;
 762                        bi->bi_private = sh;
 763
 764                        pr_debug("%s: for %llu schedule op %ld on disc %d\n",
 765                                __func__, (unsigned long long)sh->sector,
 766                                bi->bi_rw, i);
 767                        atomic_inc(&sh->count);
 768                        if (use_new_offset(conf, sh))
 769                                bi->bi_sector = (sh->sector
 770                                                 + rdev->new_data_offset);
 771                        else
 772                                bi->bi_sector = (sh->sector
 773                                                 + rdev->data_offset);
 774                        if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
 775                                bi->bi_rw |= REQ_FLUSH;
 776
 777                        bi->bi_vcnt = 1;
 778                        bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
 779                        bi->bi_io_vec[0].bv_offset = 0;
 780                        bi->bi_size = STRIPE_SIZE;
 781                        /*
 782                         * If this is discard request, set bi_vcnt 0. We don't
 783                         * want to confuse SCSI because SCSI will replace payload
 784                         */
 785                        if (rw & REQ_DISCARD)
 786                                bi->bi_vcnt = 0;
 787                        if (rrdev)
 788                                set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
 789
 790                        if (conf->mddev->gendisk)
 791                                trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
 792                                                      bi, disk_devt(conf->mddev->gendisk),
 793                                                      sh->dev[i].sector);
 794                        generic_make_request(bi);
 795                }
 796                if (rrdev) {
 797                        if (s->syncing || s->expanding || s->expanded
 798                            || s->replacing)
 799                                md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
 800
 801                        set_bit(STRIPE_IO_STARTED, &sh->state);
 802
 803                        bio_reset(rbi);
 804                        rbi->bi_bdev = rrdev->bdev;
 805                        rbi->bi_rw = rw;
 806                        BUG_ON(!(rw & WRITE));
 807                        rbi->bi_end_io = raid5_end_write_request;
 808                        rbi->bi_private = sh;
 809
 810                        pr_debug("%s: for %llu schedule op %ld on "
 811                                 "replacement disc %d\n",
 812                                __func__, (unsigned long long)sh->sector,
 813                                rbi->bi_rw, i);
 814                        atomic_inc(&sh->count);
 815                        if (use_new_offset(conf, sh))
 816                                rbi->bi_sector = (sh->sector
 817                                                  + rrdev->new_data_offset);
 818                        else
 819                                rbi->bi_sector = (sh->sector
 820                                                  + rrdev->data_offset);
 821                        rbi->bi_vcnt = 1;
 822                        rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
 823                        rbi->bi_io_vec[0].bv_offset = 0;
 824                        rbi->bi_size = STRIPE_SIZE;
 825                        /*
 826                         * If this is discard request, set bi_vcnt 0. We don't
 827                         * want to confuse SCSI because SCSI will replace payload
 828                         */
 829                        if (rw & REQ_DISCARD)
 830                                rbi->bi_vcnt = 0;
 831                        if (conf->mddev->gendisk)
 832                                trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
 833                                                      rbi, disk_devt(conf->mddev->gendisk),
 834                                                      sh->dev[i].sector);
 835                        generic_make_request(rbi);
 836                }
 837                if (!rdev && !rrdev) {
 838                        if (rw & WRITE)
 839                                set_bit(STRIPE_DEGRADED, &sh->state);
 840                        pr_debug("skip op %ld on disc %d for sector %llu\n",
 841                                bi->bi_rw, i, (unsigned long long)sh->sector);
 842                        clear_bit(R5_LOCKED, &sh->dev[i].flags);
 843                        set_bit(STRIPE_HANDLE, &sh->state);
 844                }
 845        }
 846}
 847
 848static struct dma_async_tx_descriptor *
 849async_copy_data(int frombio, struct bio *bio, struct page *page,
 850        sector_t sector, struct dma_async_tx_descriptor *tx)
 851{
 852        struct bio_vec *bvl;
 853        struct page *bio_page;
 854        int i;
 855        int page_offset;
 856        struct async_submit_ctl submit;
 857        enum async_tx_flags flags = 0;
 858
 859        if (bio->bi_sector >= sector)
 860                page_offset = (signed)(bio->bi_sector - sector) * 512;
 861        else
 862                page_offset = (signed)(sector - bio->bi_sector) * -512;
 863
 864        if (frombio)
 865                flags |= ASYNC_TX_FENCE;
 866        init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
 867
 868        bio_for_each_segment(bvl, bio, i) {
 869                int len = bvl->bv_len;
 870                int clen;
 871                int b_offset = 0;
 872
 873                if (page_offset < 0) {
 874                        b_offset = -page_offset;
 875                        page_offset += b_offset;
 876                        len -= b_offset;
 877                }
 878
 879                if (len > 0 && page_offset + len > STRIPE_SIZE)
 880                        clen = STRIPE_SIZE - page_offset;
 881                else
 882                        clen = len;
 883
 884                if (clen > 0) {
 885                        b_offset += bvl->bv_offset;
 886                        bio_page = bvl->bv_page;
 887                        if (frombio)
 888                                tx = async_memcpy(page, bio_page, page_offset,
 889                                                  b_offset, clen, &submit);
 890                        else
 891                                tx = async_memcpy(bio_page, page, b_offset,
 892                                                  page_offset, clen, &submit);
 893                }
 894                /* chain the operations */
 895                submit.depend_tx = tx;
 896
 897                if (clen < len) /* hit end of page */
 898                        break;
 899                page_offset +=  len;
 900        }
 901
 902        return tx;
 903}
 904
 905static void ops_complete_biofill(void *stripe_head_ref)
 906{
 907        struct stripe_head *sh = stripe_head_ref;
 908        struct bio *return_bi = NULL;
 909        int i;
 910
 911        pr_debug("%s: stripe %llu\n", __func__,
 912                (unsigned long long)sh->sector);
 913
 914        /* clear completed biofills */
 915        for (i = sh->disks; i--; ) {
 916                struct r5dev *dev = &sh->dev[i];
 917
 918                /* acknowledge completion of a biofill operation */
 919                /* and check if we need to reply to a read request,
 920                 * new R5_Wantfill requests are held off until
 921                 * !STRIPE_BIOFILL_RUN
 922                 */
 923                if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
 924                        struct bio *rbi, *rbi2;
 925
 926                        BUG_ON(!dev->read);
 927                        rbi = dev->read;
 928                        dev->read = NULL;
 929                        while (rbi && rbi->bi_sector <
 930                                dev->sector + STRIPE_SECTORS) {
 931                                rbi2 = r5_next_bio(rbi, dev->sector);
 932                                if (!raid5_dec_bi_active_stripes(rbi)) {
 933                                        rbi->bi_next = return_bi;
 934                                        return_bi = rbi;
 935                                }
 936                                rbi = rbi2;
 937                        }
 938                }
 939        }
 940        clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
 941
 942        return_io(return_bi);
 943
 944        set_bit(STRIPE_HANDLE, &sh->state);
 945        release_stripe(sh);
 946}
 947
 948static void ops_run_biofill(struct stripe_head *sh)
 949{
 950        struct dma_async_tx_descriptor *tx = NULL;
 951        struct async_submit_ctl submit;
 952        int i;
 953
 954        pr_debug("%s: stripe %llu\n", __func__,
 955                (unsigned long long)sh->sector);
 956
 957        for (i = sh->disks; i--; ) {
 958                struct r5dev *dev = &sh->dev[i];
 959                if (test_bit(R5_Wantfill, &dev->flags)) {
 960                        struct bio *rbi;
 961                        spin_lock_irq(&sh->stripe_lock);
 962                        dev->read = rbi = dev->toread;
 963                        dev->toread = NULL;
 964                        spin_unlock_irq(&sh->stripe_lock);
 965                        while (rbi && rbi->bi_sector <
 966                                dev->sector + STRIPE_SECTORS) {
 967                                tx = async_copy_data(0, rbi, dev->page,
 968                                        dev->sector, tx);
 969                                rbi = r5_next_bio(rbi, dev->sector);
 970                        }
 971                }
 972        }
 973
 974        atomic_inc(&sh->count);
 975        init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
 976        async_trigger_callback(&submit);
 977}
 978
 979static void mark_target_uptodate(struct stripe_head *sh, int target)
 980{
 981        struct r5dev *tgt;
 982
 983        if (target < 0)
 984                return;
 985
 986        tgt = &sh->dev[target];
 987        set_bit(R5_UPTODATE, &tgt->flags);
 988        BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
 989        clear_bit(R5_Wantcompute, &tgt->flags);
 990}
 991
 992static void ops_complete_compute(void *stripe_head_ref)
 993{
 994        struct stripe_head *sh = stripe_head_ref;
 995
 996        pr_debug("%s: stripe %llu\n", __func__,
 997                (unsigned long long)sh->sector);
 998
 999        /* mark the computed target(s) as uptodate */
1000        mark_target_uptodate(sh, sh->ops.target);
1001        mark_target_uptodate(sh, sh->ops.target2);
1002
1003        clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
1004        if (sh->check_state == check_state_compute_run)
1005                sh->check_state = check_state_compute_result;
1006        set_bit(STRIPE_HANDLE, &sh->state);
1007        release_stripe(sh);
1008}
1009
1010/* return a pointer to the address conversion region of the scribble buffer */
1011static addr_conv_t *to_addr_conv(struct stripe_head *sh,
1012                                 struct raid5_percpu *percpu)
1013{
1014        return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
1015}
1016
1017static struct dma_async_tx_descriptor *
1018ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1019{
1020        int disks = sh->disks;
1021        struct page **xor_srcs = percpu->scribble;
1022        int target = sh->ops.target;
1023        struct r5dev *tgt = &sh->dev[target];
1024        struct page *xor_dest = tgt->page;
1025        int count = 0;
1026        struct dma_async_tx_descriptor *tx;
1027        struct async_submit_ctl submit;
1028        int i;
1029
1030        pr_debug("%s: stripe %llu block: %d\n",
1031                __func__, (unsigned long long)sh->sector, target);
1032        BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1033
1034        for (i = disks; i--; )
1035                if (i != target)
1036                        xor_srcs[count++] = sh->dev[i].page;
1037
1038        atomic_inc(&sh->count);
1039
1040        init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
1041                          ops_complete_compute, sh, to_addr_conv(sh, percpu));
1042        if (unlikely(count == 1))
1043                tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1044        else
1045                tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1046
1047        return tx;
1048}
1049
1050/* set_syndrome_sources - populate source buffers for gen_syndrome
1051 * @srcs - (struct page *) array of size sh->disks
1052 * @sh - stripe_head to parse
1053 *
1054 * Populates srcs in proper layout order for the stripe and returns the
1055 * 'count' of sources to be used in a call to async_gen_syndrome.  The P
1056 * destination buffer is recorded in srcs[count] and the Q destination
1057 * is recorded in srcs[count+1]].
1058 */
1059static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
1060{
1061        int disks = sh->disks;
1062        int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
1063        int d0_idx = raid6_d0(sh);
1064        int count;
1065        int i;
1066
1067        for (i = 0; i < disks; i++)
1068                srcs[i] = NULL;
1069
1070        count = 0;
1071        i = d0_idx;
1072        do {
1073                int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1074
1075                srcs[slot] = sh->dev[i].page;
1076                i = raid6_next_disk(i, disks);
1077        } while (i != d0_idx);
1078
1079        return syndrome_disks;
1080}
1081
1082static struct dma_async_tx_descriptor *
1083ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
1084{
1085        int disks = sh->disks;
1086        struct page **blocks = percpu->scribble;
1087        int target;
1088        int qd_idx = sh->qd_idx;
1089        struct dma_async_tx_descriptor *tx;
1090        struct async_submit_ctl submit;
1091        struct r5dev *tgt;
1092        struct page *dest;
1093        int i;
1094        int count;
1095
1096        if (sh->ops.target < 0)
1097                target = sh->ops.target2;
1098        else if (sh->ops.target2 < 0)
1099                target = sh->ops.target;
1100        else
1101                /* we should only have one valid target */
1102                BUG();
1103        BUG_ON(target < 0);
1104        pr_debug("%s: stripe %llu block: %d\n",
1105                __func__, (unsigned long long)sh->sector, target);
1106
1107        tgt = &sh->dev[target];
1108        BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1109        dest = tgt->page;
1110
1111        atomic_inc(&sh->count);
1112
1113        if (target == qd_idx) {
1114                count = set_syndrome_sources(blocks, sh);
1115                blocks[count] = NULL; /* regenerating p is not necessary */
1116                BUG_ON(blocks[count+1] != dest); /* q should already be set */
1117                init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1118                                  ops_complete_compute, sh,
1119                                  to_addr_conv(sh, percpu));
1120                tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
1121        } else {
1122                /* Compute any data- or p-drive using XOR */
1123                count = 0;
1124                for (i = disks; i-- ; ) {
1125                        if (i == target || i == qd_idx)
1126                                continue;
1127                        blocks[count++] = sh->dev[i].page;
1128                }
1129
1130                init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1131                                  NULL, ops_complete_compute, sh,
1132                                  to_addr_conv(sh, percpu));
1133                tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
1134        }
1135
1136        return tx;
1137}
1138
1139static struct dma_async_tx_descriptor *
1140ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
1141{
1142        int i, count, disks = sh->disks;
1143        int syndrome_disks = sh->ddf_layout ? disks : disks-2;
1144        int d0_idx = raid6_d0(sh);
1145        int faila = -1, failb = -1;
1146        int target = sh->ops.target;
1147        int target2 = sh->ops.target2;
1148        struct r5dev *tgt = &sh->dev[target];
1149        struct r5dev *tgt2 = &sh->dev[target2];
1150        struct dma_async_tx_descriptor *tx;
1151        struct page **blocks = percpu->scribble;
1152        struct async_submit_ctl submit;
1153
1154        pr_debug("%s: stripe %llu block1: %d block2: %d\n",
1155                 __func__, (unsigned long long)sh->sector, target, target2);
1156        BUG_ON(target < 0 || target2 < 0);
1157        BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1158        BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
1159
1160        /* we need to open-code set_syndrome_sources to handle the
1161         * slot number conversion for 'faila' and 'failb'
1162         */
1163        for (i = 0; i < disks ; i++)
1164                blocks[i] = NULL;
1165        count = 0;
1166        i = d0_idx;
1167        do {
1168                int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1169
1170                blocks[slot] = sh->dev[i].page;
1171
1172                if (i == target)
1173                        faila = slot;
1174                if (i == target2)
1175                        failb = slot;
1176                i = raid6_next_disk(i, disks);
1177        } while (i != d0_idx);
1178
1179        BUG_ON(faila == failb);
1180        if (failb < faila)
1181                swap(faila, failb);
1182        pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
1183                 __func__, (unsigned long long)sh->sector, faila, failb);
1184
1185        atomic_inc(&sh->count);
1186
1187        if (failb == syndrome_disks+1) {
1188                /* Q disk is one of the missing disks */
1189                if (faila == syndrome_disks) {
1190                        /* Missing P+Q, just recompute */
1191                        init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1192                                          ops_complete_compute, sh,
1193                                          to_addr_conv(sh, percpu));
1194                        return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1195                                                  STRIPE_SIZE, &submit);
1196                } else {
1197                        struct page *dest;
1198                        int data_target;
1199                        int qd_idx = sh->qd_idx;
1200
1201                        /* Missing D+Q: recompute D from P, then recompute Q */
1202                        if (target == qd_idx)
1203                                data_target = target2;
1204                        else
1205                                data_target = target;
1206
1207                        count = 0;
1208                        for (i = disks; i-- ; ) {
1209                                if (i == data_target || i == qd_idx)
1210                                        continue;
1211                                blocks[count++] = sh->dev[i].page;
1212                        }
1213                        dest = sh->dev[data_target].page;
1214                        init_async_submit(&submit,
1215                                          ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1216                                          NULL, NULL, NULL,
1217                                          to_addr_conv(sh, percpu));
1218                        tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
1219                                       &submit);
1220
1221                        count = set_syndrome_sources(blocks, sh);
1222                        init_async_submit(&submit, ASYNC_TX_FENCE, tx,
1223                                          ops_complete_compute, sh,
1224                                          to_addr_conv(sh, percpu));
1225                        return async_gen_syndrome(blocks, 0, count+2,
1226                                                  STRIPE_SIZE, &submit);
1227                }
1228        } else {
1229                init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1230                                  ops_complete_compute, sh,
1231                                  to_addr_conv(sh, percpu));
1232                if (failb == syndrome_disks) {
1233                        /* We're missing D+P. */
1234                        return async_raid6_datap_recov(syndrome_disks+2,
1235                                                       STRIPE_SIZE, faila,
1236                                                       blocks, &submit);
1237                } else {
1238                        /* We're missing D+D. */
1239                        return async_raid6_2data_recov(syndrome_disks+2,
1240                                                       STRIPE_SIZE, faila, failb,
1241                                                       blocks, &submit);
1242                }
1243        }
1244}
1245
1246
1247static void ops_complete_prexor(void *stripe_head_ref)
1248{
1249        struct stripe_head *sh = stripe_head_ref;
1250
1251        pr_debug("%s: stripe %llu\n", __func__,
1252                (unsigned long long)sh->sector);
1253}
1254
1255static struct dma_async_tx_descriptor *
1256ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
1257               struct dma_async_tx_descriptor *tx)
1258{
1259        int disks = sh->disks;
1260        struct page **xor_srcs = percpu->scribble;
1261        int count = 0, pd_idx = sh->pd_idx, i;
1262        struct async_submit_ctl submit;
1263
1264        /* existing parity data subtracted */
1265        struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1266
1267        pr_debug("%s: stripe %llu\n", __func__,
1268                (unsigned long long)sh->sector);
1269
1270        for (i = disks; i--; ) {
1271                struct r5dev *dev = &sh->dev[i];
1272                /* Only process blocks that are known to be uptodate */
1273                if (test_bit(R5_Wantdrain, &dev->flags))
1274                        xor_srcs[count++] = dev->page;
1275        }
1276
1277        init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
1278                          ops_complete_prexor, sh, to_addr_conv(sh, percpu));
1279        tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1280
1281        return tx;
1282}
1283
1284static struct dma_async_tx_descriptor *
1285ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1286{
1287        int disks = sh->disks;
1288        int i;
1289
1290        pr_debug("%s: stripe %llu\n", __func__,
1291                (unsigned long long)sh->sector);
1292
1293        for (i = disks; i--; ) {
1294                struct r5dev *dev = &sh->dev[i];
1295                struct bio *chosen;
1296
1297                if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
1298                        struct bio *wbi;
1299
1300                        spin_lock_irq(&sh->stripe_lock);
1301                        chosen = dev->towrite;
1302                        dev->towrite = NULL;
1303                        BUG_ON(dev->written);
1304                        wbi = dev->written = chosen;
1305                        spin_unlock_irq(&sh->stripe_lock);
1306
1307                        while (wbi && wbi->bi_sector <
1308                                dev->sector + STRIPE_SECTORS) {
1309                                if (wbi->bi_rw & REQ_FUA)
1310                                        set_bit(R5_WantFUA, &dev->flags);
1311                                if (wbi->bi_rw & REQ_SYNC)
1312                                        set_bit(R5_SyncIO, &dev->flags);
1313                                if (wbi->bi_rw & REQ_DISCARD)
1314                                        set_bit(R5_Discard, &dev->flags);
1315                                else
1316                                        tx = async_copy_data(1, wbi, dev->page,
1317                                                dev->sector, tx);
1318                                wbi = r5_next_bio(wbi, dev->sector);
1319                        }
1320                }
1321        }
1322
1323        return tx;
1324}
1325
1326static void ops_complete_reconstruct(void *stripe_head_ref)
1327{
1328        struct stripe_head *sh = stripe_head_ref;
1329        int disks = sh->disks;
1330        int pd_idx = sh->pd_idx;
1331        int qd_idx = sh->qd_idx;
1332        int i;
1333        bool fua = false, sync = false, discard = false;
1334
1335        pr_debug("%s: stripe %llu\n", __func__,
1336                (unsigned long long)sh->sector);
1337
1338        for (i = disks; i--; ) {
1339                fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
1340                sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
1341                discard |= test_bit(R5_Discard, &sh->dev[i].flags);
1342        }
1343
1344        for (i = disks; i--; ) {
1345                struct r5dev *dev = &sh->dev[i];
1346
1347                if (dev->written || i == pd_idx || i == qd_idx) {
1348                        if (!discard)
1349                                set_bit(R5_UPTODATE, &dev->flags);
1350                        if (fua)
1351                                set_bit(R5_WantFUA, &dev->flags);
1352                        if (sync)
1353                                set_bit(R5_SyncIO, &dev->flags);
1354                }
1355        }
1356
1357        if (sh->reconstruct_state == reconstruct_state_drain_run)
1358                sh->reconstruct_state = reconstruct_state_drain_result;
1359        else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
1360                sh->reconstruct_state = reconstruct_state_prexor_drain_result;
1361        else {
1362                BUG_ON(sh->reconstruct_state != reconstruct_state_run);
1363                sh->reconstruct_state = reconstruct_state_result;
1364        }
1365
1366        set_bit(STRIPE_HANDLE, &sh->state);
1367        release_stripe(sh);
1368}
1369
1370static void
1371ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1372                     struct dma_async_tx_descriptor *tx)
1373{
1374        int disks = sh->disks;
1375        struct page **xor_srcs = percpu->scribble;
1376        struct async_submit_ctl submit;
1377        int count = 0, pd_idx = sh->pd_idx, i;
1378        struct page *xor_dest;
1379        int prexor = 0;
1380        unsigned long flags;
1381
1382        pr_debug("%s: stripe %llu\n", __func__,
1383                (unsigned long long)sh->sector);
1384
1385        for (i = 0; i < sh->disks; i++) {
1386                if (pd_idx == i)
1387                        continue;
1388                if (!test_bit(R5_Discard, &sh->dev[i].flags))
1389                        break;
1390        }
1391        if (i >= sh->disks) {
1392                atomic_inc(&sh->count);
1393                set_bit(R5_Discard, &sh->dev[pd_idx].flags);
1394                ops_complete_reconstruct(sh);
1395                return;
1396        }
1397        /* check if prexor is active which means only process blocks
1398         * that are part of a read-modify-write (written)
1399         */
1400        if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1401                prexor = 1;
1402                xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1403                for (i = disks; i--; ) {
1404                        struct r5dev *dev = &sh->dev[i];
1405                        if (dev->written)
1406                                xor_srcs[count++] = dev->page;
1407                }
1408        } else {
1409                xor_dest = sh->dev[pd_idx].page;
1410                for (i = disks; i--; ) {
1411                        struct r5dev *dev = &sh->dev[i];
1412                        if (i != pd_idx)
1413                                xor_srcs[count++] = dev->page;
1414                }
1415        }
1416
1417        /* 1/ if we prexor'd then the dest is reused as a source
1418         * 2/ if we did not prexor then we are redoing the parity
1419         * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
1420         * for the synchronous xor case
1421         */
1422        flags = ASYNC_TX_ACK |
1423                (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
1424
1425        atomic_inc(&sh->count);
1426
1427        init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
1428                          to_addr_conv(sh, percpu));
1429        if (unlikely(count == 1))
1430                tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1431        else
1432                tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1433}
1434
1435static void
1436ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
1437                     struct dma_async_tx_descriptor *tx)
1438{
1439        struct async_submit_ctl submit;
1440        struct page **blocks = percpu->scribble;
1441        int count, i;
1442
1443        pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
1444
1445        for (i = 0; i < sh->disks; i++) {
1446                if (sh->pd_idx == i || sh->qd_idx == i)
1447                        continue;
1448                if (!test_bit(R5_Discard, &sh->dev[i].flags))
1449                        break;
1450        }
1451        if (i >= sh->disks) {
1452                atomic_inc(&sh->count);
1453                set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
1454                set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
1455                ops_complete_reconstruct(sh);
1456                return;
1457        }
1458
1459        count = set_syndrome_sources(blocks, sh);
1460
1461        atomic_inc(&sh->count);
1462
1463        init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
1464                          sh, to_addr_conv(sh, percpu));
1465        async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1466}
1467
1468static void ops_complete_check(void *stripe_head_ref)
1469{
1470        struct stripe_head *sh = stripe_head_ref;
1471
1472        pr_debug("%s: stripe %llu\n", __func__,
1473                (unsigned long long)sh->sector);
1474
1475        sh->check_state = check_state_check_result;
1476        set_bit(STRIPE_HANDLE, &sh->state);
1477        release_stripe(sh);
1478}
1479
1480static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1481{
1482        int disks = sh->disks;
1483        int pd_idx = sh->pd_idx;
1484        int qd_idx = sh->qd_idx;
1485        struct page *xor_dest;
1486        struct page **xor_srcs = percpu->scribble;
1487        struct dma_async_tx_descriptor *tx;
1488        struct async_submit_ctl submit;
1489        int count;
1490        int i;
1491
1492        pr_debug("%s: stripe %llu\n", __func__,
1493                (unsigned long long)sh->sector);
1494
1495        count = 0;
1496        xor_dest = sh->dev[pd_idx].page;
1497        xor_srcs[count++] = xor_dest;
1498        for (i = disks; i--; ) {
1499                if (i == pd_idx || i == qd_idx)
1500                        continue;
1501                xor_srcs[count++] = sh->dev[i].page;
1502        }
1503
1504        init_async_submit(&submit, 0, NULL, NULL, NULL,
1505                          to_addr_conv(sh, percpu));
1506        tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1507                           &sh->ops.zero_sum_result, &submit);
1508
1509        atomic_inc(&sh->count);
1510        init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
1511        tx = async_trigger_callback(&submit);
1512}
1513
1514static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
1515{
1516        struct page **srcs = percpu->scribble;
1517        struct async_submit_ctl submit;
1518        int count;
1519
1520        pr_debug("%s: stripe %llu checkp: %d\n", __func__,
1521                (unsigned long long)sh->sector, checkp);
1522
1523        count = set_syndrome_sources(srcs, sh);
1524        if (!checkp)
1525                srcs[count] = NULL;
1526
1527        atomic_inc(&sh->count);
1528        init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
1529                          sh, to_addr_conv(sh, percpu));
1530        async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
1531                           &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1532}
1533
1534static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1535{
1536        int overlap_clear = 0, i, disks = sh->disks;
1537        struct dma_async_tx_descriptor *tx = NULL;
1538        struct r5conf *conf = sh->raid_conf;
1539        int level = conf->level;
1540        struct raid5_percpu *percpu;
1541        unsigned long cpu;
1542
1543        cpu = get_cpu();
1544        percpu = per_cpu_ptr(conf->percpu, cpu);
1545        if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1546                ops_run_biofill(sh);
1547                overlap_clear++;
1548        }
1549
1550        if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1551                if (level < 6)
1552                        tx = ops_run_compute5(sh, percpu);
1553                else {
1554                        if (sh->ops.target2 < 0 || sh->ops.target < 0)
1555                                tx = ops_run_compute6_1(sh, percpu);
1556                        else
1557                                tx = ops_run_compute6_2(sh, percpu);
1558                }
1559                /* terminate the chain if reconstruct is not set to be run */
1560                if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1561                        async_tx_ack(tx);
1562        }
1563
1564        if (test_bit(STRIPE_OP_PREXOR, &ops_request))
1565                tx = ops_run_prexor(sh, percpu, tx);
1566
1567        if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1568                tx = ops_run_biodrain(sh, tx);
1569                overlap_clear++;
1570        }
1571
1572        if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
1573                if (level < 6)
1574                        ops_run_reconstruct5(sh, percpu, tx);
1575                else
1576                        ops_run_reconstruct6(sh, percpu, tx);
1577        }
1578
1579        if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
1580                if (sh->check_state == check_state_run)
1581                        ops_run_check_p(sh, percpu);
1582                else if (sh->check_state == check_state_run_q)
1583                        ops_run_check_pq(sh, percpu, 0);
1584                else if (sh->check_state == check_state_run_pq)
1585                        ops_run_check_pq(sh, percpu, 1);
1586                else
1587                        BUG();
1588        }
1589
1590        if (overlap_clear)
1591                for (i = disks; i--; ) {
1592                        struct r5dev *dev = &sh->dev[i];
1593                        if (test_and_clear_bit(R5_Overlap, &dev->flags))
1594                                wake_up(&sh->raid_conf->wait_for_overlap);
1595                }
1596        put_cpu();
1597}
1598
1599static int grow_one_stripe(struct r5conf *conf)
1600{
1601        struct stripe_head *sh;
1602        sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL);
1603        if (!sh)
1604                return 0;
1605
1606        sh->raid_conf = conf;
1607
1608        spin_lock_init(&sh->stripe_lock);
1609
1610        if (grow_buffers(sh)) {
1611                shrink_buffers(sh);
1612                kmem_cache_free(conf->slab_cache, sh);
1613                return 0;
1614        }
1615        /* we just created an active stripe so... */
1616        atomic_set(&sh->count, 1);
1617        atomic_inc(&conf->active_stripes);
1618        INIT_LIST_HEAD(&sh->lru);
1619        release_stripe(sh);
1620        return 1;
1621}
1622
1623static int grow_stripes(struct r5conf *conf, int num)
1624{
1625        struct kmem_cache *sc;
1626        int devs = max(conf->raid_disks, conf->previous_raid_disks);
1627
1628        if (conf->mddev->gendisk)
1629                sprintf(conf->cache_name[0],
1630                        "raid%d-%s", conf->level, mdname(conf->mddev));
1631        else
1632                sprintf(conf->cache_name[0],
1633                        "raid%d-%p", conf->level, conf->mddev);
1634        sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
1635
1636        conf->active_name = 0;
1637        sc = kmem_cache_create(conf->cache_name[conf->active_name],
1638                               sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
1639                               0, 0, NULL);
1640        if (!sc)
1641                return 1;
1642        conf->slab_cache = sc;
1643        conf->pool_size = devs;
1644        while (num--)
1645                if (!grow_one_stripe(conf))
1646                        return 1;
1647        return 0;
1648}
1649
1650/**
1651 * scribble_len - return the required size of the scribble region
1652 * @num - total number of disks in the array
1653 *
1654 * The size must be enough to contain:
1655 * 1/ a struct page pointer for each device in the array +2
1656 * 2/ room to convert each entry in (1) to its corresponding dma
1657 *    (dma_map_page()) or page (page_address()) address.
1658 *
1659 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
1660 * calculate over all devices (not just the data blocks), using zeros in place
1661 * of the P and Q blocks.
1662 */
1663static size_t scribble_len(int num)
1664{
1665        size_t len;
1666
1667        len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
1668
1669        return len;
1670}
1671
1672static int resize_stripes(struct r5conf *conf, int newsize)
1673{
1674        /* Make all the stripes able to hold 'newsize' devices.
1675         * New slots in each stripe get 'page' set to a new page.
1676         *
1677         * This happens in stages:
1678         * 1/ create a new kmem_cache and allocate the required number of
1679         *    stripe_heads.
1680         * 2/ gather all the old stripe_heads and transfer the pages across
1681         *    to the new stripe_heads.  This will have the side effect of
1682         *    freezing the array as once all stripe_heads have been collected,
1683         *    no IO will be possible.  Old stripe heads are freed once their
1684         *    pages have been transferred over, and the old kmem_cache is
1685         *    freed when all stripes are done.
1686         * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
1687         *    we simple return a failre status - no need to clean anything up.
1688         * 4/ allocate new pages for the new slots in the new stripe_heads.
1689         *    If this fails, we don't bother trying the shrink the
1690         *    stripe_heads down again, we just leave them as they are.
1691         *    As each stripe_head is processed the new one is released into
1692         *    active service.
1693         *
1694         * Once step2 is started, we cannot afford to wait for a write,
1695         * so we use GFP_NOIO allocations.
1696         */
1697        struct stripe_head *osh, *nsh;
1698        LIST_HEAD(newstripes);
1699        struct disk_info *ndisks;
1700        unsigned long cpu;
1701        int err;
1702        struct kmem_cache *sc;
1703        int i;
1704
1705        if (newsize <= conf->pool_size)
1706                return 0; /* never bother to shrink */
1707
1708        err = md_allow_write(conf->mddev);
1709        if (err)
1710                return err;
1711
1712        /* Step 1 */
1713        sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
1714                               sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
1715                               0, 0, NULL);
1716        if (!sc)
1717                return -ENOMEM;
1718
1719        for (i = conf->max_nr_stripes; i; i--) {
1720                nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
1721                if (!nsh)
1722                        break;
1723
1724                nsh->raid_conf = conf;
1725                spin_lock_init(&nsh->stripe_lock);
1726
1727                list_add(&nsh->lru, &newstripes);
1728        }
1729        if (i) {
1730                /* didn't get enough, give up */
1731                while (!list_empty(&newstripes)) {
1732                        nsh = list_entry(newstripes.next, struct stripe_head, lru);
1733                        list_del(&nsh->lru);
1734                        kmem_cache_free(sc, nsh);
1735                }
1736                kmem_cache_destroy(sc);
1737                return -ENOMEM;
1738        }
1739        /* Step 2 - Must use GFP_NOIO now.
1740         * OK, we have enough stripes, start collecting inactive
1741         * stripes and copying them over
1742         */
1743        list_for_each_entry(nsh, &newstripes, lru) {
1744                spin_lock_irq(&conf->device_lock);
1745                wait_event_lock_irq(conf->wait_for_stripe,
1746                                    !list_empty(&conf->inactive_list),
1747                                    conf->device_lock);
1748                osh = get_free_stripe(conf);
1749                spin_unlock_irq(&conf->device_lock);
1750                atomic_set(&nsh->count, 1);
1751                for(i=0; i<conf->pool_size; i++)
1752                        nsh->dev[i].page = osh->dev[i].page;
1753                for( ; i<newsize; i++)
1754                        nsh->dev[i].page = NULL;
1755                kmem_cache_free(conf->slab_cache, osh);
1756        }
1757        kmem_cache_destroy(conf->slab_cache);
1758
1759        /* Step 3.
1760         * At this point, we are holding all the stripes so the array
1761         * is completely stalled, so now is a good time to resize
1762         * conf->disks and the scribble region
1763         */
1764        ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
1765        if (ndisks) {
1766                for (i=0; i<conf->raid_disks; i++)
1767                        ndisks[i] = conf->disks[i];
1768                kfree(conf->disks);
1769                conf->disks = ndisks;
1770        } else
1771                err = -ENOMEM;
1772
1773        get_online_cpus();
1774        conf->scribble_len = scribble_len(newsize);
1775        for_each_present_cpu(cpu) {
1776                struct raid5_percpu *percpu;
1777                void *scribble;
1778
1779                percpu = per_cpu_ptr(conf->percpu, cpu);
1780                scribble = kmalloc(conf->scribble_len, GFP_NOIO);
1781
1782                if (scribble) {
1783                        kfree(percpu->scribble);
1784                        percpu->scribble = scribble;
1785                } else {
1786                        err = -ENOMEM;
1787                        break;
1788                }
1789        }
1790        put_online_cpus();
1791
1792        /* Step 4, return new stripes to service */
1793        while(!list_empty(&newstripes)) {
1794                nsh = list_entry(newstripes.next, struct stripe_head, lru);
1795                list_del_init(&nsh->lru);
1796
1797                for (i=conf->raid_disks; i < newsize; i++)
1798                        if (nsh->dev[i].page == NULL) {
1799                                struct page *p = alloc_page(GFP_NOIO);
1800                                nsh->dev[i].page = p;
1801                                if (!p)
1802                                        err = -ENOMEM;
1803                        }
1804                release_stripe(nsh);
1805        }
1806        /* critical section pass, GFP_NOIO no longer needed */
1807
1808        conf->slab_cache = sc;
1809        conf->active_name = 1-conf->active_name;
1810        conf->pool_size = newsize;
1811        return err;
1812}
1813
1814static int drop_one_stripe(struct r5conf *conf)
1815{
1816        struct stripe_head *sh;
1817
1818        spin_lock_irq(&conf->device_lock);
1819        sh = get_free_stripe(conf);
1820        spin_unlock_irq(&conf->device_lock);
1821        if (!sh)
1822                return 0;
1823        BUG_ON(atomic_read(&sh->count));
1824        shrink_buffers(sh);
1825        kmem_cache_free(conf->slab_cache, sh);
1826        atomic_dec(&conf->active_stripes);
1827        return 1;
1828}
1829
1830static void shrink_stripes(struct r5conf *conf)
1831{
1832        while (drop_one_stripe(conf))
1833                ;
1834
1835        if (conf->slab_cache)
1836                kmem_cache_destroy(conf->slab_cache);
1837        conf->slab_cache = NULL;
1838}
1839
1840static void raid5_end_read_request(struct bio * bi, int error)
1841{
1842        struct stripe_head *sh = bi->bi_private;
1843        struct r5conf *conf = sh->raid_conf;
1844        int disks = sh->disks, i;
1845        int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1846        char b[BDEVNAME_SIZE];
1847        struct md_rdev *rdev = NULL;
1848        sector_t s;
1849
1850        for (i=0 ; i<disks; i++)
1851                if (bi == &sh->dev[i].req)
1852                        break;
1853
1854        pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
1855                (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1856                uptodate);
1857        if (i == disks) {
1858                BUG();
1859                return;
1860        }
1861        if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
1862                /* If replacement finished while this request was outstanding,
1863                 * 'replacement' might be NULL already.
1864                 * In that case it moved down to 'rdev'.
1865                 * rdev is not removed until all requests are finished.
1866                 */
1867                rdev = conf->disks[i].replacement;
1868        if (!rdev)
1869                rdev = conf->disks[i].rdev;
1870
1871        if (use_new_offset(conf, sh))
1872                s = sh->sector + rdev->new_data_offset;
1873        else
1874                s = sh->sector + rdev->data_offset;
1875        if (uptodate) {
1876                set_bit(R5_UPTODATE, &sh->dev[i].flags);
1877                if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
1878                        /* Note that this cannot happen on a
1879                         * replacement device.  We just fail those on
1880                         * any error
1881                         */
1882                        printk_ratelimited(
1883                                KERN_INFO
1884                                "md/raid:%s: read error corrected"
1885                                " (%lu sectors at %llu on %s)\n",
1886                                mdname(conf->mddev), STRIPE_SECTORS,
1887                                (unsigned long long)s,
1888                                bdevname(rdev->bdev, b));
1889                        atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
1890                        clear_bit(R5_ReadError, &sh->dev[i].flags);
1891                        clear_bit(R5_ReWrite, &sh->dev[i].flags);
1892                } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
1893                        clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
1894
1895                if (atomic_read(&rdev->read_errors))
1896                        atomic_set(&rdev->read_errors, 0);
1897        } else {
1898                const char *bdn = bdevname(rdev->bdev, b);
1899                int retry = 0;
1900                int set_bad = 0;
1901
1902                clear_bit(R5_UPTODATE, &sh->dev[i].flags);
1903                atomic_inc(&rdev->read_errors);
1904                if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
1905                        printk_ratelimited(
1906                                KERN_WARNING
1907                                "md/raid:%s: read error on replacement device "
1908                                "(sector %llu on %s).\n",
1909                                mdname(conf->mddev),
1910                                (unsigned long long)s,
1911                                bdn);
1912                else if (conf->mddev->degraded >= conf->max_degraded) {
1913                        set_bad = 1;
1914                        printk_ratelimited(
1915                                KERN_WARNING
1916                                "md/raid:%s: read error not correctable "
1917                                "(sector %llu on %s).\n",
1918                                mdname(conf->mddev),
1919                                (unsigned long long)s,
1920                                bdn);
1921                } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
1922                        /* Oh, no!!! */
1923                        set_bad = 1;
1924                        printk_ratelimited(
1925                                KERN_WARNING
1926                                "md/raid:%s: read error NOT corrected!! "
1927                                "(sector %llu on %s).\n",
1928                                mdname(conf->mddev),
1929                                (unsigned long long)s,
1930                                bdn);
1931                } else if (atomic_read(&rdev->read_errors)
1932                         > conf->max_nr_stripes)
1933                        printk(KERN_WARNING
1934                               "md/raid:%s: Too many read errors, failing device %s.\n",
1935                               mdname(conf->mddev), bdn);
1936                else
1937                        retry = 1;
1938                if (retry)
1939                        if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
1940                                set_bit(R5_ReadError, &sh->dev[i].flags);
1941                                clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
1942                        } else
1943                                set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
1944                else {
1945                        clear_bit(R5_ReadError, &sh->dev[i].flags);
1946                        clear_bit(R5_ReWrite, &sh->dev[i].flags);
1947                        if (!(set_bad
1948                              && test_bit(In_sync, &rdev->flags)
1949                              && rdev_set_badblocks(
1950                                      rdev, sh->sector, STRIPE_SECTORS, 0)))
1951                                md_error(conf->mddev, rdev);
1952                }
1953        }
1954        rdev_dec_pending(rdev, conf->mddev);
1955        clear_bit(R5_LOCKED, &sh->dev[i].flags);
1956        set_bit(STRIPE_HANDLE, &sh->state);
1957        release_stripe(sh);
1958}
1959
1960static void raid5_end_write_request(struct bio *bi, int error)
1961{
1962        struct stripe_head *sh = bi->bi_private;
1963        struct r5conf *conf = sh->raid_conf;
1964        int disks = sh->disks, i;
1965        struct md_rdev *uninitialized_var(rdev);
1966        int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1967        sector_t first_bad;
1968        int bad_sectors;
1969        int replacement = 0;
1970
1971        for (i = 0 ; i < disks; i++) {
1972                if (bi == &sh->dev[i].req) {
1973                        rdev = conf->disks[i].rdev;
1974                        break;
1975                }
1976                if (bi == &sh->dev[i].rreq) {
1977                        rdev = conf->disks[i].replacement;
1978                        if (rdev)
1979                                replacement = 1;
1980                        else
1981                                /* rdev was removed and 'replacement'
1982                                 * replaced it.  rdev is not removed
1983                                 * until all requests are finished.
1984                                 */
1985                                rdev = conf->disks[i].rdev;
1986                        break;
1987                }
1988        }
1989        pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
1990                (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1991                uptodate);
1992        if (i == disks) {
1993                BUG();
1994                return;
1995        }
1996
1997        if (replacement) {
1998                if (!uptodate)
1999                        md_error(conf->mddev, rdev);
2000                else if (is_badblock(rdev, sh->sector,
2001                                     STRIPE_SECTORS,
2002                                     &first_bad, &bad_sectors))
2003                        set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2004        } else {
2005                if (!uptodate) {
2006                        set_bit(WriteErrorSeen, &rdev->flags);
2007                        set_bit(R5_WriteError, &sh->dev[i].flags);
2008                        if (!test_and_set_bit(WantReplacement, &rdev->flags))
2009                                set_bit(MD_RECOVERY_NEEDED,
2010                                        &rdev->mddev->recovery);
2011                } else if (is_badblock(rdev, sh->sector,
2012                                       STRIPE_SECTORS,
2013                                       &first_bad, &bad_sectors)) {
2014                        set_bit(R5_MadeGood, &sh->dev[i].flags);
2015                        if (test_bit(R5_ReadError, &sh->dev[i].flags))
2016                                /* That was a successful write so make
2017                                 * sure it looks like we already did
2018                                 * a re-write.
2019                                 */
2020                                set_bit(R5_ReWrite, &sh->dev[i].flags);
2021                }
2022        }
2023        rdev_dec_pending(rdev, conf->mddev);
2024
2025        if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2026                clear_bit(R5_LOCKED, &sh->dev[i].flags);
2027        set_bit(STRIPE_HANDLE, &sh->state);
2028        release_stripe(sh);
2029}
2030
2031static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
2032        
2033static void raid5_build_block(struct stripe_head *sh, int i, int previous)
2034{
2035        struct r5dev *dev = &sh->dev[i];
2036
2037        bio_init(&dev->req);
2038        dev->req.bi_io_vec = &dev->vec;
2039        dev->req.bi_vcnt++;
2040        dev->req.bi_max_vecs++;
2041        dev->req.bi_private = sh;
2042        dev->vec.bv_page = dev->page;
2043
2044        bio_init(&dev->rreq);
2045        dev->rreq.bi_io_vec = &dev->rvec;
2046        dev->rreq.bi_vcnt++;
2047        dev->rreq.bi_max_vecs++;
2048        dev->rreq.bi_private = sh;
2049        dev->rvec.bv_page = dev->page;
2050
2051        dev->flags = 0;
2052        dev->sector = compute_blocknr(sh, i, previous);
2053}
2054
2055static void error(struct mddev *mddev, struct md_rdev *rdev)
2056{
2057        char b[BDEVNAME_SIZE];
2058        struct r5conf *conf = mddev->private;
2059        unsigned long flags;
2060        pr_debug("raid456: error called\n");
2061
2062        spin_lock_irqsave(&conf->device_lock, flags);
2063        clear_bit(In_sync, &rdev->flags);
2064        mddev->degraded = calc_degraded(conf);
2065        spin_unlock_irqrestore(&conf->device_lock, flags);
2066        set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2067
2068        set_bit(Blocked, &rdev->flags);
2069        set_bit(Faulty, &rdev->flags);
2070        set_bit(MD_CHANGE_DEVS, &mddev->flags);
2071        printk(KERN_ALERT
2072               "md/raid:%s: Disk failure on %s, disabling device.\n"
2073               "md/raid:%s: Operation continuing on %d devices.\n",
2074               mdname(mddev),
2075               bdevname(rdev->bdev, b),
2076               mdname(mddev),
2077               conf->raid_disks - mddev->degraded);
2078}
2079
2080/*
2081 * Input: a 'big' sector number,
2082 * Output: index of the data and parity disk, and the sector # in them.
2083 */
2084static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
2085                                     int previous, int *dd_idx,
2086                                     struct stripe_head *sh)
2087{
2088        sector_t stripe, stripe2;
2089        sector_t chunk_number;
2090        unsigned int chunk_offset;
2091        int pd_idx, qd_idx;
2092        int ddf_layout = 0;
2093        sector_t new_sector;
2094        int algorithm = previous ? conf->prev_algo
2095                                 : conf->algorithm;
2096        int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2097                                         : conf->chunk_sectors;
2098        int raid_disks = previous ? conf->previous_raid_disks
2099                                  : conf->raid_disks;
2100        int data_disks = raid_disks - conf->max_degraded;
2101
2102        /* First compute the information on this sector */
2103
2104        /*
2105         * Compute the chunk number and the sector offset inside the chunk
2106         */
2107        chunk_offset = sector_div(r_sector, sectors_per_chunk);
2108        chunk_number = r_sector;
2109
2110        /*
2111         * Compute the stripe number
2112         */
2113        stripe = chunk_number;
2114        *dd_idx = sector_div(stripe, data_disks);
2115        stripe2 = stripe;
2116        /*
2117         * Select the parity disk based on the user selected algorithm.
2118         */
2119        pd_idx = qd_idx = -1;
2120        switch(conf->level) {
2121        case 4:
2122                pd_idx = data_disks;
2123                break;
2124        case 5:
2125                switch (algorithm) {
2126                case ALGORITHM_LEFT_ASYMMETRIC:
2127                        pd_idx = data_disks - sector_div(stripe2, raid_disks);
2128                        if (*dd_idx >= pd_idx)
2129                                (*dd_idx)++;
2130                        break;
2131                case ALGORITHM_RIGHT_ASYMMETRIC:
2132                        pd_idx = sector_div(stripe2, raid_disks);
2133                        if (*dd_idx >= pd_idx)
2134                                (*dd_idx)++;
2135                        break;
2136                case ALGORITHM_LEFT_SYMMETRIC:
2137                        pd_idx = data_disks - sector_div(stripe2, raid_disks);
2138                        *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2139                        break;
2140                case ALGORITHM_RIGHT_SYMMETRIC:
2141                        pd_idx = sector_div(stripe2, raid_disks);
2142                        *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2143                        break;
2144                case ALGORITHM_PARITY_0:
2145                        pd_idx = 0;
2146                        (*dd_idx)++;
2147                        break;
2148                case ALGORITHM_PARITY_N:
2149                        pd_idx = data_disks;
2150                        break;
2151                default:
2152                        BUG();
2153                }
2154                break;
2155        case 6:
2156
2157                switch (algorithm) {
2158                case ALGORITHM_LEFT_ASYMMETRIC:
2159                        pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2160                        qd_idx = pd_idx + 1;
2161                        if (pd_idx == raid_disks-1) {
2162                                (*dd_idx)++;    /* Q D D D P */
2163                                qd_idx = 0;
2164                        } else if (*dd_idx >= pd_idx)
2165                                (*dd_idx) += 2; /* D D P Q D */
2166                        break;
2167                case ALGORITHM_RIGHT_ASYMMETRIC:
2168                        pd_idx = sector_div(stripe2, raid_disks);
2169                        qd_idx = pd_idx + 1;
2170                        if (pd_idx == raid_disks-1) {
2171                                (*dd_idx)++;    /* Q D D D P */
2172                                qd_idx = 0;
2173                        } else if (*dd_idx >= pd_idx)
2174                                (*dd_idx) += 2; /* D D P Q D */
2175                        break;
2176                case ALGORITHM_LEFT_SYMMETRIC:
2177                        pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2178                        qd_idx = (pd_idx + 1) % raid_disks;
2179                        *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2180                        break;
2181                case ALGORITHM_RIGHT_SYMMETRIC:
2182                        pd_idx = sector_div(stripe2, raid_disks);
2183                        qd_idx = (pd_idx + 1) % raid_disks;
2184                        *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2185                        break;
2186
2187                case ALGORITHM_PARITY_0:
2188                        pd_idx = 0;
2189                        qd_idx = 1;
2190                        (*dd_idx) += 2;
2191                        break;
2192                case ALGORITHM_PARITY_N:
2193                        pd_idx = data_disks;
2194                        qd_idx = data_disks + 1;
2195                        break;
2196
2197                case ALGORITHM_ROTATING_ZERO_RESTART:
2198                        /* Exactly the same as RIGHT_ASYMMETRIC, but or
2199                         * of blocks for computing Q is different.
2200                         */
2201                        pd_idx = sector_div(stripe2, raid_disks);
2202                        qd_idx = pd_idx + 1;
2203                        if (pd_idx == raid_disks-1) {
2204                                (*dd_idx)++;    /* Q D D D P */
2205                                qd_idx = 0;
2206                        } else if (*dd_idx >= pd_idx)
2207                                (*dd_idx) += 2; /* D D P Q D */
2208                        ddf_layout = 1;
2209                        break;
2210
2211                case ALGORITHM_ROTATING_N_RESTART:
2212                        /* Same a left_asymmetric, by first stripe is
2213                         * D D D P Q  rather than
2214                         * Q D D D P
2215                         */
2216                        stripe2 += 1;
2217                        pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2218                        qd_idx = pd_idx + 1;
2219                        if (pd_idx == raid_disks-1) {
2220                                (*dd_idx)++;    /* Q D D D P */
2221                                qd_idx = 0;
2222                        } else if (*dd_idx >= pd_idx)
2223                                (*dd_idx) += 2; /* D D P Q D */
2224                        ddf_layout = 1;
2225                        break;
2226
2227                case ALGORITHM_ROTATING_N_CONTINUE:
2228                        /* Same as left_symmetric but Q is before P */
2229                        pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2230                        qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
2231                        *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2232                        ddf_layout = 1;
2233                        break;
2234
2235                case ALGORITHM_LEFT_ASYMMETRIC_6:
2236                        /* RAID5 left_asymmetric, with Q on last device */
2237                        pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2238                        if (*dd_idx >= pd_idx)
2239                                (*dd_idx)++;
2240                        qd_idx = raid_disks - 1;
2241                        break;
2242
2243                case ALGORITHM_RIGHT_ASYMMETRIC_6:
2244                        pd_idx = sector_div(stripe2, raid_disks-1);
2245                        if (*dd_idx >= pd_idx)
2246                                (*dd_idx)++;
2247                        qd_idx = raid_disks - 1;
2248                        break;
2249
2250                case ALGORITHM_LEFT_SYMMETRIC_6:
2251                        pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2252                        *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2253                        qd_idx = raid_disks - 1;
2254                        break;
2255
2256                case ALGORITHM_RIGHT_SYMMETRIC_6:
2257                        pd_idx = sector_div(stripe2, raid_disks-1);
2258                        *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2259                        qd_idx = raid_disks - 1;
2260                        break;
2261
2262                case ALGORITHM_PARITY_0_6:
2263                        pd_idx = 0;
2264                        (*dd_idx)++;
2265                        qd_idx = raid_disks - 1;
2266                        break;
2267
2268                default:
2269                        BUG();
2270                }
2271                break;
2272        }
2273
2274        if (sh) {
2275                sh->pd_idx = pd_idx;
2276                sh->qd_idx = qd_idx;
2277                sh->ddf_layout = ddf_layout;
2278        }
2279        /*
2280         * Finally, compute the new sector number
2281         */
2282        new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
2283        return new_sector;
2284}
2285
2286
2287static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
2288{
2289        struct r5conf *conf = sh->raid_conf;
2290        int raid_disks = sh->disks;
2291        int data_disks = raid_disks - conf->max_degraded;
2292        sector_t new_sector = sh->sector, check;
2293        int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2294                                         : conf->chunk_sectors;
2295        int algorithm = previous ? conf->prev_algo
2296                                 : conf->algorithm;
2297        sector_t stripe;
2298        int chunk_offset;
2299        sector_t chunk_number;
2300        int dummy1, dd_idx = i;
2301        sector_t r_sector;
2302        struct stripe_head sh2;
2303
2304
2305        chunk_offset = sector_div(new_sector, sectors_per_chunk);
2306        stripe = new_sector;
2307
2308        if (i == sh->pd_idx)
2309                return 0;
2310        switch(conf->level) {
2311        case 4: break;
2312        case 5:
2313                switch (algorithm) {
2314                case ALGORITHM_LEFT_ASYMMETRIC:
2315                case ALGORITHM_RIGHT_ASYMMETRIC:
2316                        if (i > sh->pd_idx)
2317                                i--;
2318                        break;
2319                case ALGORITHM_LEFT_SYMMETRIC:
2320                case ALGORITHM_RIGHT_SYMMETRIC:
2321                        if (i < sh->pd_idx)
2322                                i += raid_disks;
2323                        i -= (sh->pd_idx + 1);
2324                        break;
2325                case ALGORITHM_PARITY_0:
2326                        i -= 1;
2327                        break;
2328                case ALGORITHM_PARITY_N:
2329                        break;
2330                default:
2331                        BUG();
2332                }
2333                break;
2334        case 6:
2335                if (i == sh->qd_idx)
2336                        return 0; /* It is the Q disk */
2337                switch (algorithm) {
2338                case ALGORITHM_LEFT_ASYMMETRIC:
2339                case ALGORITHM_RIGHT_ASYMMETRIC:
2340                case ALGORITHM_ROTATING_ZERO_RESTART:
2341                case ALGORITHM_ROTATING_N_RESTART:
2342                        if (sh->pd_idx == raid_disks-1)
2343                                i--;    /* Q D D D P */
2344                        else if (i > sh->pd_idx)
2345                                i -= 2; /* D D P Q D */
2346                        break;
2347                case ALGORITHM_LEFT_SYMMETRIC:
2348                case ALGORITHM_RIGHT_SYMMETRIC:
2349                        if (sh->pd_idx == raid_disks-1)
2350                                i--; /* Q D D D P */
2351                        else {
2352                                /* D D P Q D */
2353                                if (i < sh->pd_idx)
2354                                        i += raid_disks;
2355                                i -= (sh->pd_idx + 2);
2356                        }
2357                        break;
2358                case ALGORITHM_PARITY_0:
2359                        i -= 2;
2360                        break;
2361                case ALGORITHM_PARITY_N:
2362                        break;
2363                case ALGORITHM_ROTATING_N_CONTINUE:
2364                        /* Like left_symmetric, but P is before Q */
2365                        if (sh->pd_idx == 0)
2366                                i--;    /* P D D D Q */
2367                        else {
2368                                /* D D Q P D */
2369                                if (i < sh->pd_idx)
2370                                        i += raid_disks;
2371                                i -= (sh->pd_idx + 1);
2372                        }
2373                        break;
2374                case ALGORITHM_LEFT_ASYMMETRIC_6:
2375                case ALGORITHM_RIGHT_ASYMMETRIC_6:
2376                        if (i > sh->pd_idx)
2377                                i--;
2378                        break;
2379                case ALGORITHM_LEFT_SYMMETRIC_6:
2380                case ALGORITHM_RIGHT_SYMMETRIC_6:
2381                        if (i < sh->pd_idx)
2382                                i += data_disks + 1;
2383                        i -= (sh->pd_idx + 1);
2384                        break;
2385                case ALGORITHM_PARITY_0_6:
2386                        i -= 1;
2387                        break;
2388                default:
2389                        BUG();
2390                }
2391                break;
2392        }
2393
2394        chunk_number = stripe * data_disks + i;
2395        r_sector = chunk_number * sectors_per_chunk + chunk_offset;
2396
2397        check = raid5_compute_sector(conf, r_sector,
2398                                     previous, &dummy1, &sh2);
2399        if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
2400                || sh2.qd_idx != sh->qd_idx) {
2401                printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
2402                       mdname(conf->mddev));
2403                return 0;
2404        }
2405        return r_sector;
2406}
2407
2408
2409static void
2410schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
2411                         int rcw, int expand)
2412{
2413        int i, pd_idx = sh->pd_idx, disks = sh->disks;
2414        struct r5conf *conf = sh->raid_conf;
2415        int level = conf->level;
2416
2417        if (rcw) {
2418
2419                for (i = disks; i--; ) {
2420                        struct r5dev *dev = &sh->dev[i];
2421
2422                        if (dev->towrite) {
2423                                set_bit(R5_LOCKED, &dev->flags);
2424                                set_bit(R5_Wantdrain, &dev->flags);
2425                                if (!expand)
2426                                        clear_bit(R5_UPTODATE, &dev->flags);
2427                                s->locked++;
2428                        }
2429                }
2430                /* if we are not expanding this is a proper write request, and
2431                 * there will be bios with new data to be drained into the
2432                 * stripe cache
2433                 */
2434                if (!expand) {
2435                        if (!s->locked)
2436                                /* False alarm, nothing to do */
2437                                return;
2438                        sh->reconstruct_state = reconstruct_state_drain_run;
2439                        set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2440                } else
2441                        sh->reconstruct_state = reconstruct_state_run;
2442
2443                set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2444
2445                if (s->locked + conf->max_degraded == disks)
2446                        if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2447                                atomic_inc(&conf->pending_full_writes);
2448        } else {
2449                BUG_ON(level == 6);
2450                BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
2451                        test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
2452
2453                for (i = disks; i--; ) {
2454                        struct r5dev *dev = &sh->dev[i];
2455                        if (i == pd_idx)
2456                                continue;
2457
2458                        if (dev->towrite &&
2459                            (test_bit(R5_UPTODATE, &dev->flags) ||
2460                             test_bit(R5_Wantcompute, &dev->flags))) {
2461                                set_bit(R5_Wantdrain, &dev->flags);
2462                                set_bit(R5_LOCKED, &dev->flags);
2463                                clear_bit(R5_UPTODATE, &dev->flags);
2464                                s->locked++;
2465                        }
2466                }
2467                if (!s->locked)
2468                        /* False alarm - nothing to do */
2469                        return;
2470                sh->reconstruct_state = reconstruct_state_prexor_drain_run;
2471                set_bit(STRIPE_OP_PREXOR, &s->ops_request);
2472                set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2473                set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2474        }
2475
2476        /* keep the parity disk(s) locked while asynchronous operations
2477         * are in flight
2478         */
2479        set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
2480        clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2481        s->locked++;
2482
2483        if (level == 6) {
2484                int qd_idx = sh->qd_idx;
2485                struct r5dev *dev = &sh->dev[qd_idx];
2486
2487                set_bit(R5_LOCKED, &dev->flags);
2488                clear_bit(R5_UPTODATE, &dev->flags);
2489                s->locked++;
2490        }
2491
2492        pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2493                __func__, (unsigned long long)sh->sector,
2494                s->locked, s->ops_request);
2495}
2496
2497/*
2498 * Each stripe/dev can have one or more bion attached.
2499 * toread/towrite point to the first in a chain.
2500 * The bi_next chain must be in order.
2501 */
2502static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
2503{
2504        struct bio **bip;
2505        struct r5conf *conf = sh->raid_conf;
2506        int firstwrite=0;
2507
2508        pr_debug("adding bi b#%llu to stripe s#%llu\n",
2509                (unsigned long long)bi->bi_sector,
2510                (unsigned long long)sh->sector);
2511
2512        /*
2513         * If several bio share a stripe. The bio bi_phys_segments acts as a
2514         * reference count to avoid race. The reference count should already be
2515         * increased before this function is called (for example, in
2516         * make_request()), so other bio sharing this stripe will not free the
2517         * stripe. If a stripe is owned by one stripe, the stripe lock will
2518         * protect it.
2519         */
2520        spin_lock_irq(&sh->stripe_lock);
2521        if (forwrite) {
2522                bip = &sh->dev[dd_idx].towrite;
2523                if (*bip == NULL)
2524                        firstwrite = 1;
2525        } else
2526                bip = &sh->dev[dd_idx].toread;
2527        while (*bip && (*bip)->bi_sector < bi->bi_sector) {
2528                if (bio_end_sector(*bip) > bi->bi_sector)
2529                        goto overlap;
2530                bip = & (*bip)->bi_next;
2531        }
2532        if (*bip && (*bip)->bi_sector < bio_end_sector(bi))
2533                goto overlap;
2534
2535        BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
2536        if (*bip)
2537                bi->bi_next = *bip;
2538        *bip = bi;
2539        raid5_inc_bi_active_stripes(bi);
2540
2541        if (forwrite) {
2542                /* check if page is covered */
2543                sector_t sector = sh->dev[dd_idx].sector;
2544                for (bi=sh->dev[dd_idx].towrite;
2545                     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
2546                             bi && bi->bi_sector <= sector;
2547                     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
2548                        if (bio_end_sector(bi) >= sector)
2549                                sector = bio_end_sector(bi);
2550                }
2551                if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
2552                        set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
2553        }
2554
2555        pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
2556                (unsigned long long)(*bip)->bi_sector,
2557                (unsigned long long)sh->sector, dd_idx);
2558        spin_unlock_irq(&sh->stripe_lock);
2559
2560        if (conf->mddev->bitmap && firstwrite) {
2561                bitmap_startwrite(conf->mddev->bitmap, sh->sector,
2562                                  STRIPE_SECTORS, 0);
2563                sh->bm_seq = conf->seq_flush+1;
2564                set_bit(STRIPE_BIT_DELAY, &sh->state);
2565        }
2566        return 1;
2567
2568 overlap:
2569        set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
2570        spin_unlock_irq(&sh->stripe_lock);
2571        return 0;
2572}
2573
2574static void end_reshape(struct r5conf *conf);
2575
2576static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
2577                            struct stripe_head *sh)
2578{
2579        int sectors_per_chunk =
2580                previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2581        int dd_idx;
2582        int chunk_offset = sector_div(stripe, sectors_per_chunk);
2583        int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2584
2585        raid5_compute_sector(conf,
2586                             stripe * (disks - conf->max_degraded)
2587                             *sectors_per_chunk + chunk_offset,
2588                             previous,
2589                             &dd_idx, sh);
2590}
2591
2592static void
2593handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
2594                                struct stripe_head_state *s, int disks,
2595                                struct bio **return_bi)
2596{
2597        int i;
2598        for (i = disks; i--; ) {
2599                struct bio *bi;
2600                int bitmap_end = 0;
2601
2602                if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2603                        struct md_rdev *rdev;
2604                        rcu_read_lock();
2605                        rdev = rcu_dereference(conf->disks[i].rdev);
2606                        if (rdev && test_bit(In_sync, &rdev->flags))
2607                                atomic_inc(&rdev->nr_pending);
2608                        else
2609                                rdev = NULL;
2610                        rcu_read_unlock();
2611                        if (rdev) {
2612                                if (!rdev_set_badblocks(
2613                                            rdev,
2614                                            sh->sector,
2615                                            STRIPE_SECTORS, 0))
2616                                        md_error(conf->mddev, rdev);
2617                                rdev_dec_pending(rdev, conf->mddev);
2618                        }
2619                }
2620                spin_lock_irq(&sh->stripe_lock);
2621                /* fail all writes first */
2622                bi = sh->dev[i].towrite;
2623                sh->dev[i].towrite = NULL;
2624                spin_unlock_irq(&sh->stripe_lock);
2625                if (bi)
2626                        bitmap_end = 1;
2627
2628                if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2629                        wake_up(&conf->wait_for_overlap);
2630
2631                while (bi && bi->bi_sector <
2632                        sh->dev[i].sector + STRIPE_SECTORS) {
2633                        struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
2634                        clear_bit(BIO_UPTODATE, &bi->bi_flags);
2635                        if (!raid5_dec_bi_active_stripes(bi)) {
2636                                md_write_end(conf->mddev);
2637                                bi->bi_next = *return_bi;
2638                                *return_bi = bi;
2639                        }
2640                        bi = nextbi;
2641                }
2642                if (bitmap_end)
2643                        bitmap_endwrite(conf->mddev->bitmap, sh->sector,
2644                                STRIPE_SECTORS, 0, 0);
2645                bitmap_end = 0;
2646                /* and fail all 'written' */
2647                bi = sh->dev[i].written;
2648                sh->dev[i].written = NULL;
2649                if (bi) bitmap_end = 1;
2650                while (bi && bi->bi_sector <
2651                       sh->dev[i].sector + STRIPE_SECTORS) {
2652                        struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
2653                        clear_bit(BIO_UPTODATE, &bi->bi_flags);
2654                        if (!raid5_dec_bi_active_stripes(bi)) {
2655                                md_write_end(conf->mddev);
2656                                bi->bi_next = *return_bi;
2657                                *return_bi = bi;
2658                        }
2659                        bi = bi2;
2660                }
2661
2662                /* fail any reads if this device is non-operational and
2663                 * the data has not reached the cache yet.
2664                 */
2665                if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
2666                    (!test_bit(R5_Insync, &sh->dev[i].flags) ||
2667                      test_bit(R5_ReadError, &sh->dev[i].flags))) {
2668                        spin_lock_irq(&sh->stripe_lock);
2669                        bi = sh->dev[i].toread;
2670                        sh->dev[i].toread = NULL;
2671                        spin_unlock_irq(&sh->stripe_lock);
2672                        if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2673                                wake_up(&conf->wait_for_overlap);
2674                        while (bi && bi->bi_sector <
2675                               sh->dev[i].sector + STRIPE_SECTORS) {
2676                                struct bio *nextbi =
2677                                        r5_next_bio(bi, sh->dev[i].sector);
2678                                clear_bit(BIO_UPTODATE, &bi->bi_flags);
2679                                if (!raid5_dec_bi_active_stripes(bi)) {
2680                                        bi->bi_next = *return_bi;
2681                                        *return_bi = bi;
2682                                }
2683                                bi = nextbi;
2684                        }
2685                }
2686                if (bitmap_end)
2687                        bitmap_endwrite(conf->mddev->bitmap, sh->sector,
2688                                        STRIPE_SECTORS, 0, 0);
2689                /* If we were in the middle of a write the parity block might
2690                 * still be locked - so just clear all R5_LOCKED flags
2691                 */
2692                clear_bit(R5_LOCKED, &sh->dev[i].flags);
2693        }
2694
2695        if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2696                if (atomic_dec_and_test(&conf->pending_full_writes))
2697                        md_wakeup_thread(conf->mddev->thread);
2698}
2699
2700static void
2701handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
2702                   struct stripe_head_state *s)
2703{
2704        int abort = 0;
2705        int i;
2706
2707        clear_bit(STRIPE_SYNCING, &sh->state);
2708        if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
2709                wake_up(&conf->wait_for_overlap);
2710        s->syncing = 0;
2711        s->replacing = 0;
2712        /* There is nothing more to do for sync/check/repair.
2713         * Don't even need to abort as that is handled elsewhere
2714         * if needed, and not always wanted e.g. if there is a known
2715         * bad block here.
2716         * For recover/replace we need to record a bad block on all
2717         * non-sync devices, or abort the recovery
2718         */
2719        if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
2720                /* During recovery devices cannot be removed, so
2721                 * locking and refcounting of rdevs is not needed
2722                 */
2723                for (i = 0; i < conf->raid_disks; i++) {
2724                        struct md_rdev *rdev = conf->disks[i].rdev;
2725                        if (rdev
2726                            && !test_bit(Faulty, &rdev->flags)
2727                            && !test_bit(In_sync, &rdev->flags)
2728                            && !rdev_set_badblocks(rdev, sh->sector,
2729                                                   STRIPE_SECTORS, 0))
2730                                abort = 1;
2731                        rdev = conf->disks[i].replacement;
2732                        if (rdev
2733                            && !test_bit(Faulty, &rdev->flags)
2734                            && !test_bit(In_sync, &rdev->flags)
2735                            && !rdev_set_badblocks(rdev, sh->sector,
2736                                                   STRIPE_SECTORS, 0))
2737                                abort = 1;
2738                }
2739                if (abort)
2740                        conf->recovery_disabled =
2741                                conf->mddev->recovery_disabled;
2742        }
2743        md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
2744}
2745
2746static int want_replace(struct stripe_head *sh, int disk_idx)
2747{
2748        struct md_rdev *rdev;
2749        int rv = 0;
2750        /* Doing recovery so rcu locking not required */
2751        rdev = sh->raid_conf->disks[disk_idx].replacement;
2752        if (rdev
2753            && !test_bit(Faulty, &rdev->flags)
2754            && !test_bit(In_sync, &rdev->flags)
2755            && (rdev->recovery_offset <= sh->sector
2756                || rdev->mddev->recovery_cp <= sh->sector))
2757                rv = 1;
2758
2759        return rv;
2760}
2761
2762/* fetch_block - checks the given member device to see if its data needs
2763 * to be read or computed to satisfy a request.
2764 *
2765 * Returns 1 when no more member devices need to be checked, otherwise returns
2766 * 0 to tell the loop in handle_stripe_fill to continue
2767 */
2768static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
2769                       int disk_idx, int disks)
2770{
2771        struct r5dev *dev = &sh->dev[disk_idx];
2772        struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
2773                                  &sh->dev[s->failed_num[1]] };
2774
2775        /* is the data in this block needed, and can we get it? */
2776        if (!test_bit(R5_LOCKED, &dev->flags) &&
2777            !test_bit(R5_UPTODATE, &dev->flags) &&
2778            (dev->toread ||
2779             (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
2780             s->syncing || s->expanding ||
2781             (s->replacing && want_replace(sh, disk_idx)) ||
2782             (s->failed >= 1 && fdev[0]->toread) ||
2783             (s->failed >= 2 && fdev[1]->toread) ||
2784             (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite &&
2785              !test_bit(R5_OVERWRITE, &fdev[0]->flags)) ||
2786             (sh->raid_conf->level == 6 && s->failed && s->to_write))) {
2787                /* we would like to get this block, possibly by computing it,
2788                 * otherwise read it if the backing disk is insync
2789                 */
2790                BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
2791                BUG_ON(test_bit(R5_Wantread, &dev->flags));
2792                if ((s->uptodate == disks - 1) &&
2793                    (s->failed && (disk_idx == s->failed_num[0] ||
2794                                   disk_idx == s->failed_num[1]))) {
2795                        /* have disk failed, and we're requested to fetch it;
2796                         * do compute it
2797                         */
2798                        pr_debug("Computing stripe %llu block %d\n",
2799                               (unsigned long long)sh->sector, disk_idx);
2800                        set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2801                        set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2802                        set_bit(R5_Wantcompute, &dev->flags);
2803                        sh->ops.target = disk_idx;
2804                        sh->ops.target2 = -1; /* no 2nd target */
2805                        s->req_compute = 1;
2806                        /* Careful: from this point on 'uptodate' is in the eye
2807                         * of raid_run_ops which services 'compute' operations
2808                         * before writes. R5_Wantcompute flags a block that will
2809                         * be R5_UPTODATE by the time it is needed for a
2810                         * subsequent operation.
2811                         */
2812                        s->uptodate++;
2813                        return 1;
2814                } else if (s->uptodate == disks-2 && s->failed >= 2) {
2815                        /* Computing 2-failure is *very* expensive; only
2816                         * do it if failed >= 2
2817                         */
2818                        int other;
2819                        for (other = disks; other--; ) {
2820                                if (other == disk_idx)
2821                                        continue;
2822                                if (!test_bit(R5_UPTODATE,
2823                                      &sh->dev[other].flags))
2824                                        break;
2825                        }
2826                        BUG_ON(other < 0);
2827                        pr_debug("Computing stripe %llu blocks %d,%d\n",
2828                               (unsigned long long)sh->sector,
2829                               disk_idx, other);
2830                        set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2831                        set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2832                        set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
2833                        set_bit(R5_Wantcompute, &sh->dev[other].flags);
2834                        sh->ops.target = disk_idx;
2835                        sh->ops.target2 = other;
2836                        s->uptodate += 2;
2837                        s->req_compute = 1;
2838                        return 1;
2839                } else if (test_bit(R5_Insync, &dev->flags)) {
2840                        set_bit(R5_LOCKED, &dev->flags);
2841                        set_bit(R5_Wantread, &dev->flags);
2842                        s->locked++;
2843                        pr_debug("Reading block %d (sync=%d)\n",
2844                                disk_idx, s->syncing);
2845                }
2846        }
2847
2848        return 0;
2849}
2850
2851/**
2852 * handle_stripe_fill - read or compute data to satisfy pending requests.
2853 */
2854static void handle_stripe_fill(struct stripe_head *sh,
2855                               struct stripe_head_state *s,
2856                               int disks)
2857{
2858        int i;
2859
2860        /* look for blocks to read/compute, skip this if a compute
2861         * is already in flight, or if the stripe contents are in the
2862         * midst of changing due to a write
2863         */
2864        if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
2865            !sh->reconstruct_state)
2866                for (i = disks; i--; )
2867                        if (fetch_block(sh, s, i, disks))
2868                                break;
2869        set_bit(STRIPE_HANDLE, &sh->state);
2870}
2871
2872
2873/* handle_stripe_clean_event
2874 * any written block on an uptodate or failed drive can be returned.
2875 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
2876 * never LOCKED, so we don't need to test 'failed' directly.
2877 */
2878static void handle_stripe_clean_event(struct r5conf *conf,
2879        struct stripe_head *sh, int disks, struct bio **return_bi)
2880{
2881        int i;
2882        struct r5dev *dev;
2883        int discard_pending = 0;
2884
2885        for (i = disks; i--; )
2886                if (sh->dev[i].written) {
2887                        dev = &sh->dev[i];
2888                        if (!test_bit(R5_LOCKED, &dev->flags) &&
2889                            (test_bit(R5_UPTODATE, &dev->flags) ||
2890                             test_bit(R5_Discard, &dev->flags))) {
2891                                /* We can return any write requests */
2892                                struct bio *wbi, *wbi2;
2893                                pr_debug("Return write for disc %d\n", i);
2894                                if (test_and_clear_bit(R5_Discard, &dev->flags))
2895                                        clear_bit(R5_UPTODATE, &dev->flags);
2896                                wbi = dev->written;
2897                                dev->written = NULL;
2898                                while (wbi && wbi->bi_sector <
2899                                        dev->sector + STRIPE_SECTORS) {
2900                                        wbi2 = r5_next_bio(wbi, dev->sector);
2901                                        if (!raid5_dec_bi_active_stripes(wbi)) {
2902                                                md_write_end(conf->mddev);
2903                                                wbi->bi_next = *return_bi;
2904                                                *return_bi = wbi;
2905                                        }
2906                                        wbi = wbi2;
2907                                }
2908                                bitmap_endwrite(conf->mddev->bitmap, sh->sector,
2909                                                STRIPE_SECTORS,
2910                                         !test_bit(STRIPE_DEGRADED, &sh->state),
2911                                                0);
2912                        } else if (test_bit(R5_Discard, &dev->flags))
2913                                discard_pending = 1;
2914                }
2915        if (!discard_pending &&
2916            test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
2917                clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
2918                clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
2919                if (sh->qd_idx >= 0) {
2920                        clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
2921                        clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
2922                }
2923                /* now that discard is done we can proceed with any sync */
2924                clear_bit(STRIPE_DISCARD, &sh->state);
2925                /*
2926                 * SCSI discard will change some bio fields and the stripe has
2927                 * no updated data, so remove it from hash list and the stripe
2928                 * will be reinitialized
2929                 */
2930                spin_lock_irq(&conf->device_lock);
2931                remove_hash(sh);
2932                spin_unlock_irq(&conf->device_lock);
2933                if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
2934                        set_bit(STRIPE_HANDLE, &sh->state);
2935
2936        }
2937
2938        if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2939                if (atomic_dec_and_test(&conf->pending_full_writes))
2940                        md_wakeup_thread(conf->mddev->thread);
2941}
2942
2943static void handle_stripe_dirtying(struct r5conf *conf,
2944                                   struct stripe_head *sh,
2945                                   struct stripe_head_state *s,
2946                                   int disks)
2947{
2948        int rmw = 0, rcw = 0, i;
2949        sector_t recovery_cp = conf->mddev->recovery_cp;
2950
2951        /* RAID6 requires 'rcw' in current implementation.
2952         * Otherwise, check whether resync is now happening or should start.
2953         * If yes, then the array is dirty (after unclean shutdown or
2954         * initial creation), so parity in some stripes might be inconsistent.
2955         * In this case, we need to always do reconstruct-write, to ensure
2956         * that in case of drive failure or read-error correction, we
2957         * generate correct data from the parity.
2958         */
2959        if (conf->max_degraded == 2 ||
2960            (recovery_cp < MaxSector && sh->sector >= recovery_cp)) {
2961                /* Calculate the real rcw later - for now make it
2962                 * look like rcw is cheaper
2963                 */
2964                rcw = 1; rmw = 2;
2965                pr_debug("force RCW max_degraded=%u, recovery_cp=%llu sh->sector=%llu\n",
2966                         conf->max_degraded, (unsigned long long)recovery_cp,
2967                         (unsigned long long)sh->sector);
2968        } else for (i = disks; i--; ) {
2969                /* would I have to read this buffer for read_modify_write */
2970                struct r5dev *dev = &sh->dev[i];
2971                if ((dev->towrite || i == sh->pd_idx) &&
2972                    !test_bit(R5_LOCKED, &dev->flags) &&
2973                    !(test_bit(R5_UPTODATE, &dev->flags) ||
2974                      test_bit(R5_Wantcompute, &dev->flags))) {
2975                        if (test_bit(R5_Insync, &dev->flags))
2976                                rmw++;
2977                        else
2978                                rmw += 2*disks;  /* cannot read it */
2979                }
2980                /* Would I have to read this buffer for reconstruct_write */
2981                if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
2982                    !test_bit(R5_LOCKED, &dev->flags) &&
2983                    !(test_bit(R5_UPTODATE, &dev->flags) ||
2984                    test_bit(R5_Wantcompute, &dev->flags))) {
2985                        if (test_bit(R5_Insync, &dev->flags)) rcw++;
2986                        else
2987                                rcw += 2*disks;
2988                }
2989        }
2990        pr_debug("for sector %llu, rmw=%d rcw=%d\n",
2991                (unsigned long long)sh->sector, rmw, rcw);
2992        set_bit(STRIPE_HANDLE, &sh->state);
2993        if (rmw < rcw && rmw > 0) {
2994                /* prefer read-modify-write, but need to get some data */
2995                if (conf->mddev->queue)
2996                        blk_add_trace_msg(conf->mddev->queue,
2997                                          "raid5 rmw %llu %d",
2998                                          (unsigned long long)sh->sector, rmw);
2999                for (i = disks; i--; ) {
3000                        struct r5dev *dev = &sh->dev[i];
3001                        if ((dev->towrite || i == sh->pd_idx) &&
3002                            !test_bit(R5_LOCKED, &dev->flags) &&
3003                            !(test_bit(R5_UPTODATE, &dev->flags) ||
3004                            test_bit(R5_Wantcompute, &dev->flags)) &&
3005                            test_bit(R5_Insync, &dev->flags)) {
3006                                if (
3007                                  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
3008                                        pr_debug("Read_old block "
3009                                                 "%d for r-m-w\n", i);
3010                                        set_bit(R5_LOCKED, &dev->flags);
3011                                        set_bit(R5_Wantread, &dev->flags);
3012                                        s->locked++;
3013                                } else {
3014                                        set_bit(STRIPE_DELAYED, &sh->state);
3015                                        set_bit(STRIPE_HANDLE, &sh->state);
3016                                }
3017                        }
3018                }
3019        }
3020        if (rcw <= rmw && rcw > 0) {
3021                /* want reconstruct write, but need to get some data */
3022                int qread =0;
3023                rcw = 0;
3024                for (i = disks; i--; ) {
3025                        struct r5dev *dev = &sh->dev[i];
3026                        if (!test_bit(R5_OVERWRITE, &dev->flags) &&
3027                            i != sh->pd_idx && i != sh->qd_idx &&
3028                            !test_bit(R5_LOCKED, &dev->flags) &&
3029                            !(test_bit(R5_UPTODATE, &dev->flags) ||
3030                              test_bit(R5_Wantcompute, &dev->flags))) {
3031                                rcw++;
3032                                if (!test_bit(R5_Insync, &dev->flags))
3033                                        continue; /* it's a failed drive */
3034                                if (
3035                                  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
3036                                        pr_debug("Read_old block "
3037                                                "%d for Reconstruct\n", i);
3038                                        set_bit(R5_LOCKED, &dev->flags);
3039                                        set_bit(R5_Wantread, &dev->flags);
3040                                        s->locked++;
3041                                        qread++;
3042                                } else {
3043                                        set_bit(STRIPE_DELAYED, &sh->state);
3044                                        set_bit(STRIPE_HANDLE, &sh->state);
3045                                }
3046                        }
3047                }
3048                if (rcw && conf->mddev->queue)
3049                        blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
3050                                          (unsigned long long)sh->sector,
3051                                          rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
3052        }
3053        /* now if nothing is locked, and if we have enough data,
3054         * we can start a write request
3055         */
3056        /* since handle_stripe can be called at any time we need to handle the
3057         * case where a compute block operation has been submitted and then a
3058         * subsequent call wants to start a write request.  raid_run_ops only
3059         * handles the case where compute block and reconstruct are requested
3060         * simultaneously.  If this is not the case then new writes need to be
3061         * held off until the compute completes.
3062         */
3063        if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
3064            (s->locked == 0 && (rcw == 0 || rmw == 0) &&
3065            !test_bit(STRIPE_BIT_DELAY, &sh->state)))
3066                schedule_reconstruction(sh, s, rcw == 0, 0);
3067}
3068
3069static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
3070                                struct stripe_head_state *s, int disks)
3071{
3072        struct r5dev *dev = NULL;
3073
3074        set_bit(STRIPE_HANDLE, &sh->state);
3075
3076        switch (sh->check_state) {
3077        case check_state_idle:
3078                /* start a new check operation if there are no failures */
3079                if (s->failed == 0) {
3080                        BUG_ON(s->uptodate != disks);
3081                        sh->check_state = check_state_run;
3082                        set_bit(STRIPE_OP_CHECK, &s->ops_request);
3083                        clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
3084                        s->uptodate--;
3085                        break;
3086                }
3087                dev = &sh->dev[s->failed_num[0]];
3088                /* fall through */
3089        case check_state_compute_result:
3090                sh->check_state = check_state_idle;
3091                if (!dev)
3092                        dev = &sh->dev[sh->pd_idx];
3093
3094                /* check that a write has not made the stripe insync */
3095                if (test_bit(STRIPE_INSYNC, &sh->state))
3096                        break;
3097
3098                /* either failed parity check, or recovery is happening */
3099                BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
3100                BUG_ON(s->uptodate != disks);
3101
3102                set_bit(R5_LOCKED, &dev->flags);
3103                s->locked++;
3104                set_bit(R5_Wantwrite, &dev->flags);
3105
3106                clear_bit(STRIPE_DEGRADED, &sh->state);
3107                set_bit(STRIPE_INSYNC, &sh->state);
3108                break;
3109        case check_state_run:
3110                break; /* we will be called again upon completion */
3111        case check_state_check_result:
3112                sh->check_state = check_state_idle;
3113
3114                /* if a failure occurred during the check operation, leave
3115                 * STRIPE_INSYNC not set and let the stripe be handled again
3116                 */
3117                if (s->failed)
3118                        break;
3119
3120                /* handle a successful check operation, if parity is correct
3121                 * we are done.  Otherwise update the mismatch count and repair
3122                 * parity if !MD_RECOVERY_CHECK
3123                 */
3124                if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
3125                        /* parity is correct (on disc,
3126                         * not in buffer any more)
3127                         */
3128                        set_bit(STRIPE_INSYNC, &sh->state);
3129                else {
3130                        atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3131                        if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
3132                                /* don't try to repair!! */
3133                                set_bit(STRIPE_INSYNC, &sh->state);
3134                        else {
3135                                sh->check_state = check_state_compute_run;
3136                                set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3137                                set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3138                                set_bit(R5_Wantcompute,
3139                                        &sh->dev[sh->pd_idx].flags);
3140                                sh->ops.target = sh->pd_idx;
3141                                sh->ops.target2 = -1;
3142                                s->uptodate++;
3143                        }
3144                }
3145                break;
3146        case check_state_compute_run:
3147                break;
3148        default:
3149                printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3150                       __func__, sh->check_state,
3151                       (unsigned long long) sh->sector);
3152                BUG();
3153        }
3154}
3155
3156
3157static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
3158                                  struct stripe_head_state *s,
3159                                  int disks)
3160{
3161        int pd_idx = sh->pd_idx;
3162        int qd_idx = sh->qd_idx;
3163        struct r5dev *dev;
3164
3165        set_bit(STRIPE_HANDLE, &sh->state);
3166
3167        BUG_ON(s->failed > 2);
3168
3169        /* Want to check and possibly repair P and Q.
3170         * However there could be one 'failed' device, in which
3171         * case we can only check one of them, possibly using the
3172         * other to generate missing data
3173         */
3174
3175        switch (sh->check_state) {
3176        case check_state_idle:
3177                /* start a new check operation if there are < 2 failures */
3178                if (s->failed == s->q_failed) {
3179                        /* The only possible failed device holds Q, so it
3180                         * makes sense to check P (If anything else were failed,
3181                         * we would have used P to recreate it).
3182                         */
3183                        sh->check_state = check_state_run;
3184                }
3185                if (!s->q_failed && s->failed < 2) {
3186                        /* Q is not failed, and we didn't use it to generate
3187                         * anything, so it makes sense to check it
3188                         */
3189                        if (sh->check_state == check_state_run)
3190                                sh->check_state = check_state_run_pq;
3191                        else
3192                                sh->check_state = check_state_run_q;
3193                }
3194
3195                /* discard potentially stale zero_sum_result */
3196                sh->ops.zero_sum_result = 0;
3197
3198                if (sh->check_state == check_state_run) {
3199                        /* async_xor_zero_sum destroys the contents of P */
3200                        clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
3201                        s->uptodate--;
3202                }
3203                if (sh->check_state >= check_state_run &&
3204                    sh->check_state <= check_state_run_pq) {
3205                        /* async_syndrome_zero_sum preserves P and Q, so
3206                         * no need to mark them !uptodate here
3207                         */
3208                        set_bit(STRIPE_OP_CHECK, &s->ops_request);
3209                        break;
3210                }
3211
3212                /* we have 2-disk failure */
3213                BUG_ON(s->failed != 2);
3214                /* fall through */
3215        case check_state_compute_result:
3216                sh->check_state = check_state_idle;
3217
3218                /* check that a write has not made the stripe insync */
3219                if (test_bit(STRIPE_INSYNC, &sh->state))
3220                        break;
3221
3222                /* now write out any block on a failed drive,
3223                 * or P or Q if they were recomputed
3224                 */
3225                BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
3226                if (s->failed == 2) {
3227                        dev = &sh->dev[s->failed_num[1]];
3228                        s->locked++;
3229                        set_bit(R5_LOCKED, &dev->flags);
3230                        set_bit(R5_Wantwrite, &dev->flags);
3231                }
3232                if (s->failed >= 1) {
3233                        dev = &sh->dev[s->failed_num[0]];
3234                        s->locked++;
3235                        set_bit(R5_LOCKED, &dev->flags);
3236                        set_bit(R5_Wantwrite, &dev->flags);
3237                }
3238                if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3239                        dev = &sh->dev[pd_idx];
3240                        s->locked++;
3241                        set_bit(R5_LOCKED, &dev->flags);
3242                        set_bit(R5_Wantwrite, &dev->flags);
3243                }
3244                if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3245                        dev = &sh->dev[qd_idx];
3246                        s->locked++;
3247                        set_bit(R5_LOCKED, &dev->flags);
3248                        set_bit(R5_Wantwrite, &dev->flags);
3249                }
3250                clear_bit(STRIPE_DEGRADED, &sh->state);
3251
3252                set_bit(STRIPE_INSYNC, &sh->state);
3253                break;
3254        case check_state_run:
3255        case check_state_run_q:
3256        case check_state_run_pq:
3257                break; /* we will be called again upon completion */
3258        case check_state_check_result:
3259                sh->check_state = check_state_idle;
3260
3261                /* handle a successful check operation, if parity is correct
3262                 * we are done.  Otherwise update the mismatch count and repair
3263                 * parity if !MD_RECOVERY_CHECK
3264                 */
3265                if (sh->ops.zero_sum_result == 0) {
3266                        /* both parities are correct */
3267                        if (!s->failed)
3268                                set_bit(STRIPE_INSYNC, &sh->state);
3269                        else {
3270                                /* in contrast to the raid5 case we can validate
3271                                 * parity, but still have a failure to write
3272                                 * back
3273                                 */
3274                                sh->check_state = check_state_compute_result;
3275                                /* Returning at this point means that we may go
3276                                 * off and bring p and/or q uptodate again so
3277                                 * we make sure to check zero_sum_result again
3278                                 * to verify if p or q need writeback
3279                                 */
3280                        }
3281                } else {
3282                        atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3283                        if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
3284                                /* don't try to repair!! */
3285                                set_bit(STRIPE_INSYNC, &sh->state);
3286                        else {
3287                                int *target = &sh->ops.target;
3288
3289                                sh->ops.target = -1;
3290                                sh->ops.target2 = -1;
3291                                sh->check_state = check_state_compute_run;
3292                                set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3293                                set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3294                                if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3295                                        set_bit(R5_Wantcompute,
3296                                                &sh->dev[pd_idx].flags);
3297                                        *target = pd_idx;
3298                                        target = &sh->ops.target2;
3299                                        s->uptodate++;
3300                                }
3301                                if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3302                                        set_bit(R5_Wantcompute,
3303                                                &sh->dev[qd_idx].flags);
3304                                        *target = qd_idx;
3305                                        s->uptodate++;
3306                                }
3307                        }
3308                }
3309                break;
3310        case check_state_compute_run:
3311                break;
3312        default:
3313                printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3314                       __func__, sh->check_state,
3315                       (unsigned long long) sh->sector);
3316                BUG();
3317        }
3318}
3319
3320static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
3321{
3322        int i;
3323
3324        /* We have read all the blocks in this stripe and now we need to
3325         * copy some of them into a target stripe for expand.
3326         */
3327        struct dma_async_tx_descriptor *tx = NULL;
3328        clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3329        for (i = 0; i < sh->disks; i++)
3330                if (i != sh->pd_idx && i != sh->qd_idx) {
3331                        int dd_idx, j;
3332                        struct stripe_head *sh2;
3333                        struct async_submit_ctl submit;
3334
3335                        sector_t bn = compute_blocknr(sh, i, 1);
3336                        sector_t s = raid5_compute_sector(conf, bn, 0,
3337                                                          &dd_idx, NULL);
3338                        sh2 = get_active_stripe(conf, s, 0, 1, 1);
3339                        if (sh2 == NULL)
3340                                /* so far only the early blocks of this stripe
3341                                 * have been requested.  When later blocks
3342                                 * get requested, we will try again
3343                                 */
3344                                continue;
3345                        if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
3346                           test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
3347                                /* must have already done this block */
3348                                release_stripe(sh2);
3349                                continue;
3350                        }
3351
3352                        /* place all the copies on one channel */
3353                        init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
3354                        tx = async_memcpy(sh2->dev[dd_idx].page,
3355                                          sh->dev[i].page, 0, 0, STRIPE_SIZE,
3356                                          &submit);
3357
3358                        set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
3359                        set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
3360                        for (j = 0; j < conf->raid_disks; j++)
3361                                if (j != sh2->pd_idx &&
3362                                    j != sh2->qd_idx &&
3363                                    !test_bit(R5_Expanded, &sh2->dev[j].flags))
3364                                        break;
3365                        if (j == conf->raid_disks) {
3366                                set_bit(STRIPE_EXPAND_READY, &sh2->state);
3367                                set_bit(STRIPE_HANDLE, &sh2->state);
3368                        }
3369                        release_stripe(sh2);
3370
3371                }
3372        /* done submitting copies, wait for them to complete */
3373        async_tx_quiesce(&tx);
3374}
3375
3376/*
3377 * handle_stripe - do things to a stripe.
3378 *
3379 * We lock the stripe by setting STRIPE_ACTIVE and then examine the
3380 * state of various bits to see what needs to be done.
3381 * Possible results:
3382 *    return some read requests which now have data
3383 *    return some write requests which are safely on storage
3384 *    schedule a read on some buffers
3385 *    schedule a write of some buffers
3386 *    return confirmation of parity correctness
3387 *
3388 */
3389
3390static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
3391{
3392        struct r5conf *conf = sh->raid_conf;
3393        int disks = sh->disks;
3394        struct r5dev *dev;
3395        int i;
3396        int do_recovery = 0;
3397
3398        memset(s, 0, sizeof(*s));
3399
3400        s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3401        s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
3402        s->failed_num[0] = -1;
3403        s->failed_num[1] = -1;
3404
3405        /* Now to look around and see what can be done */
3406        rcu_read_lock();
3407        for (i=disks; i--; ) {
3408                struct md_rdev *rdev;
3409                sector_t first_bad;
3410                int bad_sectors;
3411                int is_bad = 0;
3412
3413                dev = &sh->dev[i];
3414
3415                pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
3416                         i, dev->flags,
3417                         dev->toread, dev->towrite, dev->written);
3418                /* maybe we can reply to a read
3419                 *
3420                 * new wantfill requests are only permitted while
3421                 * ops_complete_biofill is guaranteed to be inactive
3422                 */
3423                if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
3424                    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
3425                        set_bit(R5_Wantfill, &dev->flags);
3426
3427                /* now count some things */
3428                if (test_bit(R5_LOCKED, &dev->flags))
3429                        s->locked++;
3430                if (test_bit(R5_UPTODATE, &dev->flags))
3431                        s->uptodate++;
3432                if (test_bit(R5_Wantcompute, &dev->flags)) {
3433                        s->compute++;
3434                        BUG_ON(s->compute > 2);
3435                }
3436
3437                if (test_bit(R5_Wantfill, &dev->flags))
3438                        s->to_fill++;
3439                else if (dev->toread)
3440                        s->to_read++;
3441                if (dev->towrite) {
3442                        s->to_write++;
3443                        if (!test_bit(R5_OVERWRITE, &dev->flags))
3444                                s->non_overwrite++;
3445                }
3446                if (dev->written)
3447                        s->written++;
3448                /* Prefer to use the replacement for reads, but only
3449                 * if it is recovered enough and has no bad blocks.
3450                 */
3451                rdev = rcu_dereference(conf->disks[i].replacement);
3452                if (rdev && !test_bit(Faulty, &rdev->flags) &&
3453                    rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
3454                    !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
3455                                 &first_bad, &bad_sectors))
3456                        set_bit(R5_ReadRepl, &dev->flags);
3457                else {
3458                        if (rdev)
3459                                set_bit(R5_NeedReplace, &dev->flags);
3460                        rdev = rcu_dereference(conf->disks[i].rdev);
3461                        clear_bit(R5_ReadRepl, &dev->flags);
3462                }
3463                if (rdev && test_bit(Faulty, &rdev->flags))
3464                        rdev = NULL;
3465                if (rdev) {
3466                        is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
3467                                             &first_bad, &bad_sectors);
3468                        if (s->blocked_rdev == NULL
3469                            && (test_bit(Blocked, &rdev->flags)
3470                                || is_bad < 0)) {
3471                                if (is_bad < 0)
3472                                        set_bit(BlockedBadBlocks,
3473                                                &rdev->flags);
3474                                s->blocked_rdev = rdev;
3475                                atomic_inc(&rdev->nr_pending);
3476                        }
3477                }
3478                clear_bit(R5_Insync, &dev->flags);
3479                if (!rdev)
3480                        /* Not in-sync */;
3481                else if (is_bad) {
3482                        /* also not in-sync */
3483                        if (!test_bit(WriteErrorSeen, &rdev->flags) &&
3484                            test_bit(R5_UPTODATE, &dev->flags)) {
3485                                /* treat as in-sync, but with a read error
3486                                 * which we can now try to correct
3487                                 */
3488                                set_bit(R5_Insync, &dev->flags);
3489                                set_bit(R5_ReadError, &dev->flags);
3490                        }
3491                } else if (test_bit(In_sync, &rdev->flags))
3492                        set_bit(R5_Insync, &dev->flags);
3493                else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
3494                        /* in sync if before recovery_offset */
3495                        set_bit(R5_Insync, &dev->flags);
3496                else if (test_bit(R5_UPTODATE, &dev->flags) &&
3497                         test_bit(R5_Expanded, &dev->flags))
3498                        /* If we've reshaped into here, we assume it is Insync.
3499                         * We will shortly update recovery_offset to make
3500                         * it official.
3501                         */
3502                        set_bit(R5_Insync, &dev->flags);
3503
3504                if (rdev && test_bit(R5_WriteError, &dev->flags)) {
3505                        /* This flag does not apply to '.replacement'
3506                         * only to .rdev, so make sure to check that*/
3507                        struct md_rdev *rdev2 = rcu_dereference(
3508                                conf->disks[i].rdev);
3509                        if (rdev2 == rdev)
3510                                clear_bit(R5_Insync, &dev->flags);
3511                        if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3512                                s->handle_bad_blocks = 1;
3513                                atomic_inc(&rdev2->nr_pending);
3514                        } else
3515                                clear_bit(R5_WriteError, &dev->flags);
3516                }
3517                if (rdev && test_bit(R5_MadeGood, &dev->flags)) {
3518                        /* This flag does not apply to '.replacement'
3519                         * only to .rdev, so make sure to check that*/
3520                        struct md_rdev *rdev2 = rcu_dereference(
3521                                conf->disks[i].rdev);
3522                        if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3523                                s->handle_bad_blocks = 1;
3524                                atomic_inc(&rdev2->nr_pending);
3525                        } else
3526                                clear_bit(R5_MadeGood, &dev->flags);
3527                }
3528                if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
3529                        struct md_rdev *rdev2 = rcu_dereference(
3530                                conf->disks[i].replacement);
3531                        if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3532                                s->handle_bad_blocks = 1;
3533                                atomic_inc(&rdev2->nr_pending);
3534                        } else
3535                                clear_bit(R5_MadeGoodRepl, &dev->flags);
3536                }
3537                if (!test_bit(R5_Insync, &dev->flags)) {
3538                        /* The ReadError flag will just be confusing now */
3539                        clear_bit(R5_ReadError, &dev->flags);
3540                        clear_bit(R5_ReWrite, &dev->flags);
3541                }
3542                if (test_bit(R5_ReadError, &dev->flags))
3543                        clear_bit(R5_Insync, &dev->flags);
3544                if (!test_bit(R5_Insync, &dev->flags)) {
3545                        if (s->failed < 2)
3546                                s->failed_num[s->failed] = i;
3547                        s->failed++;
3548                        if (rdev && !test_bit(Faulty, &rdev->flags))
3549                                do_recovery = 1;
3550                }
3551        }
3552        if (test_bit(STRIPE_SYNCING, &sh->state)) {
3553                /* If there is a failed device being replaced,
3554                 *     we must be recovering.
3555                 * else if we are after recovery_cp, we must be syncing
3556                 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
3557                 * else we can only be replacing
3558                 * sync and recovery both need to read all devices, and so
3559                 * use the same flag.
3560                 */
3561                if (do_recovery ||
3562                    sh->sector >= conf->mddev->recovery_cp ||
3563                    test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
3564                        s->syncing = 1;
3565                else
3566                        s->replacing = 1;
3567        }
3568        rcu_read_unlock();
3569}
3570
3571static void handle_stripe(struct stripe_head *sh)
3572{
3573        struct stripe_head_state s;
3574        struct r5conf *conf = sh->raid_conf;
3575        int i;
3576        int prexor;
3577        int disks = sh->disks;
3578        struct r5dev *pdev, *qdev;
3579
3580        clear_bit(STRIPE_HANDLE, &sh->state);
3581        if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
3582                /* already being handled, ensure it gets handled
3583                 * again when current action finishes */
3584                set_bit(STRIPE_HANDLE, &sh->state);
3585                return;
3586        }
3587
3588        if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
3589                spin_lock(&sh->stripe_lock);
3590                /* Cannot process 'sync' concurrently with 'discard' */
3591                if (!test_bit(STRIPE_DISCARD, &sh->state) &&
3592                    test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
3593                        set_bit(STRIPE_SYNCING, &sh->state);
3594                        clear_bit(STRIPE_INSYNC, &sh->state);
3595                        clear_bit(STRIPE_REPLACED, &sh->state);
3596                }
3597                spin_unlock(&sh->stripe_lock);
3598        }
3599        clear_bit(STRIPE_DELAYED, &sh->state);
3600
3601        pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
3602                "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
3603               (unsigned long long)sh->sector, sh->state,
3604               atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
3605               sh->check_state, sh->reconstruct_state);
3606
3607        analyse_stripe(sh, &s);
3608
3609        if (s.handle_bad_blocks) {
3610                set_bit(STRIPE_HANDLE, &sh->state);
3611                goto finish;
3612        }
3613
3614        if (unlikely(s.blocked_rdev)) {
3615                if (s.syncing || s.expanding || s.expanded ||
3616                    s.replacing || s.to_write || s.written) {
3617                        set_bit(STRIPE_HANDLE, &sh->state);
3618                        goto finish;
3619                }
3620                /* There is nothing for the blocked_rdev to block */
3621                rdev_dec_pending(s.blocked_rdev, conf->mddev);
3622                s.blocked_rdev = NULL;
3623        }
3624
3625        if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
3626                set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
3627                set_bit(STRIPE_BIOFILL_RUN, &sh->state);
3628        }
3629
3630        pr_debug("locked=%d uptodate=%d to_read=%d"
3631               " to_write=%d failed=%d failed_num=%d,%d\n",
3632               s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
3633               s.failed_num[0], s.failed_num[1]);
3634        /* check if the array has lost more than max_degraded devices and,
3635         * if so, some requests might need to be failed.
3636         */
3637        if (s.failed > conf->max_degraded) {
3638                sh->check_state = 0;
3639                sh->reconstruct_state = 0;
3640                if (s.to_read+s.to_write+s.written)
3641                        handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
3642                if (s.syncing + s.replacing)
3643                        handle_failed_sync(conf, sh, &s);
3644        }
3645
3646        /* Now we check to see if any write operations have recently
3647         * completed
3648         */
3649        prexor = 0;
3650        if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
3651                prexor = 1;
3652        if (sh->reconstruct_state == reconstruct_state_drain_result ||
3653            sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
3654                sh->reconstruct_state = reconstruct_state_idle;
3655
3656                /* All the 'written' buffers and the parity block are ready to
3657                 * be written back to disk
3658                 */
3659                BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
3660                       !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
3661                BUG_ON(sh->qd_idx >= 0 &&
3662                       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
3663                       !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
3664                for (i = disks; i--; ) {
3665                        struct r5dev *dev = &sh->dev[i];
3666                        if (test_bit(R5_LOCKED, &dev->flags) &&
3667                                (i == sh->pd_idx || i == sh->qd_idx ||
3668                                 dev->written)) {
3669                                pr_debug("Writing block %d\n", i);
3670                                set_bit(R5_Wantwrite, &dev->flags);
3671                                if (prexor)
3672                                        continue;
3673                                if (!test_bit(R5_Insync, &dev->flags) ||
3674                                    ((i == sh->pd_idx || i == sh->qd_idx)  &&
3675                                     s.failed == 0))
3676                                        set_bit(STRIPE_INSYNC, &sh->state);
3677                        }
3678                }
3679                if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3680                        s.dec_preread_active = 1;
3681        }
3682
3683        /*
3684         * might be able to return some write requests if the parity blocks
3685         * are safe, or on a failed drive
3686         */
3687        pdev = &sh->dev[sh->pd_idx];
3688        s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
3689                || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
3690        qdev = &sh->dev[sh->qd_idx];
3691        s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
3692                || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
3693                || conf->level < 6;
3694
3695        if (s.written &&
3696            (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
3697                             && !test_bit(R5_LOCKED, &pdev->flags)
3698                             && (test_bit(R5_UPTODATE, &pdev->flags) ||
3699                                 test_bit(R5_Discard, &pdev->flags))))) &&
3700            (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
3701                             && !test_bit(R5_LOCKED, &qdev->flags)
3702                             && (test_bit(R5_UPTODATE, &qdev->flags) ||
3703                                 test_bit(R5_Discard, &qdev->flags))))))
3704                handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
3705
3706        /* Now we might consider reading some blocks, either to check/generate
3707         * parity, or to satisfy requests
3708         * or to load a block that is being partially written.
3709         */
3710        if (s.to_read || s.non_overwrite
3711            || (conf->level == 6 && s.to_write && s.failed)
3712            || (s.syncing && (s.uptodate + s.compute < disks))
3713            || s.replacing
3714            || s.expanding)
3715                handle_stripe_fill(sh, &s, disks);
3716
3717        /* Now to consider new write requests and what else, if anything
3718         * should be read.  We do not handle new writes when:
3719         * 1/ A 'write' operation (copy+xor) is already in flight.
3720         * 2/ A 'check' operation is in flight, as it may clobber the parity
3721         *    block.
3722         */
3723        if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3724                handle_stripe_dirtying(conf, sh, &s, disks);
3725
3726        /* maybe we need to check and possibly fix the parity for this stripe
3727         * Any reads will already have been scheduled, so we just see if enough
3728         * data is available.  The parity check is held off while parity
3729         * dependent operations are in flight.
3730         */
3731        if (sh->check_state ||
3732            (s.syncing && s.locked == 0 &&
3733             !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3734             !test_bit(STRIPE_INSYNC, &sh->state))) {
3735                if (conf->level == 6)
3736                        handle_parity_checks6(conf, sh, &s, disks);
3737                else
3738                        handle_parity_checks5(conf, sh, &s, disks);
3739        }
3740
3741        if ((s.replacing || s.syncing) && s.locked == 0
3742            && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
3743            && !test_bit(STRIPE_REPLACED, &sh->state)) {
3744                /* Write out to replacement devices where possible */
3745                for (i = 0; i < conf->raid_disks; i++)
3746                        if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
3747                                WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
3748                                set_bit(R5_WantReplace, &sh->dev[i].flags);
3749                                set_bit(R5_LOCKED, &sh->dev[i].flags);
3750                                s.locked++;
3751                        }
3752                if (s.replacing)
3753                        set_bit(STRIPE_INSYNC, &sh->state);
3754                set_bit(STRIPE_REPLACED, &sh->state);
3755        }
3756        if ((s.syncing || s.replacing) && s.locked == 0 &&
3757            !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3758            test_bit(STRIPE_INSYNC, &sh->state)) {
3759                md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3760                clear_bit(STRIPE_SYNCING, &sh->state);
3761                if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
3762                        wake_up(&conf->wait_for_overlap);
3763        }
3764
3765        /* If the failed drives are just a ReadError, then we might need
3766         * to progress the repair/check process
3767         */
3768        if (s.failed <= conf->max_degraded && !conf->mddev->ro)
3769                for (i = 0; i < s.failed; i++) {
3770                        struct r5dev *dev = &sh->dev[s.failed_num[i]];
3771                        if (test_bit(R5_ReadError, &dev->flags)
3772                            && !test_bit(R5_LOCKED, &dev->flags)
3773                            && test_bit(R5_UPTODATE, &dev->flags)
3774                                ) {
3775                                if (!test_bit(R5_ReWrite, &dev->flags)) {
3776                                        set_bit(R5_Wantwrite, &dev->flags);
3777                                        set_bit(R5_ReWrite, &dev->flags);
3778                                        set_bit(R5_LOCKED, &dev->flags);
3779                                        s.locked++;
3780                                } else {
3781                                        /* let's read it back */
3782                                        set_bit(R5_Wantread, &dev->flags);
3783                                        set_bit(R5_LOCKED, &dev->flags);
3784                                        s.locked++;
3785                                }
3786                        }
3787                }
3788
3789
3790        /* Finish reconstruct operations initiated by the expansion process */
3791        if (sh->reconstruct_state == reconstruct_state_result) {
3792                struct stripe_head *sh_src
3793                        = get_active_stripe(conf, sh->sector, 1, 1, 1);
3794                if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
3795                        /* sh cannot be written until sh_src has been read.
3796                         * so arrange for sh to be delayed a little
3797                         */
3798                        set_bit(STRIPE_DELAYED, &sh->state);
3799                        set_bit(STRIPE_HANDLE, &sh->state);
3800                        if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
3801                                              &sh_src->state))
3802                                atomic_inc(&conf->preread_active_stripes);
3803                        release_stripe(sh_src);
3804                        goto finish;
3805                }
3806                if (sh_src)
3807                        release_stripe(sh_src);
3808
3809                sh->reconstruct_state = reconstruct_state_idle;
3810                clear_bit(STRIPE_EXPANDING, &sh->state);
3811                for (i = conf->raid_disks; i--; ) {
3812                        set_bit(R5_Wantwrite, &sh->dev[i].flags);
3813                        set_bit(R5_LOCKED, &sh->dev[i].flags);
3814                        s.locked++;
3815                }
3816        }
3817
3818        if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
3819            !sh->reconstruct_state) {
3820                /* Need to write out all blocks after computing parity */
3821                sh->disks = conf->raid_disks;
3822                stripe_set_idx(sh->sector, conf, 0, sh);
3823                schedule_reconstruction(sh, &s, 1, 1);
3824        } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3825                clear_bit(STRIPE_EXPAND_READY, &sh->state);
3826                atomic_dec(&conf->reshape_stripes);
3827                wake_up(&conf->wait_for_overlap);
3828                md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3829        }
3830
3831        if (s.expanding && s.locked == 0 &&
3832            !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3833                handle_stripe_expansion(conf, sh);
3834
3835finish:
3836        /* wait for this device to become unblocked */
3837        if (unlikely(s.blocked_rdev)) {
3838                if (conf->mddev->external)
3839                        md_wait_for_blocked_rdev(s.blocked_rdev,
3840                                                 conf->mddev);
3841                else
3842                        /* Internal metadata will immediately
3843                         * be written by raid5d, so we don't
3844                         * need to wait here.
3845                         */
3846                        rdev_dec_pending(s.blocked_rdev,
3847                                         conf->mddev);
3848        }
3849
3850        if (s.handle_bad_blocks)
3851                for (i = disks; i--; ) {
3852                        struct md_rdev *rdev;
3853                        struct r5dev *dev = &sh->dev[i];
3854                        if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
3855                                /* We own a safe reference to the rdev */
3856                                rdev = conf->disks[i].rdev;
3857                                if (!rdev_set_badblocks(rdev, sh->sector,
3858                                                        STRIPE_SECTORS, 0))
3859                                        md_error(conf->mddev, rdev);
3860                                rdev_dec_pending(rdev, conf->mddev);
3861                        }
3862                        if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
3863                                rdev = conf->disks[i].rdev;
3864                                rdev_clear_badblocks(rdev, sh->sector,
3865                                                     STRIPE_SECTORS, 0);
3866                                rdev_dec_pending(rdev, conf->mddev);
3867                        }
3868                        if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
3869                                rdev = conf->disks[i].replacement;
3870                                if (!rdev)
3871                                        /* rdev have been moved down */
3872                                        rdev = conf->disks[i].rdev;
3873                                rdev_clear_badblocks(rdev, sh->sector,
3874                                                     STRIPE_SECTORS, 0);
3875                                rdev_dec_pending(rdev, conf->mddev);
3876                        }
3877                }
3878
3879        if (s.ops_request)
3880                raid_run_ops(sh, s.ops_request);
3881
3882        ops_run_io(sh, &s);
3883
3884        if (s.dec_preread_active) {
3885                /* We delay this until after ops_run_io so that if make_request
3886                 * is waiting on a flush, it won't continue until the writes
3887                 * have actually been submitted.
3888                 */
3889                atomic_dec(&conf->preread_active_stripes);
3890                if (atomic_read(&conf->preread_active_stripes) <
3891                    IO_THRESHOLD)
3892                        md_wakeup_thread(conf->mddev->thread);
3893        }
3894
3895        return_io(s.return_bi);
3896
3897        clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
3898}
3899
3900static void raid5_activate_delayed(struct r5conf *conf)
3901{
3902        if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
3903                while (!list_empty(&conf->delayed_list)) {
3904                        struct list_head *l = conf->delayed_list.next;
3905                        struct stripe_head *sh;
3906                        sh = list_entry(l, struct stripe_head, lru);
3907                        list_del_init(l);
3908                        clear_bit(STRIPE_DELAYED, &sh->state);
3909                        if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3910                                atomic_inc(&conf->preread_active_stripes);
3911                        list_add_tail(&sh->lru, &conf->hold_list);
3912                        raid5_wakeup_stripe_thread(sh);
3913                }
3914        }
3915}
3916
3917static void activate_bit_delay(struct r5conf *conf)
3918{
3919        /* device_lock is held */
3920        struct list_head head;
3921        list_add(&head, &conf->bitmap_list);
3922        list_del_init(&conf->bitmap_list);
3923        while (!list_empty(&head)) {
3924                struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
3925                list_del_init(&sh->lru);
3926                atomic_inc(&sh->count);
3927                __release_stripe(conf, sh);
3928        }
3929}
3930
3931int md_raid5_congested(struct mddev *mddev, int bits)
3932{
3933        struct r5conf *conf = mddev->private;
3934
3935        /* No difference between reads and writes.  Just check
3936         * how busy the stripe_cache is
3937         */
3938
3939        if (conf->inactive_blocked)
3940                return 1;
3941        if (conf->quiesce)
3942                return 1;
3943        if (list_empty_careful(&conf->inactive_list))
3944                return 1;
3945
3946        return 0;
3947}
3948EXPORT_SYMBOL_GPL(md_raid5_congested);
3949
3950static int raid5_congested(void *data, int bits)
3951{
3952        struct mddev *mddev = data;
3953
3954        return mddev_congested(mddev, bits) ||
3955                md_raid5_congested(mddev, bits);
3956}
3957
3958/* We want read requests to align with chunks where possible,
3959 * but write requests don't need to.
3960 */
3961static int raid5_mergeable_bvec(struct request_queue *q,
3962                                struct bvec_merge_data *bvm,
3963                                struct bio_vec *biovec)
3964{
3965        struct mddev *mddev = q->queuedata;
3966        sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
3967        int max;
3968        unsigned int chunk_sectors = mddev->chunk_sectors;
3969        unsigned int bio_sectors = bvm->bi_size >> 9;
3970
3971        if ((bvm->bi_rw & 1) == WRITE)
3972                return biovec->bv_len; /* always allow writes to be mergeable */
3973
3974        if (mddev->new_chunk_sectors < mddev->chunk_sectors)
3975                chunk_sectors = mddev->new_chunk_sectors;
3976        max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
3977        if (max < 0) max = 0;
3978        if (max <= biovec->bv_len && bio_sectors == 0)
3979                return biovec->bv_len;
3980        else
3981                return max;
3982}
3983
3984
3985static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
3986{
3987        sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
3988        unsigned int chunk_sectors = mddev->chunk_sectors;
3989        unsigned int bio_sectors = bio_sectors(bio);
3990
3991        if (mddev->new_chunk_sectors < mddev->chunk_sectors)
3992                chunk_sectors = mddev->new_chunk_sectors;
3993        return  chunk_sectors >=
3994                ((sector & (chunk_sectors - 1)) + bio_sectors);
3995}
3996
3997/*
3998 *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
3999 *  later sampled by raid5d.
4000 */
4001static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
4002{
4003        unsigned long flags;
4004
4005        spin_lock_irqsave(&conf->device_lock, flags);
4006
4007        bi->bi_next = conf->retry_read_aligned_list;
4008        conf->retry_read_aligned_list = bi;
4009
4010        spin_unlock_irqrestore(&conf->device_lock, flags);
4011        md_wakeup_thread(conf->mddev->thread);
4012}
4013
4014
4015static struct bio *remove_bio_from_retry(struct r5conf *conf)
4016{
4017        struct bio *bi;
4018
4019        bi = conf->retry_read_aligned;
4020        if (bi) {
4021                conf->retry_read_aligned = NULL;
4022                return bi;
4023        }
4024        bi = conf->retry_read_aligned_list;
4025        if(bi) {
4026                conf->retry_read_aligned_list = bi->bi_next;
4027                bi->bi_next = NULL;
4028                /*
4029                 * this sets the active strip count to 1 and the processed
4030                 * strip count to zero (upper 8 bits)
4031                 */
4032                raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
4033        }
4034
4035        return bi;
4036}
4037
4038
4039/*
4040 *  The "raid5_align_endio" should check if the read succeeded and if it
4041 *  did, call bio_endio on the original bio (having bio_put the new bio
4042 *  first).
4043 *  If the read failed..
4044 */
4045static void raid5_align_endio(struct bio *bi, int error)
4046{
4047        struct bio* raid_bi  = bi->bi_private;
4048        struct mddev *mddev;
4049        struct r5conf *conf;
4050        int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
4051        struct md_rdev *rdev;
4052
4053        bio_put(bi);
4054
4055        rdev = (void*)raid_bi->bi_next;
4056        raid_bi->bi_next = NULL;
4057        mddev = rdev->mddev;
4058        conf = mddev->private;
4059
4060        rdev_dec_pending(rdev, conf->mddev);
4061
4062        if (!error && uptodate) {
4063                trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
4064                                         raid_bi, 0);
4065                bio_endio(raid_bi, 0);
4066                if (atomic_dec_and_test(&conf->active_aligned_reads))
4067                        wake_up(&conf->wait_for_stripe);
4068                return;
4069        }
4070
4071
4072        pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
4073
4074        add_bio_to_retry(raid_bi, conf);
4075}
4076
4077static int bio_fits_rdev(struct bio *bi)
4078{
4079        struct request_queue *q = bdev_get_queue(bi->bi_bdev);
4080
4081        if (bio_sectors(bi) > queue_max_sectors(q))
4082                return 0;
4083        blk_recount_segments(q, bi);
4084        if (bi->bi_phys_segments > queue_max_segments(q))
4085                return 0;
4086
4087        if (q->merge_bvec_fn)
4088                /* it's too hard to apply the merge_bvec_fn at this stage,
4089                 * just just give up
4090                 */
4091                return 0;
4092
4093        return 1;
4094}
4095
4096
4097static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4098{
4099        struct r5conf *conf = mddev->private;
4100        int dd_idx;
4101        struct bio* align_bi;
4102        struct md_rdev *rdev;
4103        sector_t end_sector;
4104
4105        if (!in_chunk_boundary(mddev, raid_bio)) {
4106                pr_debug("chunk_aligned_read : non aligned\n");
4107                return 0;
4108        }
4109        /*
4110         * use bio_clone_mddev to make a copy of the bio
4111         */
4112        align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
4113        if (!align_bi)
4114                return 0;
4115        /*
4116         *   set bi_end_io to a new function, and set bi_private to the
4117         *     original bio.
4118         */
4119        align_bi->bi_end_io  = raid5_align_endio;
4120        align_bi->bi_private = raid_bio;
4121        /*
4122         *      compute position
4123         */
4124        align_bi->bi_sector =  raid5_compute_sector(conf, raid_bio->bi_sector,
4125                                                    0,
4126                                                    &dd_idx, NULL);
4127
4128        end_sector = bio_end_sector(align_bi);
4129        rcu_read_lock();
4130        rdev = rcu_dereference(conf->disks[dd_idx].replacement);
4131        if (!rdev || test_bit(Faulty, &rdev->flags) ||
4132            rdev->recovery_offset < end_sector) {
4133                rdev = rcu_dereference(conf->disks[dd_idx].rdev);
4134                if (rdev &&
4135                    (test_bit(Faulty, &rdev->flags) ||
4136                    !(test_bit(In_sync, &rdev->flags) ||
4137                      rdev->recovery_offset >= end_sector)))
4138                        rdev = NULL;
4139        }
4140        if (rdev) {
4141                sector_t first_bad;
4142                int bad_sectors;
4143
4144                atomic_inc(&rdev->nr_pending);
4145                rcu_read_unlock();
4146                raid_bio->bi_next = (void*)rdev;
4147                align_bi->bi_bdev =  rdev->bdev;
4148                align_bi->bi_flags &= ~(1 << BIO_SEG_VALID);
4149
4150                if (!bio_fits_rdev(align_bi) ||
4151                    is_badblock(rdev, align_bi->bi_sector, bio_sectors(align_bi),
4152                                &first_bad, &bad_sectors)) {
4153                        /* too big in some way, or has a known bad block */
4154                        bio_put(align_bi);
4155                        rdev_dec_pending(rdev, mddev);
4156                        return 0;
4157                }
4158
4159                /* No reshape active, so we can trust rdev->data_offset */
4160                align_bi->bi_sector += rdev->data_offset;
4161
4162                spin_lock_irq(&conf->device_lock);
4163                wait_event_lock_irq(conf->wait_for_stripe,
4164                                    conf->quiesce == 0,
4165                                    conf->device_lock);
4166                atomic_inc(&conf->active_aligned_reads);
4167                spin_unlock_irq(&conf->device_lock);
4168
4169                if (mddev->gendisk)
4170                        trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
4171                                              align_bi, disk_devt(mddev->gendisk),
4172                                              raid_bio->bi_sector);
4173                generic_make_request(align_bi);
4174                return 1;
4175        } else {
4176                rcu_read_unlock();
4177                bio_put(align_bi);
4178                return 0;
4179        }
4180}
4181
4182/* __get_priority_stripe - get the next stripe to process
4183 *
4184 * Full stripe writes are allowed to pass preread active stripes up until
4185 * the bypass_threshold is exceeded.  In general the bypass_count
4186 * increments when the handle_list is handled before the hold_list; however, it
4187 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
4188 * stripe with in flight i/o.  The bypass_count will be reset when the
4189 * head of the hold_list has changed, i.e. the head was promoted to the
4190 * handle_list.
4191 */
4192static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
4193{
4194        struct stripe_head *sh = NULL, *tmp;
4195        struct list_head *handle_list = NULL;
4196        struct r5worker_group *wg = NULL;
4197
4198        if (conf->worker_cnt_per_group == 0) {
4199                handle_list = &conf->handle_list;
4200        } else if (group != ANY_GROUP) {
4201                handle_list = &conf->worker_groups[group].handle_list;
4202                wg = &conf->worker_groups[group];
4203        } else {
4204                int i;
4205                for (i = 0; i < conf->group_cnt; i++) {
4206                        handle_list = &conf->worker_groups[i].handle_list;
4207                        wg = &conf->worker_groups[i];
4208                        if (!list_empty(handle_list))
4209                                break;
4210                }
4211        }
4212
4213        pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
4214                  __func__,
4215                  list_empty(handle_list) ? "empty" : "busy",
4216                  list_empty(&conf->hold_list) ? "empty" : "busy",
4217                  atomic_read(&conf->pending_full_writes), conf->bypass_count);
4218
4219        if (!list_empty(handle_list)) {
4220                sh = list_entry(handle_list->next, typeof(*sh), lru);
4221
4222                if (list_empty(&conf->hold_list))
4223                        conf->bypass_count = 0;
4224                else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
4225                        if (conf->hold_list.next == conf->last_hold)
4226                                conf->bypass_count++;
4227                        else {
4228                                conf->last_hold = conf->hold_list.next;
4229                                conf->bypass_count -= conf->bypass_threshold;
4230                                if (conf->bypass_count < 0)
4231                                        conf->bypass_count = 0;
4232                        }
4233                }
4234        } else if (!list_empty(&conf->hold_list) &&
4235                   ((conf->bypass_threshold &&
4236                     conf->bypass_count > conf->bypass_threshold) ||
4237                    atomic_read(&conf->pending_full_writes) == 0)) {
4238
4239                list_for_each_entry(tmp, &conf->hold_list,  lru) {
4240                        if (conf->worker_cnt_per_group == 0 ||
4241                            group == ANY_GROUP ||
4242                            !cpu_online(tmp->cpu) ||
4243                            cpu_to_group(tmp->cpu) == group) {
4244                                sh = tmp;
4245                                break;
4246                        }
4247                }
4248
4249                if (sh) {
4250                        conf->bypass_count -= conf->bypass_threshold;
4251                        if (conf->bypass_count < 0)
4252                                conf->bypass_count = 0;
4253                }
4254                wg = NULL;
4255        }
4256
4257        if (!sh)
4258                return NULL;
4259
4260        if (wg) {
4261                wg->stripes_cnt--;
4262                sh->group = NULL;
4263        }
4264        list_del_init(&sh->lru);
4265        atomic_inc(&sh->count);
4266        BUG_ON(atomic_read(&sh->count) != 1);
4267        return sh;
4268}
4269
4270struct raid5_plug_cb {
4271        struct blk_plug_cb      cb;
4272        struct list_head        list;
4273};
4274
4275static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
4276{
4277        struct raid5_plug_cb *cb = container_of(
4278                blk_cb, struct raid5_plug_cb, cb);
4279        struct stripe_head *sh;
4280        struct mddev *mddev = cb->cb.data;
4281        struct r5conf *conf = mddev->private;
4282        int cnt = 0;
4283
4284        if (cb->list.next && !list_empty(&cb->list)) {
4285                spin_lock_irq(&conf->device_lock);
4286                while (!list_empty(&cb->list)) {
4287                        sh = list_first_entry(&cb->list, struct stripe_head, lru);
4288                        list_del_init(&sh->lru);
4289                        /*
4290                         * avoid race release_stripe_plug() sees
4291                         * STRIPE_ON_UNPLUG_LIST clear but the stripe
4292                         * is still in our list
4293                         */
4294                        smp_mb__before_clear_bit();
4295                        clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
4296                        /*
4297                         * STRIPE_ON_RELEASE_LIST could be set here. In that
4298                         * case, the count is always > 1 here
4299                         */
4300                        __release_stripe(conf, sh);
4301                        cnt++;
4302                }
4303                spin_unlock_irq(&conf->device_lock);
4304        }
4305        if (mddev->queue)
4306                trace_block_unplug(mddev->queue, cnt, !from_schedule);
4307        kfree(cb);
4308}
4309
4310static void release_stripe_plug(struct mddev *mddev,
4311                                struct stripe_head *sh)
4312{
4313        struct blk_plug_cb *blk_cb = blk_check_plugged(
4314                raid5_unplug, mddev,
4315                sizeof(struct raid5_plug_cb));
4316        struct raid5_plug_cb *cb;
4317
4318        if (!blk_cb) {
4319                release_stripe(sh);
4320                return;
4321        }
4322
4323        cb = container_of(blk_cb, struct raid5_plug_cb, cb);
4324
4325        if (cb->list.next == NULL)
4326                INIT_LIST_HEAD(&cb->list);
4327
4328        if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
4329                list_add_tail(&sh->lru, &cb->list);
4330        else
4331                release_stripe(sh);
4332}
4333
4334static void make_discard_request(struct mddev *mddev, struct bio *bi)
4335{
4336        struct r5conf *conf = mddev->private;
4337        sector_t logical_sector, last_sector;
4338        struct stripe_head *sh;
4339        int remaining;
4340        int stripe_sectors;
4341
4342        if (mddev->reshape_position != MaxSector)
4343                /* Skip discard while reshape is happening */
4344                return;
4345
4346        logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4347        last_sector = bi->bi_sector + (bi->bi_size>>9);
4348
4349        bi->bi_next = NULL;
4350        bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
4351
4352        stripe_sectors = conf->chunk_sectors *
4353                (conf->raid_disks - conf->max_degraded);
4354        logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
4355                                               stripe_sectors);
4356        sector_div(last_sector, stripe_sectors);
4357
4358        logical_sector *= conf->chunk_sectors;
4359        last_sector *= conf->chunk_sectors;
4360
4361        for (; logical_sector < last_sector;
4362             logical_sector += STRIPE_SECTORS) {
4363                DEFINE_WAIT(w);
4364                int d;
4365        again:
4366                sh = get_active_stripe(conf, logical_sector, 0, 0, 0);
4367                prepare_to_wait(&conf->wait_for_overlap, &w,
4368                                TASK_UNINTERRUPTIBLE);
4369                set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
4370                if (test_bit(STRIPE_SYNCING, &sh->state)) {
4371                        release_stripe(sh);
4372                        schedule();
4373                        goto again;
4374                }
4375                clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
4376                spin_lock_irq(&sh->stripe_lock);
4377                for (d = 0; d < conf->raid_disks; d++) {
4378                        if (d == sh->pd_idx || d == sh->qd_idx)
4379                                continue;
4380                        if (sh->dev[d].towrite || sh->dev[d].toread) {
4381                                set_bit(R5_Overlap, &sh->dev[d].flags);
4382                                spin_unlock_irq(&sh->stripe_lock);
4383                                release_stripe(sh);
4384                                schedule();
4385                                goto again;
4386                        }
4387                }
4388                set_bit(STRIPE_DISCARD, &sh->state);
4389                finish_wait(&conf->wait_for_overlap, &w);
4390                for (d = 0; d < conf->raid_disks; d++) {
4391                        if (d == sh->pd_idx || d == sh->qd_idx)
4392                                continue;
4393                        sh->dev[d].towrite = bi;
4394                        set_bit(R5_OVERWRITE, &sh->dev[d].flags);
4395                        raid5_inc_bi_active_stripes(bi);
4396                }
4397                spin_unlock_irq(&sh->stripe_lock);
4398                if (conf->mddev->bitmap) {
4399                        for (d = 0;
4400                             d < conf->raid_disks - conf->max_degraded;
4401                             d++)
4402                                bitmap_startwrite(mddev->bitmap,
4403                                                  sh->sector,
4404                                                  STRIPE_SECTORS,
4405                                                  0);
4406                        sh->bm_seq = conf->seq_flush + 1;
4407                        set_bit(STRIPE_BIT_DELAY, &sh->state);
4408                }
4409
4410                set_bit(STRIPE_HANDLE, &sh->state);
4411                clear_bit(STRIPE_DELAYED, &sh->state);
4412                if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4413                        atomic_inc(&conf->preread_active_stripes);
4414                release_stripe_plug(mddev, sh);
4415        }
4416
4417        remaining = raid5_dec_bi_active_stripes(bi);
4418        if (remaining == 0) {
4419                md_write_end(mddev);
4420                bio_endio(bi, 0);
4421        }
4422}
4423
4424static void make_request(struct mddev *mddev, struct bio * bi)
4425{
4426        struct r5conf *conf = mddev->private;
4427        int dd_idx;
4428        sector_t new_sector;
4429        sector_t logical_sector, last_sector;
4430        struct stripe_head *sh;
4431        const int rw = bio_data_dir(bi);
4432        int remaining;
4433
4434        if (unlikely(bi->bi_rw & REQ_FLUSH)) {
4435                md_flush_request(mddev, bi);
4436                return;
4437        }
4438
4439        md_write_start(mddev, bi);
4440
4441        if (rw == READ &&
4442             mddev->reshape_position == MaxSector &&
4443             chunk_aligned_read(mddev,bi))
4444                return;
4445
4446        if (unlikely(bi->bi_rw & REQ_DISCARD)) {
4447                make_discard_request(mddev, bi);
4448                return;
4449        }
4450
4451        logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4452        last_sector = bio_end_sector(bi);
4453        bi->bi_next = NULL;
4454        bi->bi_phys_segments = 1;       /* over-loaded to count active stripes */
4455
4456        for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
4457                DEFINE_WAIT(w);
4458                int previous;
4459                int seq;
4460
4461        retry:
4462                seq = read_seqcount_begin(&conf->gen_lock);
4463                previous = 0;
4464                prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
4465                if (unlikely(conf->reshape_progress != MaxSector)) {
4466                        /* spinlock is needed as reshape_progress may be
4467                         * 64bit on a 32bit platform, and so it might be
4468                         * possible to see a half-updated value
4469                         * Of course reshape_progress could change after
4470                         * the lock is dropped, so once we get a reference
4471                         * to the stripe that we think it is, we will have
4472                         * to check again.
4473                         */
4474                        spin_lock_irq(&conf->device_lock);
4475                        if (mddev->reshape_backwards
4476                            ? logical_sector < conf->reshape_progress
4477                            : logical_sector >= conf->reshape_progress) {
4478                                previous = 1;
4479                        } else {
4480                                if (mddev->reshape_backwards
4481                                    ? logical_sector < conf->reshape_safe
4482                                    : logical_sector >= conf->reshape_safe) {
4483                                        spin_unlock_irq(&conf->device_lock);
4484                                        schedule();
4485                                        goto retry;
4486                                }
4487                        }
4488                        spin_unlock_irq(&conf->device_lock);
4489                }
4490
4491                new_sector = raid5_compute_sector(conf, logical_sector,
4492                                                  previous,
4493                                                  &dd_idx, NULL);
4494                pr_debug("raid456: make_request, sector %llu logical %llu\n",
4495                        (unsigned long long)new_sector,
4496                        (unsigned long long)logical_sector);
4497
4498                sh = get_active_stripe(conf, new_sector, previous,
4499                                       (bi->bi_rw&RWA_MASK), 0);
4500                if (sh) {
4501                        if (unlikely(previous)) {
4502                                /* expansion might have moved on while waiting for a
4503                                 * stripe, so we must do the range check again.
4504                                 * Expansion could still move past after this
4505                                 * test, but as we are holding a reference to
4506                                 * 'sh', we know that if that happens,
4507                                 *  STRIPE_EXPANDING will get set and the expansion
4508                                 * won't proceed until we finish with the stripe.
4509                                 */
4510                                int must_retry = 0;
4511                                spin_lock_irq(&conf->device_lock);
4512                                if (mddev->reshape_backwards
4513                                    ? logical_sector >= conf->reshape_progress
4514                                    : logical_sector < conf->reshape_progress)
4515                                        /* mismatch, need to try again */
4516                                        must_retry = 1;
4517                                spin_unlock_irq(&conf->device_lock);
4518                                if (must_retry) {
4519                                        release_stripe(sh);
4520                                        schedule();
4521                                        goto retry;
4522                                }
4523                        }
4524                        if (read_seqcount_retry(&conf->gen_lock, seq)) {
4525                                /* Might have got the wrong stripe_head
4526                                 * by accident
4527                                 */
4528                                release_stripe(sh);
4529                                goto retry;
4530                        }
4531
4532                        if (rw == WRITE &&
4533                            logical_sector >= mddev->suspend_lo &&
4534                            logical_sector < mddev->suspend_hi) {
4535                                release_stripe(sh);
4536                                /* As the suspend_* range is controlled by
4537                                 * userspace, we want an interruptible
4538                                 * wait.
4539                                 */
4540                                flush_signals(current);
4541                                prepare_to_wait(&conf->wait_for_overlap,
4542                                                &w, TASK_INTERRUPTIBLE);
4543                                if (logical_sector >= mddev->suspend_lo &&
4544                                    logical_sector < mddev->suspend_hi)
4545                                        schedule();
4546                                goto retry;
4547                        }
4548
4549                        if (test_bit(STRIPE_EXPANDING, &sh->state) ||
4550                            !add_stripe_bio(sh, bi, dd_idx, rw)) {
4551                                /* Stripe is busy expanding or
4552                                 * add failed due to overlap.  Flush everything
4553                                 * and wait a while
4554                                 */
4555                                md_wakeup_thread(mddev->thread);
4556                                release_stripe(sh);
4557                                schedule();
4558                                goto retry;
4559                        }
4560                        finish_wait(&conf->wait_for_overlap, &w);
4561                        set_bit(STRIPE_HANDLE, &sh->state);
4562                        clear_bit(STRIPE_DELAYED, &sh->state);
4563                        if ((bi->bi_rw & REQ_SYNC) &&
4564                            !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4565                                atomic_inc(&conf->preread_active_stripes);
4566                        release_stripe_plug(mddev, sh);
4567                } else {
4568                        /* cannot get stripe for read-ahead, just give-up */
4569                        clear_bit(BIO_UPTODATE, &bi->bi_flags);
4570                        finish_wait(&conf->wait_for_overlap, &w);
4571                        break;
4572                }
4573        }
4574
4575        remaining = raid5_dec_bi_active_stripes(bi);
4576        if (remaining == 0) {
4577
4578                if ( rw == WRITE )
4579                        md_write_end(mddev);
4580
4581                trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
4582                                         bi, 0);
4583                bio_endio(bi, 0);
4584        }
4585}
4586
4587static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
4588
4589static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
4590{
4591        /* reshaping is quite different to recovery/resync so it is
4592         * handled quite separately ... here.
4593         *
4594         * On each call to sync_request, we gather one chunk worth of
4595         * destination stripes and flag them as expanding.
4596         * Then we find all the source stripes and request reads.
4597         * As the reads complete, handle_stripe will copy the data
4598         * into the destination stripe and release that stripe.
4599         */
4600        struct r5conf *conf = mddev->private;
4601        struct stripe_head *sh;
4602        sector_t first_sector, last_sector;
4603        int raid_disks = conf->previous_raid_disks;
4604        int data_disks = raid_disks - conf->max_degraded;
4605        int new_data_disks = conf->raid_disks - conf->max_degraded;
4606        int i;
4607        int dd_idx;
4608        sector_t writepos, readpos, safepos;
4609        sector_t stripe_addr;
4610        int reshape_sectors;
4611        struct list_head stripes;
4612
4613        if (sector_nr == 0) {
4614                /* If restarting in the middle, skip the initial sectors */
4615                if (mddev->reshape_backwards &&
4616                    conf->reshape_progress < raid5_size(mddev, 0, 0)) {
4617                        sector_nr = raid5_size(mddev, 0, 0)
4618                                - conf->reshape_progress;
4619                } else if (!mddev->reshape_backwards &&
4620                           conf->reshape_progress > 0)
4621                        sector_nr = conf->reshape_progress;
4622                sector_div(sector_nr, new_data_disks);
4623                if (sector_nr) {
4624                        mddev->curr_resync_completed = sector_nr;
4625                        sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4626                        *skipped = 1;
4627                        return sector_nr;
4628                }
4629        }
4630
4631        /* We need to process a full chunk at a time.
4632         * If old and new chunk sizes differ, we need to process the
4633         * largest of these
4634         */
4635        if (mddev->new_chunk_sectors > mddev->chunk_sectors)
4636                reshape_sectors = mddev->new_chunk_sectors;
4637        else
4638                reshape_sectors = mddev->chunk_sectors;
4639
4640        /* We update the metadata at least every 10 seconds, or when
4641         * the data about to be copied would over-write the source of
4642         * the data at the front of the range.  i.e. one new_stripe
4643         * along from reshape_progress new_maps to after where
4644         * reshape_safe old_maps to
4645         */
4646        writepos = conf->reshape_progress;
4647        sector_div(writepos, new_data_disks);
4648        readpos = conf->reshape_progress;
4649        sector_div(readpos, data_disks);
4650        safepos = conf->reshape_safe;
4651        sector_div(safepos, data_disks);
4652        if (mddev->reshape_backwards) {
4653                writepos -= min_t(sector_t, reshape_sectors, writepos);
4654                readpos += reshape_sectors;
4655                safepos += reshape_sectors;
4656        } else {
4657                writepos += reshape_sectors;
4658                readpos -= min_t(sector_t, reshape_sectors, readpos);
4659                safepos -= min_t(sector_t, reshape_sectors, safepos);
4660        }
4661
4662        /* Having calculated the 'writepos' possibly use it
4663         * to set 'stripe_addr' which is where we will write to.
4664         */
4665        if (mddev->reshape_backwards) {
4666                BUG_ON(conf->reshape_progress == 0);
4667                stripe_addr = writepos;
4668                BUG_ON((mddev->dev_sectors &
4669                        ~((sector_t)reshape_sectors - 1))
4670                       - reshape_sectors - stripe_addr
4671                       != sector_nr);
4672        } else {
4673                BUG_ON(writepos != sector_nr + reshape_sectors);
4674                stripe_addr = sector_nr;
4675        }
4676
4677        /* 'writepos' is the most advanced device address we might write.
4678         * 'readpos' is the least advanced device address we might read.
4679         * 'safepos' is the least address recorded in the metadata as having
4680         *     been reshaped.
4681         * If there is a min_offset_diff, these are adjusted either by
4682         * increasing the safepos/readpos if diff is negative, or
4683         * increasing writepos if diff is positive.
4684         * If 'readpos' is then behind 'writepos', there is no way that we can
4685         * ensure safety in the face of a crash - that must be done by userspace
4686         * making a backup of the data.  So in that case there is no particular
4687         * rush to update metadata.
4688         * Otherwise if 'safepos' is behind 'writepos', then we really need to
4689         * update the metadata to advance 'safepos' to match 'readpos' so that
4690         * we can be safe in the event of a crash.
4691         * So we insist on updating metadata if safepos is behind writepos and
4692         * readpos is beyond writepos.
4693         * In any case, update the metadata every 10 seconds.
4694         * Maybe that number should be configurable, but I'm not sure it is
4695         * worth it.... maybe it could be a multiple of safemode_delay???
4696         */
4697        if (conf->min_offset_diff < 0) {
4698                safepos += -conf->min_offset_diff;
4699                readpos += -conf->min_offset_diff;
4700        } else
4701                writepos += conf->min_offset_diff;
4702
4703        if ((mddev->reshape_backwards
4704             ? (safepos > writepos && readpos < writepos)
4705             : (safepos < writepos && readpos > writepos)) ||
4706            time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4707                /* Cannot proceed until we've updated the superblock... */
4708                wait_event(conf->wait_for_overlap,
4709                           atomic_read(&conf->reshape_stripes)==0);
4710                mddev->reshape_position = conf->reshape_progress;
4711                mddev->curr_resync_completed = sector_nr;
4712                conf->reshape_checkpoint = jiffies;
4713                set_bit(MD_CHANGE_DEVS, &mddev->flags);
4714                md_wakeup_thread(mddev->thread);
4715                wait_event(mddev->sb_wait, mddev->flags == 0 ||
4716                           kthread_should_stop());
4717                spin_lock_irq(&conf->device_lock);
4718                conf->reshape_safe = mddev->reshape_position;
4719                spin_unlock_irq(&conf->device_lock);
4720                wake_up(&conf->wait_for_overlap);
4721                sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4722        }
4723
4724        INIT_LIST_HEAD(&stripes);
4725        for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
4726                int j;
4727                int skipped_disk = 0;
4728                sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
4729                set_bit(STRIPE_EXPANDING, &sh->state);
4730                atomic_inc(&conf->reshape_stripes);
4731                /* If any of this stripe is beyond the end of the old
4732                 * array, then we need to zero those blocks
4733                 */
4734                for (j=sh->disks; j--;) {
4735                        sector_t s;
4736                        if (j == sh->pd_idx)
4737                                continue;
4738                        if (conf->level == 6 &&
4739                            j == sh->qd_idx)
4740                                continue;
4741                        s = compute_blocknr(sh, j, 0);
4742                        if (s < raid5_size(mddev, 0, 0)) {
4743                                skipped_disk = 1;
4744                                continue;
4745                        }
4746                        memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
4747                        set_bit(R5_Expanded, &sh->dev[j].flags);
4748                        set_bit(R5_UPTODATE, &sh->dev[j].flags);
4749                }
4750                if (!skipped_disk) {
4751                        set_bit(STRIPE_EXPAND_READY, &sh->state);
4752                        set_bit(STRIPE_HANDLE, &sh->state);
4753                }
4754                list_add(&sh->lru, &stripes);
4755        }
4756        spin_lock_irq(&conf->device_lock);
4757        if (mddev->reshape_backwards)
4758                conf->reshape_progress -= reshape_sectors * new_data_disks;
4759        else
4760                conf->reshape_progress += reshape_sectors * new_data_disks;
4761        spin_unlock_irq(&conf->device_lock);
4762        /* Ok, those stripe are ready. We can start scheduling
4763         * reads on the source stripes.
4764         * The source stripes are determined by mapping the first and last
4765         * block on the destination stripes.
4766         */
4767        first_sector =
4768                raid5_compute_sector(conf, stripe_addr*(new_data_disks),
4769                                     1, &dd_idx, NULL);
4770        last_sector =
4771                raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
4772                                            * new_data_disks - 1),
4773                                     1, &dd_idx, NULL);
4774        if (last_sector >= mddev->dev_sectors)
4775                last_sector = mddev->dev_sectors - 1;
4776        while (first_sector <= last_sector) {
4777                sh = get_active_stripe(conf, first_sector, 1, 0, 1);
4778                set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
4779                set_bit(STRIPE_HANDLE, &sh->state);
4780                release_stripe(sh);
4781                first_sector += STRIPE_SECTORS;
4782        }
4783        /* Now that the sources are clearly marked, we can release
4784         * the destination stripes
4785         */
4786        while (!list_empty(&stripes)) {
4787                sh = list_entry(stripes.next, struct stripe_head, lru);
4788                list_del_init(&sh->lru);
4789                release_stripe(sh);
4790        }
4791        /* If this takes us to the resync_max point where we have to pause,
4792         * then we need to write out the superblock.
4793         */
4794        sector_nr += reshape_sectors;
4795        if ((sector_nr -