linux/drivers/md/raid10.c
<<
>>
Prefs
   1/*
   2 * raid10.c : Multiple Devices driver for Linux
   3 *
   4 * Copyright (C) 2000-2004 Neil Brown
   5 *
   6 * RAID-10 support for md.
   7 *
   8 * Base on code in raid1.c.  See raid1.c for further copyright information.
   9 *
  10 *
  11 * This program is free software; you can redistribute it and/or modify
  12 * it under the terms of the GNU General Public License as published by
  13 * the Free Software Foundation; either version 2, or (at your option)
  14 * any later version.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * (for example /usr/src/linux/COPYING); if not, write to the Free
  18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19 */
  20
  21#include <linux/slab.h>
  22#include <linux/delay.h>
  23#include <linux/blkdev.h>
  24#include <linux/module.h>
  25#include <linux/seq_file.h>
  26#include <linux/ratelimit.h>
  27#include <linux/kthread.h>
  28#include "md.h"
  29#include "raid10.h"
  30#include "raid0.h"
  31#include "bitmap.h"
  32
  33/*
  34 * RAID10 provides a combination of RAID0 and RAID1 functionality.
  35 * The layout of data is defined by
  36 *    chunk_size
  37 *    raid_disks
  38 *    near_copies (stored in low byte of layout)
  39 *    far_copies (stored in second byte of layout)
  40 *    far_offset (stored in bit 16 of layout )
  41 *
  42 * The data to be stored is divided into chunks using chunksize.
  43 * Each device is divided into far_copies sections.
  44 * In each section, chunks are laid out in a style similar to raid0, but
  45 * near_copies copies of each chunk is stored (each on a different drive).
  46 * The starting device for each section is offset near_copies from the starting
  47 * device of the previous section.
  48 * Thus they are (near_copies*far_copies) of each chunk, and each is on a different
  49 * drive.
  50 * near_copies and far_copies must be at least one, and their product is at most
  51 * raid_disks.
  52 *
  53 * If far_offset is true, then the far_copies are handled a bit differently.
  54 * The copies are still in different stripes, but instead of be very far apart
  55 * on disk, there are adjacent stripes.
  56 */
  57
  58/*
  59 * Number of guaranteed r10bios in case of extreme VM load:
  60 */
  61#define NR_RAID10_BIOS 256
  62
  63/* when we get a read error on a read-only array, we redirect to another
  64 * device without failing the first device, or trying to over-write to
  65 * correct the read error.  To keep track of bad blocks on a per-bio
  66 * level, we store IO_BLOCKED in the appropriate 'bios' pointer
  67 */
  68#define IO_BLOCKED ((struct bio *)1)
  69/* When we successfully write to a known bad-block, we need to remove the
  70 * bad-block marking which must be done from process context.  So we record
  71 * the success by setting devs[n].bio to IO_MADE_GOOD
  72 */
  73#define IO_MADE_GOOD ((struct bio *)2)
  74
  75#define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
  76
  77/* When there are this many requests queued to be written by
  78 * the raid10 thread, we become 'congested' to provide back-pressure
  79 * for writeback.
  80 */
  81static int max_queued_requests = 1024;
  82
  83static void allow_barrier(struct r10conf *conf);
  84static void lower_barrier(struct r10conf *conf);
  85static int enough(struct r10conf *conf, int ignore);
  86static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
  87                                int *skipped);
  88static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
  89static void end_reshape_write(struct bio *bio, int error);
  90static void end_reshape(struct r10conf *conf);
  91
  92static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
  93{
  94        struct r10conf *conf = data;
  95        int size = offsetof(struct r10bio, devs[conf->copies]);
  96
  97        /* allocate a r10bio with room for raid_disks entries in the
  98         * bios array */
  99        return kzalloc(size, gfp_flags);
 100}
 101
 102static void r10bio_pool_free(void *r10_bio, void *data)
 103{
 104        kfree(r10_bio);
 105}
 106
 107/* Maximum size of each resync request */
 108#define RESYNC_BLOCK_SIZE (64*1024)
 109#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
 110/* amount of memory to reserve for resync requests */
 111#define RESYNC_WINDOW (1024*1024)
 112/* maximum number of concurrent requests, memory permitting */
 113#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
 114
 115/*
 116 * When performing a resync, we need to read and compare, so
 117 * we need as many pages are there are copies.
 118 * When performing a recovery, we need 2 bios, one for read,
 119 * one for write (we recover only one drive per r10buf)
 120 *
 121 */
 122static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
 123{
 124        struct r10conf *conf = data;
 125        struct page *page;
 126        struct r10bio *r10_bio;
 127        struct bio *bio;
 128        int i, j;
 129        int nalloc;
 130
 131        r10_bio = r10bio_pool_alloc(gfp_flags, conf);
 132        if (!r10_bio)
 133                return NULL;
 134
 135        if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
 136            test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
 137                nalloc = conf->copies; /* resync */
 138        else
 139                nalloc = 2; /* recovery */
 140
 141        /*
 142         * Allocate bios.
 143         */
 144        for (j = nalloc ; j-- ; ) {
 145                bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
 146                if (!bio)
 147                        goto out_free_bio;
 148                r10_bio->devs[j].bio = bio;
 149                if (!conf->have_replacement)
 150                        continue;
 151                bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
 152                if (!bio)
 153                        goto out_free_bio;
 154                r10_bio->devs[j].repl_bio = bio;
 155        }
 156        /*
 157         * Allocate RESYNC_PAGES data pages and attach them
 158         * where needed.
 159         */
 160        for (j = 0 ; j < nalloc; j++) {
 161                struct bio *rbio = r10_bio->devs[j].repl_bio;
 162                bio = r10_bio->devs[j].bio;
 163                for (i = 0; i < RESYNC_PAGES; i++) {
 164                        if (j > 0 && !test_bit(MD_RECOVERY_SYNC,
 165                                               &conf->mddev->recovery)) {
 166                                /* we can share bv_page's during recovery
 167                                 * and reshape */
 168                                struct bio *rbio = r10_bio->devs[0].bio;
 169                                page = rbio->bi_io_vec[i].bv_page;
 170                                get_page(page);
 171                        } else
 172                                page = alloc_page(gfp_flags);
 173                        if (unlikely(!page))
 174                                goto out_free_pages;
 175
 176                        bio->bi_io_vec[i].bv_page = page;
 177                        if (rbio)
 178                                rbio->bi_io_vec[i].bv_page = page;
 179                }
 180        }
 181
 182        return r10_bio;
 183
 184out_free_pages:
 185        for ( ; i > 0 ; i--)
 186                safe_put_page(bio->bi_io_vec[i-1].bv_page);
 187        while (j--)
 188                for (i = 0; i < RESYNC_PAGES ; i++)
 189                        safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
 190        j = 0;
 191out_free_bio:
 192        for ( ; j < nalloc; j++) {
 193                if (r10_bio->devs[j].bio)
 194                        bio_put(r10_bio->devs[j].bio);
 195                if (r10_bio->devs[j].repl_bio)
 196                        bio_put(r10_bio->devs[j].repl_bio);
 197        }
 198        r10bio_pool_free(r10_bio, conf);
 199        return NULL;
 200}
 201
 202static void r10buf_pool_free(void *__r10_bio, void *data)
 203{
 204        int i;
 205        struct r10conf *conf = data;
 206        struct r10bio *r10bio = __r10_bio;
 207        int j;
 208
 209        for (j=0; j < conf->copies; j++) {
 210                struct bio *bio = r10bio->devs[j].bio;
 211                if (bio) {
 212                        for (i = 0; i < RESYNC_PAGES; i++) {
 213                                safe_put_page(bio->bi_io_vec[i].bv_page);
 214                                bio->bi_io_vec[i].bv_page = NULL;
 215                        }
 216                        bio_put(bio);
 217                }
 218                bio = r10bio->devs[j].repl_bio;
 219                if (bio)
 220                        bio_put(bio);
 221        }
 222        r10bio_pool_free(r10bio, conf);
 223}
 224
 225static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
 226{
 227        int i;
 228
 229        for (i = 0; i < conf->copies; i++) {
 230                struct bio **bio = & r10_bio->devs[i].bio;
 231                if (!BIO_SPECIAL(*bio))
 232                        bio_put(*bio);
 233                *bio = NULL;
 234                bio = &r10_bio->devs[i].repl_bio;
 235                if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
 236                        bio_put(*bio);
 237                *bio = NULL;
 238        }
 239}
 240
 241static void free_r10bio(struct r10bio *r10_bio)
 242{
 243        struct r10conf *conf = r10_bio->mddev->private;
 244
 245        put_all_bios(conf, r10_bio);
 246        mempool_free(r10_bio, conf->r10bio_pool);
 247}
 248
 249static void put_buf(struct r10bio *r10_bio)
 250{
 251        struct r10conf *conf = r10_bio->mddev->private;
 252
 253        mempool_free(r10_bio, conf->r10buf_pool);
 254
 255        lower_barrier(conf);
 256}
 257
 258static void reschedule_retry(struct r10bio *r10_bio)
 259{
 260        unsigned long flags;
 261        struct mddev *mddev = r10_bio->mddev;
 262        struct r10conf *conf = mddev->private;
 263
 264        spin_lock_irqsave(&conf->device_lock, flags);
 265        list_add(&r10_bio->retry_list, &conf->retry_list);
 266        conf->nr_queued ++;
 267        spin_unlock_irqrestore(&conf->device_lock, flags);
 268
 269        /* wake up frozen array... */
 270        wake_up(&conf->wait_barrier);
 271
 272        md_wakeup_thread(mddev->thread);
 273}
 274
 275/*
 276 * raid_end_bio_io() is called when we have finished servicing a mirrored
 277 * operation and are ready to return a success/failure code to the buffer
 278 * cache layer.
 279 */
 280static void raid_end_bio_io(struct r10bio *r10_bio)
 281{
 282        struct bio *bio = r10_bio->master_bio;
 283        int done;
 284        struct r10conf *conf = r10_bio->mddev->private;
 285
 286        if (bio->bi_phys_segments) {
 287                unsigned long flags;
 288                spin_lock_irqsave(&conf->device_lock, flags);
 289                bio->bi_phys_segments--;
 290                done = (bio->bi_phys_segments == 0);
 291                spin_unlock_irqrestore(&conf->device_lock, flags);
 292        } else
 293                done = 1;
 294        if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
 295                clear_bit(BIO_UPTODATE, &bio->bi_flags);
 296        if (done) {
 297                bio_endio(bio, 0);
 298                /*
 299                 * Wake up any possible resync thread that waits for the device
 300                 * to go idle.
 301                 */
 302                allow_barrier(conf);
 303        }
 304        free_r10bio(r10_bio);
 305}
 306
 307/*
 308 * Update disk head position estimator based on IRQ completion info.
 309 */
 310static inline void update_head_pos(int slot, struct r10bio *r10_bio)
 311{
 312        struct r10conf *conf = r10_bio->mddev->private;
 313
 314        conf->mirrors[r10_bio->devs[slot].devnum].head_position =
 315                r10_bio->devs[slot].addr + (r10_bio->sectors);
 316}
 317
 318/*
 319 * Find the disk number which triggered given bio
 320 */
 321static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
 322                         struct bio *bio, int *slotp, int *replp)
 323{
 324        int slot;
 325        int repl = 0;
 326
 327        for (slot = 0; slot < conf->copies; slot++) {
 328                if (r10_bio->devs[slot].bio == bio)
 329                        break;
 330                if (r10_bio->devs[slot].repl_bio == bio) {
 331                        repl = 1;
 332                        break;
 333                }
 334        }
 335
 336        BUG_ON(slot == conf->copies);
 337        update_head_pos(slot, r10_bio);
 338
 339        if (slotp)
 340                *slotp = slot;
 341        if (replp)
 342                *replp = repl;
 343        return r10_bio->devs[slot].devnum;
 344}
 345
 346static void raid10_end_read_request(struct bio *bio, int error)
 347{
 348        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 349        struct r10bio *r10_bio = bio->bi_private;
 350        int slot, dev;
 351        struct md_rdev *rdev;
 352        struct r10conf *conf = r10_bio->mddev->private;
 353
 354
 355        slot = r10_bio->read_slot;
 356        dev = r10_bio->devs[slot].devnum;
 357        rdev = r10_bio->devs[slot].rdev;
 358        /*
 359         * this branch is our 'one mirror IO has finished' event handler:
 360         */
 361        update_head_pos(slot, r10_bio);
 362
 363        if (uptodate) {
 364                /*
 365                 * Set R10BIO_Uptodate in our master bio, so that
 366                 * we will return a good error code to the higher
 367                 * levels even if IO on some other mirrored buffer fails.
 368                 *
 369                 * The 'master' represents the composite IO operation to
 370                 * user-side. So if something waits for IO, then it will
 371                 * wait for the 'master' bio.
 372                 */
 373                set_bit(R10BIO_Uptodate, &r10_bio->state);
 374        } else {
 375                /* If all other devices that store this block have
 376                 * failed, we want to return the error upwards rather
 377                 * than fail the last device.  Here we redefine
 378                 * "uptodate" to mean "Don't want to retry"
 379                 */
 380                unsigned long flags;
 381                spin_lock_irqsave(&conf->device_lock, flags);
 382                if (!enough(conf, rdev->raid_disk))
 383                        uptodate = 1;
 384                spin_unlock_irqrestore(&conf->device_lock, flags);
 385        }
 386        if (uptodate) {
 387                raid_end_bio_io(r10_bio);
 388                rdev_dec_pending(rdev, conf->mddev);
 389        } else {
 390                /*
 391                 * oops, read error - keep the refcount on the rdev
 392                 */
 393                char b[BDEVNAME_SIZE];
 394                printk_ratelimited(KERN_ERR
 395                                   "md/raid10:%s: %s: rescheduling sector %llu\n",
 396                                   mdname(conf->mddev),
 397                                   bdevname(rdev->bdev, b),
 398                                   (unsigned long long)r10_bio->sector);
 399                set_bit(R10BIO_ReadError, &r10_bio->state);
 400                reschedule_retry(r10_bio);
 401        }
 402}
 403
 404static void close_write(struct r10bio *r10_bio)
 405{
 406        /* clear the bitmap if all writes complete successfully */
 407        bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
 408                        r10_bio->sectors,
 409                        !test_bit(R10BIO_Degraded, &r10_bio->state),
 410                        0);
 411        md_write_end(r10_bio->mddev);
 412}
 413
 414static void one_write_done(struct r10bio *r10_bio)
 415{
 416        if (atomic_dec_and_test(&r10_bio->remaining)) {
 417                if (test_bit(R10BIO_WriteError, &r10_bio->state))
 418                        reschedule_retry(r10_bio);
 419                else {
 420                        close_write(r10_bio);
 421                        if (test_bit(R10BIO_MadeGood, &r10_bio->state))
 422                                reschedule_retry(r10_bio);
 423                        else
 424                                raid_end_bio_io(r10_bio);
 425                }
 426        }
 427}
 428
 429static void raid10_end_write_request(struct bio *bio, int error)
 430{
 431        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 432        struct r10bio *r10_bio = bio->bi_private;
 433        int dev;
 434        int dec_rdev = 1;
 435        struct r10conf *conf = r10_bio->mddev->private;
 436        int slot, repl;
 437        struct md_rdev *rdev = NULL;
 438
 439        dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
 440
 441        if (repl)
 442                rdev = conf->mirrors[dev].replacement;
 443        if (!rdev) {
 444                smp_rmb();
 445                repl = 0;
 446                rdev = conf->mirrors[dev].rdev;
 447        }
 448        /*
 449         * this branch is our 'one mirror IO has finished' event handler:
 450         */
 451        if (!uptodate) {
 452                if (repl)
 453                        /* Never record new bad blocks to replacement,
 454                         * just fail it.
 455                         */
 456                        md_error(rdev->mddev, rdev);
 457                else {
 458                        set_bit(WriteErrorSeen, &rdev->flags);
 459                        if (!test_and_set_bit(WantReplacement, &rdev->flags))
 460                                set_bit(MD_RECOVERY_NEEDED,
 461                                        &rdev->mddev->recovery);
 462                        set_bit(R10BIO_WriteError, &r10_bio->state);
 463                        dec_rdev = 0;
 464                }
 465        } else {
 466                /*
 467                 * Set R10BIO_Uptodate in our master bio, so that
 468                 * we will return a good error code for to the higher
 469                 * levels even if IO on some other mirrored buffer fails.
 470                 *
 471                 * The 'master' represents the composite IO operation to
 472                 * user-side. So if something waits for IO, then it will
 473                 * wait for the 'master' bio.
 474                 */
 475                sector_t first_bad;
 476                int bad_sectors;
 477
 478                set_bit(R10BIO_Uptodate, &r10_bio->state);
 479
 480                /* Maybe we can clear some bad blocks. */
 481                if (is_badblock(rdev,
 482                                r10_bio->devs[slot].addr,
 483                                r10_bio->sectors,
 484                                &first_bad, &bad_sectors)) {
 485                        bio_put(bio);
 486                        if (repl)
 487                                r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
 488                        else
 489                                r10_bio->devs[slot].bio = IO_MADE_GOOD;
 490                        dec_rdev = 0;
 491                        set_bit(R10BIO_MadeGood, &r10_bio->state);
 492                }
 493        }
 494
 495        /*
 496         *
 497         * Let's see if all mirrored write operations have finished
 498         * already.
 499         */
 500        one_write_done(r10_bio);
 501        if (dec_rdev)
 502                rdev_dec_pending(rdev, conf->mddev);
 503}
 504
 505/*
 506 * RAID10 layout manager
 507 * As well as the chunksize and raid_disks count, there are two
 508 * parameters: near_copies and far_copies.
 509 * near_copies * far_copies must be <= raid_disks.
 510 * Normally one of these will be 1.
 511 * If both are 1, we get raid0.
 512 * If near_copies == raid_disks, we get raid1.
 513 *
 514 * Chunks are laid out in raid0 style with near_copies copies of the
 515 * first chunk, followed by near_copies copies of the next chunk and
 516 * so on.
 517 * If far_copies > 1, then after 1/far_copies of the array has been assigned
 518 * as described above, we start again with a device offset of near_copies.
 519 * So we effectively have another copy of the whole array further down all
 520 * the drives, but with blocks on different drives.
 521 * With this layout, and block is never stored twice on the one device.
 522 *
 523 * raid10_find_phys finds the sector offset of a given virtual sector
 524 * on each device that it is on.
 525 *
 526 * raid10_find_virt does the reverse mapping, from a device and a
 527 * sector offset to a virtual address
 528 */
 529
 530static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
 531{
 532        int n,f;
 533        sector_t sector;
 534        sector_t chunk;
 535        sector_t stripe;
 536        int dev;
 537        int slot = 0;
 538
 539        /* now calculate first sector/dev */
 540        chunk = r10bio->sector >> geo->chunk_shift;
 541        sector = r10bio->sector & geo->chunk_mask;
 542
 543        chunk *= geo->near_copies;
 544        stripe = chunk;
 545        dev = sector_div(stripe, geo->raid_disks);
 546        if (geo->far_offset)
 547                stripe *= geo->far_copies;
 548
 549        sector += stripe << geo->chunk_shift;
 550
 551        /* and calculate all the others */
 552        for (n = 0; n < geo->near_copies; n++) {
 553                int d = dev;
 554                sector_t s = sector;
 555                r10bio->devs[slot].addr = sector;
 556                r10bio->devs[slot].devnum = d;
 557                slot++;
 558
 559                for (f = 1; f < geo->far_copies; f++) {
 560                        d += geo->near_copies;
 561                        if (d >= geo->raid_disks)
 562                                d -= geo->raid_disks;
 563                        s += geo->stride;
 564                        r10bio->devs[slot].devnum = d;
 565                        r10bio->devs[slot].addr = s;
 566                        slot++;
 567                }
 568                dev++;
 569                if (dev >= geo->raid_disks) {
 570                        dev = 0;
 571                        sector += (geo->chunk_mask + 1);
 572                }
 573        }
 574}
 575
 576static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
 577{
 578        struct geom *geo = &conf->geo;
 579
 580        if (conf->reshape_progress != MaxSector &&
 581            ((r10bio->sector >= conf->reshape_progress) !=
 582             conf->mddev->reshape_backwards)) {
 583                set_bit(R10BIO_Previous, &r10bio->state);
 584                geo = &conf->prev;
 585        } else
 586                clear_bit(R10BIO_Previous, &r10bio->state);
 587
 588        __raid10_find_phys(geo, r10bio);
 589}
 590
 591static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
 592{
 593        sector_t offset, chunk, vchunk;
 594        /* Never use conf->prev as this is only called during resync
 595         * or recovery, so reshape isn't happening
 596         */
 597        struct geom *geo = &conf->geo;
 598
 599        offset = sector & geo->chunk_mask;
 600        if (geo->far_offset) {
 601                int fc;
 602                chunk = sector >> geo->chunk_shift;
 603                fc = sector_div(chunk, geo->far_copies);
 604                dev -= fc * geo->near_copies;
 605                if (dev < 0)
 606                        dev += geo->raid_disks;
 607        } else {
 608                while (sector >= geo->stride) {
 609                        sector -= geo->stride;
 610                        if (dev < geo->near_copies)
 611                                dev += geo->raid_disks - geo->near_copies;
 612                        else
 613                                dev -= geo->near_copies;
 614                }
 615                chunk = sector >> geo->chunk_shift;
 616        }
 617        vchunk = chunk * geo->raid_disks + dev;
 618        sector_div(vchunk, geo->near_copies);
 619        return (vchunk << geo->chunk_shift) + offset;
 620}
 621
 622/**
 623 *      raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
 624 *      @q: request queue
 625 *      @bvm: properties of new bio
 626 *      @biovec: the request that could be merged to it.
 627 *
 628 *      Return amount of bytes we can accept at this offset
 629 *      This requires checking for end-of-chunk if near_copies != raid_disks,
 630 *      and for subordinate merge_bvec_fns if merge_check_needed.
 631 */
 632static int raid10_mergeable_bvec(struct request_queue *q,
 633                                 struct bvec_merge_data *bvm,
 634                                 struct bio_vec *biovec)
 635{
 636        struct mddev *mddev = q->queuedata;
 637        struct r10conf *conf = mddev->private;
 638        sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
 639        int max;
 640        unsigned int chunk_sectors;
 641        unsigned int bio_sectors = bvm->bi_size >> 9;
 642        struct geom *geo = &conf->geo;
 643
 644        chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1;
 645        if (conf->reshape_progress != MaxSector &&
 646            ((sector >= conf->reshape_progress) !=
 647             conf->mddev->reshape_backwards))
 648                geo = &conf->prev;
 649
 650        if (geo->near_copies < geo->raid_disks) {
 651                max = (chunk_sectors - ((sector & (chunk_sectors - 1))
 652                                        + bio_sectors)) << 9;
 653                if (max < 0)
 654                        /* bio_add cannot handle a negative return */
 655                        max = 0;
 656                if (max <= biovec->bv_len && bio_sectors == 0)
 657                        return biovec->bv_len;
 658        } else
 659                max = biovec->bv_len;
 660
 661        if (mddev->merge_check_needed) {
 662                struct {
 663                        struct r10bio r10_bio;
 664                        struct r10dev devs[conf->copies];
 665                } on_stack;
 666                struct r10bio *r10_bio = &on_stack.r10_bio;
 667                int s;
 668                if (conf->reshape_progress != MaxSector) {
 669                        /* Cannot give any guidance during reshape */
 670                        if (max <= biovec->bv_len && bio_sectors == 0)
 671                                return biovec->bv_len;
 672                        return 0;
 673                }
 674                r10_bio->sector = sector;
 675                raid10_find_phys(conf, r10_bio);
 676                rcu_read_lock();
 677                for (s = 0; s < conf->copies; s++) {
 678                        int disk = r10_bio->devs[s].devnum;
 679                        struct md_rdev *rdev = rcu_dereference(
 680                                conf->mirrors[disk].rdev);
 681                        if (rdev && !test_bit(Faulty, &rdev->flags)) {
 682                                struct request_queue *q =
 683                                        bdev_get_queue(rdev->bdev);
 684                                if (q->merge_bvec_fn) {
 685                                        bvm->bi_sector = r10_bio->devs[s].addr
 686                                                + rdev->data_offset;
 687                                        bvm->bi_bdev = rdev->bdev;
 688                                        max = min(max, q->merge_bvec_fn(
 689                                                          q, bvm, biovec));
 690                                }
 691                        }
 692                        rdev = rcu_dereference(conf->mirrors[disk].replacement);
 693                        if (rdev && !test_bit(Faulty, &rdev->flags)) {
 694                                struct request_queue *q =
 695                                        bdev_get_queue(rdev->bdev);
 696                                if (q->merge_bvec_fn) {
 697                                        bvm->bi_sector = r10_bio->devs[s].addr
 698                                                + rdev->data_offset;
 699                                        bvm->bi_bdev = rdev->bdev;
 700                                        max = min(max, q->merge_bvec_fn(
 701                                                          q, bvm, biovec));
 702                                }
 703                        }
 704                }
 705                rcu_read_unlock();
 706        }
 707        return max;
 708}
 709
 710/*
 711 * This routine returns the disk from which the requested read should
 712 * be done. There is a per-array 'next expected sequential IO' sector
 713 * number - if this matches on the next IO then we use the last disk.
 714 * There is also a per-disk 'last know head position' sector that is
 715 * maintained from IRQ contexts, both the normal and the resync IO
 716 * completion handlers update this position correctly. If there is no
 717 * perfect sequential match then we pick the disk whose head is closest.
 718 *
 719 * If there are 2 mirrors in the same 2 devices, performance degrades
 720 * because position is mirror, not device based.
 721 *
 722 * The rdev for the device selected will have nr_pending incremented.
 723 */
 724
 725/*
 726 * FIXME: possibly should rethink readbalancing and do it differently
 727 * depending on near_copies / far_copies geometry.
 728 */
 729static struct md_rdev *read_balance(struct r10conf *conf,
 730                                    struct r10bio *r10_bio,
 731                                    int *max_sectors)
 732{
 733        const sector_t this_sector = r10_bio->sector;
 734        int disk, slot;
 735        int sectors = r10_bio->sectors;
 736        int best_good_sectors;
 737        sector_t new_distance, best_dist;
 738        struct md_rdev *best_rdev, *rdev = NULL;
 739        int do_balance;
 740        int best_slot;
 741        struct geom *geo = &conf->geo;
 742
 743        raid10_find_phys(conf, r10_bio);
 744        rcu_read_lock();
 745retry:
 746        sectors = r10_bio->sectors;
 747        best_slot = -1;
 748        best_rdev = NULL;
 749        best_dist = MaxSector;
 750        best_good_sectors = 0;
 751        do_balance = 1;
 752        /*
 753         * Check if we can balance. We can balance on the whole
 754         * device if no resync is going on (recovery is ok), or below
 755         * the resync window. We take the first readable disk when
 756         * above the resync window.
 757         */
 758        if (conf->mddev->recovery_cp < MaxSector
 759            && (this_sector + sectors >= conf->next_resync))
 760                do_balance = 0;
 761
 762        for (slot = 0; slot < conf->copies ; slot++) {
 763                sector_t first_bad;
 764                int bad_sectors;
 765                sector_t dev_sector;
 766
 767                if (r10_bio->devs[slot].bio == IO_BLOCKED)
 768                        continue;
 769                disk = r10_bio->devs[slot].devnum;
 770                rdev = rcu_dereference(conf->mirrors[disk].replacement);
 771                if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
 772                    test_bit(Unmerged, &rdev->flags) ||
 773                    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
 774                        rdev = rcu_dereference(conf->mirrors[disk].rdev);
 775                if (rdev == NULL ||
 776                    test_bit(Faulty, &rdev->flags) ||
 777                    test_bit(Unmerged, &rdev->flags))
 778                        continue;
 779                if (!test_bit(In_sync, &rdev->flags) &&
 780                    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
 781                        continue;
 782
 783                dev_sector = r10_bio->devs[slot].addr;
 784                if (is_badblock(rdev, dev_sector, sectors,
 785                                &first_bad, &bad_sectors)) {
 786                        if (best_dist < MaxSector)
 787                                /* Already have a better slot */
 788                                continue;
 789                        if (first_bad <= dev_sector) {
 790                                /* Cannot read here.  If this is the
 791                                 * 'primary' device, then we must not read
 792                                 * beyond 'bad_sectors' from another device.
 793                                 */
 794                                bad_sectors -= (dev_sector - first_bad);
 795                                if (!do_balance && sectors > bad_sectors)
 796                                        sectors = bad_sectors;
 797                                if (best_good_sectors > sectors)
 798                                        best_good_sectors = sectors;
 799                        } else {
 800                                sector_t good_sectors =
 801                                        first_bad - dev_sector;
 802                                if (good_sectors > best_good_sectors) {
 803                                        best_good_sectors = good_sectors;
 804                                        best_slot = slot;
 805                                        best_rdev = rdev;
 806                                }
 807                                if (!do_balance)
 808                                        /* Must read from here */
 809                                        break;
 810                        }
 811                        continue;
 812                } else
 813                        best_good_sectors = sectors;
 814
 815                if (!do_balance)
 816                        break;
 817
 818                /* This optimisation is debatable, and completely destroys
 819                 * sequential read speed for 'far copies' arrays.  So only
 820                 * keep it for 'near' arrays, and review those later.
 821                 */
 822                if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
 823                        break;
 824
 825                /* for far > 1 always use the lowest address */
 826                if (geo->far_copies > 1)
 827                        new_distance = r10_bio->devs[slot].addr;
 828                else
 829                        new_distance = abs(r10_bio->devs[slot].addr -
 830                                           conf->mirrors[disk].head_position);
 831                if (new_distance < best_dist) {
 832                        best_dist = new_distance;
 833                        best_slot = slot;
 834                        best_rdev = rdev;
 835                }
 836        }
 837        if (slot >= conf->copies) {
 838                slot = best_slot;
 839                rdev = best_rdev;
 840        }
 841
 842        if (slot >= 0) {
 843                atomic_inc(&rdev->nr_pending);
 844                if (test_bit(Faulty, &rdev->flags)) {
 845                        /* Cannot risk returning a device that failed
 846                         * before we inc'ed nr_pending
 847                         */
 848                        rdev_dec_pending(rdev, conf->mddev);
 849                        goto retry;
 850                }
 851                r10_bio->read_slot = slot;
 852        } else
 853                rdev = NULL;
 854        rcu_read_unlock();
 855        *max_sectors = best_good_sectors;
 856
 857        return rdev;
 858}
 859
 860int md_raid10_congested(struct mddev *mddev, int bits)
 861{
 862        struct r10conf *conf = mddev->private;
 863        int i, ret = 0;
 864
 865        if ((bits & (1 << BDI_async_congested)) &&
 866            conf->pending_count >= max_queued_requests)
 867                return 1;
 868
 869        rcu_read_lock();
 870        for (i = 0;
 871             (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
 872                     && ret == 0;
 873             i++) {
 874                struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
 875                if (rdev && !test_bit(Faulty, &rdev->flags)) {
 876                        struct request_queue *q = bdev_get_queue(rdev->bdev);
 877
 878                        ret |= bdi_congested(&q->backing_dev_info, bits);
 879                }
 880        }
 881        rcu_read_unlock();
 882        return ret;
 883}
 884EXPORT_SYMBOL_GPL(md_raid10_congested);
 885
 886static int raid10_congested(void *data, int bits)
 887{
 888        struct mddev *mddev = data;
 889
 890        return mddev_congested(mddev, bits) ||
 891                md_raid10_congested(mddev, bits);
 892}
 893
 894static void flush_pending_writes(struct r10conf *conf)
 895{
 896        /* Any writes that have been queued but are awaiting
 897         * bitmap updates get flushed here.
 898         */
 899        spin_lock_irq(&conf->device_lock);
 900
 901        if (conf->pending_bio_list.head) {
 902                struct bio *bio;
 903                bio = bio_list_get(&conf->pending_bio_list);
 904                conf->pending_count = 0;
 905                spin_unlock_irq(&conf->device_lock);
 906                /* flush any pending bitmap writes to disk
 907                 * before proceeding w/ I/O */
 908                bitmap_unplug(conf->mddev->bitmap);
 909                wake_up(&conf->wait_barrier);
 910
 911                while (bio) { /* submit pending writes */
 912                        struct bio *next = bio->bi_next;
 913                        bio->bi_next = NULL;
 914                        generic_make_request(bio);
 915                        bio = next;
 916                }
 917        } else
 918                spin_unlock_irq(&conf->device_lock);
 919}
 920
 921/* Barriers....
 922 * Sometimes we need to suspend IO while we do something else,
 923 * either some resync/recovery, or reconfigure the array.
 924 * To do this we raise a 'barrier'.
 925 * The 'barrier' is a counter that can be raised multiple times
 926 * to count how many activities are happening which preclude
 927 * normal IO.
 928 * We can only raise the barrier if there is no pending IO.
 929 * i.e. if nr_pending == 0.
 930 * We choose only to raise the barrier if no-one is waiting for the
 931 * barrier to go down.  This means that as soon as an IO request
 932 * is ready, no other operations which require a barrier will start
 933 * until the IO request has had a chance.
 934 *
 935 * So: regular IO calls 'wait_barrier'.  When that returns there
 936 *    is no backgroup IO happening,  It must arrange to call
 937 *    allow_barrier when it has finished its IO.
 938 * backgroup IO calls must call raise_barrier.  Once that returns
 939 *    there is no normal IO happeing.  It must arrange to call
 940 *    lower_barrier when the particular background IO completes.
 941 */
 942
 943static void raise_barrier(struct r10conf *conf, int force)
 944{
 945        BUG_ON(force && !conf->barrier);
 946        spin_lock_irq(&conf->resync_lock);
 947
 948        /* Wait until no block IO is waiting (unless 'force') */
 949        wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
 950                            conf->resync_lock, );
 951
 952        /* block any new IO from starting */
 953        conf->barrier++;
 954
 955        /* Now wait for all pending IO to complete */
 956        wait_event_lock_irq(conf->wait_barrier,
 957                            !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
 958                            conf->resync_lock, );
 959
 960        spin_unlock_irq(&conf->resync_lock);
 961}
 962
 963static void lower_barrier(struct r10conf *conf)
 964{
 965        unsigned long flags;
 966        spin_lock_irqsave(&conf->resync_lock, flags);
 967        conf->barrier--;
 968        spin_unlock_irqrestore(&conf->resync_lock, flags);
 969        wake_up(&conf->wait_barrier);
 970}
 971
 972static void wait_barrier(struct r10conf *conf)
 973{
 974        spin_lock_irq(&conf->resync_lock);
 975        if (conf->barrier) {
 976                conf->nr_waiting++;
 977                /* Wait for the barrier to drop.
 978                 * However if there are already pending
 979                 * requests (preventing the barrier from
 980                 * rising completely), and the
 981                 * pre-process bio queue isn't empty,
 982                 * then don't wait, as we need to empty
 983                 * that queue to get the nr_pending
 984                 * count down.
 985                 */
 986                wait_event_lock_irq(conf->wait_barrier,
 987                                    !conf->barrier ||
 988                                    (conf->nr_pending &&
 989                                     current->bio_list &&
 990                                     !bio_list_empty(current->bio_list)),
 991                                    conf->resync_lock,
 992                        );
 993                conf->nr_waiting--;
 994        }
 995        conf->nr_pending++;
 996        spin_unlock_irq(&conf->resync_lock);
 997}
 998
 999static void allow_barrier(struct r10conf *conf)
1000{
1001        unsigned long flags;
1002        spin_lock_irqsave(&conf->resync_lock, flags);
1003        conf->nr_pending--;
1004        spin_unlock_irqrestore(&conf->resync_lock, flags);
1005        wake_up(&conf->wait_barrier);
1006}
1007
1008static void freeze_array(struct r10conf *conf)
1009{
1010        /* stop syncio and normal IO and wait for everything to
1011         * go quiet.
1012         * We increment barrier and nr_waiting, and then
1013         * wait until nr_pending match nr_queued+1
1014         * This is called in the context of one normal IO request
1015         * that has failed. Thus any sync request that might be pending
1016         * will be blocked by nr_pending, and we need to wait for
1017         * pending IO requests to complete or be queued for re-try.
1018         * Thus the number queued (nr_queued) plus this request (1)
1019         * must match the number of pending IOs (nr_pending) before
1020         * we continue.
1021         */
1022        spin_lock_irq(&conf->resync_lock);
1023        conf->barrier++;
1024        conf->nr_waiting++;
1025        wait_event_lock_irq(conf->wait_barrier,
1026                            conf->nr_pending == conf->nr_queued+1,
1027                            conf->resync_lock,
1028                            flush_pending_writes(conf));
1029
1030        spin_unlock_irq(&conf->resync_lock);
1031}
1032
1033static void unfreeze_array(struct r10conf *conf)
1034{
1035        /* reverse the effect of the freeze */
1036        spin_lock_irq(&conf->resync_lock);
1037        conf->barrier--;
1038        conf->nr_waiting--;
1039        wake_up(&conf->wait_barrier);
1040        spin_unlock_irq(&conf->resync_lock);
1041}
1042
1043static sector_t choose_data_offset(struct r10bio *r10_bio,
1044                                   struct md_rdev *rdev)
1045{
1046        if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1047            test_bit(R10BIO_Previous, &r10_bio->state))
1048                return rdev->data_offset;
1049        else
1050                return rdev->new_data_offset;
1051}
1052
1053static void make_request(struct mddev *mddev, struct bio * bio)
1054{
1055        struct r10conf *conf = mddev->private;
1056        struct r10bio *r10_bio;
1057        struct bio *read_bio;
1058        int i;
1059        sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1060        int chunk_sects = chunk_mask + 1;
1061        const int rw = bio_data_dir(bio);
1062        const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
1063        const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
1064        unsigned long flags;
1065        struct md_rdev *blocked_rdev;
1066        int sectors_handled;
1067        int max_sectors;
1068        int sectors;
1069
1070        if (unlikely(bio->bi_rw & REQ_FLUSH)) {
1071                md_flush_request(mddev, bio);
1072                return;
1073        }
1074
1075        /* If this request crosses a chunk boundary, we need to
1076         * split it.  This will only happen for 1 PAGE (or less) requests.
1077         */
1078        if (unlikely((bio->bi_sector & chunk_mask) + (bio->bi_size >> 9)
1079                     > chunk_sects
1080                     && (conf->geo.near_copies < conf->geo.raid_disks
1081                         || conf->prev.near_copies < conf->prev.raid_disks))) {
1082                struct bio_pair *bp;
1083                /* Sanity check -- queue functions should prevent this happening */
1084                if (bio->bi_vcnt != 1 ||
1085                    bio->bi_idx != 0)
1086                        goto bad_map;
1087                /* This is a one page bio that upper layers
1088                 * refuse to split for us, so we need to split it.
1089                 */
1090                bp = bio_split(bio,
1091                               chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
1092
1093                /* Each of these 'make_request' calls will call 'wait_barrier'.
1094                 * If the first succeeds but the second blocks due to the resync
1095                 * thread raising the barrier, we will deadlock because the
1096                 * IO to the underlying device will be queued in generic_make_request
1097                 * and will never complete, so will never reduce nr_pending.
1098                 * So increment nr_waiting here so no new raise_barriers will
1099                 * succeed, and so the second wait_barrier cannot block.
1100                 */
1101                spin_lock_irq(&conf->resync_lock);
1102                conf->nr_waiting++;
1103                spin_unlock_irq(&conf->resync_lock);
1104
1105                make_request(mddev, &bp->bio1);
1106                make_request(mddev, &bp->bio2);
1107
1108                spin_lock_irq(&conf->resync_lock);
1109                conf->nr_waiting--;
1110                wake_up(&conf->wait_barrier);
1111                spin_unlock_irq(&conf->resync_lock);
1112
1113                bio_pair_release(bp);
1114                return;
1115        bad_map:
1116                printk("md/raid10:%s: make_request bug: can't convert block across chunks"
1117                       " or bigger than %dk %llu %d\n", mdname(mddev), chunk_sects/2,
1118                       (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
1119
1120                bio_io_error(bio);
1121                return;
1122        }
1123
1124        md_write_start(mddev, bio);
1125
1126        /*
1127         * Register the new request and wait if the reconstruction
1128         * thread has put up a bar for new requests.
1129         * Continue immediately if no resync is active currently.
1130         */
1131        wait_barrier(conf);
1132
1133        sectors = bio->bi_size >> 9;
1134        while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1135            bio->bi_sector < conf->reshape_progress &&
1136            bio->bi_sector + sectors > conf->reshape_progress) {
1137                /* IO spans the reshape position.  Need to wait for
1138                 * reshape to pass
1139                 */
1140                allow_barrier(conf);
1141                wait_event(conf->wait_barrier,
1142                           conf->reshape_progress <= bio->bi_sector ||
1143                           conf->reshape_progress >= bio->bi_sector + sectors);
1144                wait_barrier(conf);
1145        }
1146        if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1147            bio_data_dir(bio) == WRITE &&
1148            (mddev->reshape_backwards
1149             ? (bio->bi_sector < conf->reshape_safe &&
1150                bio->bi_sector + sectors > conf->reshape_progress)
1151             : (bio->bi_sector + sectors > conf->reshape_safe &&
1152                bio->bi_sector < conf->reshape_progress))) {
1153                /* Need to update reshape_position in metadata */
1154                mddev->reshape_position = conf->reshape_progress;
1155                set_bit(MD_CHANGE_DEVS, &mddev->flags);
1156                set_bit(MD_CHANGE_PENDING, &mddev->flags);
1157                md_wakeup_thread(mddev->thread);
1158                wait_event(mddev->sb_wait,
1159                           !test_bit(MD_CHANGE_PENDING, &mddev->flags));
1160
1161                conf->reshape_safe = mddev->reshape_position;
1162        }
1163
1164        r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1165
1166        r10_bio->master_bio = bio;
1167        r10_bio->sectors = sectors;
1168
1169        r10_bio->mddev = mddev;
1170        r10_bio->sector = bio->bi_sector;
1171        r10_bio->state = 0;
1172
1173        /* We might need to issue multiple reads to different
1174         * devices if there are bad blocks around, so we keep
1175         * track of the number of reads in bio->bi_phys_segments.
1176         * If this is 0, there is only one r10_bio and no locking
1177         * will be needed when the request completes.  If it is
1178         * non-zero, then it is the number of not-completed requests.
1179         */
1180        bio->bi_phys_segments = 0;
1181        clear_bit(BIO_SEG_VALID, &bio->bi_flags);
1182
1183        if (rw == READ) {
1184                /*
1185                 * read balancing logic:
1186                 */
1187                struct md_rdev *rdev;
1188                int slot;
1189
1190read_again:
1191                rdev = read_balance(conf, r10_bio, &max_sectors);
1192                if (!rdev) {
1193                        raid_end_bio_io(r10_bio);
1194                        return;
1195                }
1196                slot = r10_bio->read_slot;
1197
1198                read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1199                md_trim_bio(read_bio, r10_bio->sector - bio->bi_sector,
1200                            max_sectors);
1201
1202                r10_bio->devs[slot].bio = read_bio;
1203                r10_bio->devs[slot].rdev = rdev;
1204
1205                read_bio->bi_sector = r10_bio->devs[slot].addr +
1206                        choose_data_offset(r10_bio, rdev);
1207                read_bio->bi_bdev = rdev->bdev;
1208                read_bio->bi_end_io = raid10_end_read_request;
1209                read_bio->bi_rw = READ | do_sync;
1210                read_bio->bi_private = r10_bio;
1211
1212                if (max_sectors < r10_bio->sectors) {
1213                        /* Could not read all from this device, so we will
1214                         * need another r10_bio.
1215                         */
1216                        sectors_handled = (r10_bio->sectors + max_sectors
1217                                           - bio->bi_sector);
1218                        r10_bio->sectors = max_sectors;
1219                        spin_lock_irq(&conf->device_lock);
1220                        if (bio->bi_phys_segments == 0)
1221                                bio->bi_phys_segments = 2;
1222                        else
1223                                bio->bi_phys_segments++;
1224                        spin_unlock(&conf->device_lock);
1225                        /* Cannot call generic_make_request directly
1226                         * as that will be queued in __generic_make_request
1227                         * and subsequent mempool_alloc might block
1228                         * waiting for it.  so hand bio over to raid10d.
1229                         */
1230                        reschedule_retry(r10_bio);
1231
1232                        r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1233
1234                        r10_bio->master_bio = bio;
1235                        r10_bio->sectors = ((bio->bi_size >> 9)
1236                                            - sectors_handled);
1237                        r10_bio->state = 0;
1238                        r10_bio->mddev = mddev;
1239                        r10_bio->sector = bio->bi_sector + sectors_handled;
1240                        goto read_again;
1241                } else
1242                        generic_make_request(read_bio);
1243                return;
1244        }
1245
1246        /*
1247         * WRITE:
1248         */
1249        if (conf->pending_count >= max_queued_requests) {
1250                md_wakeup_thread(mddev->thread);
1251                wait_event(conf->wait_barrier,
1252                           conf->pending_count < max_queued_requests);
1253        }
1254        /* first select target devices under rcu_lock and
1255         * inc refcount on their rdev.  Record them by setting
1256         * bios[x] to bio
1257         * If there are known/acknowledged bad blocks on any device
1258         * on which we have seen a write error, we want to avoid
1259         * writing to those blocks.  This potentially requires several
1260         * writes to write around the bad blocks.  Each set of writes
1261         * gets its own r10_bio with a set of bios attached.  The number
1262         * of r10_bios is recored in bio->bi_phys_segments just as with
1263         * the read case.
1264         */
1265
1266        r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
1267        raid10_find_phys(conf, r10_bio);
1268retry_write:
1269        blocked_rdev = NULL;
1270        rcu_read_lock();
1271        max_sectors = r10_bio->sectors;
1272
1273        for (i = 0;  i < conf->copies; i++) {
1274                int d = r10_bio->devs[i].devnum;
1275                struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1276                struct md_rdev *rrdev = rcu_dereference(
1277                        conf->mirrors[d].replacement);
1278                if (rdev == rrdev)
1279                        rrdev = NULL;
1280                if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1281                        atomic_inc(&rdev->nr_pending);
1282                        blocked_rdev = rdev;
1283                        break;
1284                }
1285                if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
1286                        atomic_inc(&rrdev->nr_pending);
1287                        blocked_rdev = rrdev;
1288                        break;
1289                }
1290                if (rdev && (test_bit(Faulty, &rdev->flags)
1291                             || test_bit(Unmerged, &rdev->flags)))
1292                        rdev = NULL;
1293                if (rrdev && (test_bit(Faulty, &rrdev->flags)
1294                              || test_bit(Unmerged, &rrdev->flags)))
1295                        rrdev = NULL;
1296
1297                r10_bio->devs[i].bio = NULL;
1298                r10_bio->devs[i].repl_bio = NULL;
1299
1300                if (!rdev && !rrdev) {
1301                        set_bit(R10BIO_Degraded, &r10_bio->state);
1302                        continue;
1303                }
1304                if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1305                        sector_t first_bad;
1306                        sector_t dev_sector = r10_bio->devs[i].addr;
1307                        int bad_sectors;
1308                        int is_bad;
1309
1310                        is_bad = is_badblock(rdev, dev_sector,
1311                                             max_sectors,
1312                                             &first_bad, &bad_sectors);
1313                        if (is_bad < 0) {
1314                                /* Mustn't write here until the bad block
1315                                 * is acknowledged
1316                                 */
1317                                atomic_inc(&rdev->nr_pending);
1318                                set_bit(BlockedBadBlocks, &rdev->flags);
1319                                blocked_rdev = rdev;
1320                                break;
1321                        }
1322                        if (is_bad && first_bad <= dev_sector) {
1323                                /* Cannot write here at all */
1324                                bad_sectors -= (dev_sector - first_bad);
1325                                if (bad_sectors < max_sectors)
1326                                        /* Mustn't write more than bad_sectors
1327                                         * to other devices yet
1328                                         */
1329                                        max_sectors = bad_sectors;
1330                                /* We don't set R10BIO_Degraded as that
1331                                 * only applies if the disk is missing,
1332                                 * so it might be re-added, and we want to
1333                                 * know to recover this chunk.
1334                                 * In this case the device is here, and the
1335                                 * fact that this chunk is not in-sync is
1336                                 * recorded in the bad block log.
1337                                 */
1338                                continue;
1339                        }
1340                        if (is_bad) {
1341                                int good_sectors = first_bad - dev_sector;
1342                                if (good_sectors < max_sectors)
1343                                        max_sectors = good_sectors;
1344                        }
1345                }
1346                if (rdev) {
1347                        r10_bio->devs[i].bio = bio;
1348                        atomic_inc(&rdev->nr_pending);
1349                }
1350                if (rrdev) {
1351                        r10_bio->devs[i].repl_bio = bio;
1352                        atomic_inc(&rrdev->nr_pending);
1353                }
1354        }
1355        rcu_read_unlock();
1356
1357        if (unlikely(blocked_rdev)) {
1358                /* Have to wait for this device to get unblocked, then retry */
1359                int j;
1360                int d;
1361
1362                for (j = 0; j < i; j++) {
1363                        if (r10_bio->devs[j].bio) {
1364                                d = r10_bio->devs[j].devnum;
1365                                rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1366                        }
1367                        if (r10_bio->devs[j].repl_bio) {
1368                                struct md_rdev *rdev;
1369                                d = r10_bio->devs[j].devnum;
1370                                rdev = conf->mirrors[d].replacement;
1371                                if (!rdev) {
1372                                        /* Race with remove_disk */
1373                                        smp_mb();
1374                                        rdev = conf->mirrors[d].rdev;
1375                                }
1376                                rdev_dec_pending(rdev, mddev);
1377                        }
1378                }
1379                allow_barrier(conf);
1380                md_wait_for_blocked_rdev(blocked_rdev, mddev);
1381                wait_barrier(conf);
1382                goto retry_write;
1383        }
1384
1385        if (max_sectors < r10_bio->sectors) {
1386                /* We are splitting this into multiple parts, so
1387                 * we need to prepare for allocating another r10_bio.
1388                 */
1389                r10_bio->sectors = max_sectors;
1390                spin_lock_irq(&conf->device_lock);
1391                if (bio->bi_phys_segments == 0)
1392                        bio->bi_phys_segments = 2;
1393                else
1394                        bio->bi_phys_segments++;
1395                spin_unlock_irq(&conf->device_lock);
1396        }
1397        sectors_handled = r10_bio->sector + max_sectors - bio->bi_sector;
1398
1399        atomic_set(&r10_bio->remaining, 1);
1400        bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1401
1402        for (i = 0; i < conf->copies; i++) {
1403                struct bio *mbio;
1404                int d = r10_bio->devs[i].devnum;
1405
1406                if (r10_bio->devs[i].bio) {
1407                        struct md_rdev *rdev = conf->mirrors[d].rdev;
1408                        mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1409                        md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1410                                    max_sectors);
1411                        r10_bio->devs[i].bio = mbio;
1412
1413                        mbio->bi_sector = (r10_bio->devs[i].addr +
1414                                           choose_data_offset(r10_bio, rdev));
1415                        mbio->bi_bdev = rdev->bdev;
1416                        mbio->bi_end_io = raid10_end_write_request;
1417                        mbio->bi_rw = WRITE | do_sync | do_fua;
1418                        mbio->bi_private = r10_bio;
1419
1420                        atomic_inc(&r10_bio->remaining);
1421                        spin_lock_irqsave(&conf->device_lock, flags);
1422                        bio_list_add(&conf->pending_bio_list, mbio);
1423                        conf->pending_count++;
1424                        spin_unlock_irqrestore(&conf->device_lock, flags);
1425                        if (!mddev_check_plugged(mddev))
1426                                md_wakeup_thread(mddev->thread);
1427                }
1428
1429                if (r10_bio->devs[i].repl_bio) {
1430                        struct md_rdev *rdev = conf->mirrors[d].replacement;
1431                        if (rdev == NULL) {
1432                                /* Replacement just got moved to main 'rdev' */
1433                                smp_mb();
1434                                rdev = conf->mirrors[d].rdev;
1435                        }
1436                        mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1437                        md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1438                                    max_sectors);
1439                        r10_bio->devs[i].repl_bio = mbio;
1440
1441                        mbio->bi_sector = (r10_bio->devs[i].addr +
1442                                           choose_data_offset(r10_bio, rdev));
1443                        mbio->bi_bdev = rdev->bdev;
1444                        mbio->bi_end_io = raid10_end_write_request;
1445                        mbio->bi_rw = WRITE | do_sync | do_fua;
1446                        mbio->bi_private = r10_bio;
1447
1448                        atomic_inc(&r10_bio->remaining);
1449                        spin_lock_irqsave(&conf->device_lock, flags);
1450                        bio_list_add(&conf->pending_bio_list, mbio);
1451                        conf->pending_count++;
1452                        spin_unlock_irqrestore(&conf->device_lock, flags);
1453                        if (!mddev_check_plugged(mddev))
1454                                md_wakeup_thread(mddev->thread);
1455                }
1456        }
1457
1458        /* Don't remove the bias on 'remaining' (one_write_done) until
1459         * after checking if we need to go around again.
1460         */
1461
1462        if (sectors_handled < (bio->bi_size >> 9)) {
1463                one_write_done(r10_bio);
1464                /* We need another r10_bio.  It has already been counted
1465                 * in bio->bi_phys_segments.
1466                 */
1467                r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1468
1469                r10_bio->master_bio = bio;
1470                r10_bio->sectors = (bio->bi_size >> 9) - sectors_handled;
1471
1472                r10_bio->mddev = mddev;
1473                r10_bio->sector = bio->bi_sector + sectors_handled;
1474                r10_bio->state = 0;
1475                goto retry_write;
1476        }
1477        one_write_done(r10_bio);
1478
1479        /* In case raid10d snuck in to freeze_array */
1480        wake_up(&conf->wait_barrier);
1481}
1482
1483static void status(struct seq_file *seq, struct mddev *mddev)
1484{
1485        struct r10conf *conf = mddev->private;
1486        int i;
1487
1488        if (conf->geo.near_copies < conf->geo.raid_disks)
1489                seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1490        if (conf->geo.near_copies > 1)
1491                seq_printf(seq, " %d near-copies", conf->geo.near_copies);
1492        if (conf->geo.far_copies > 1) {
1493                if (conf->geo.far_offset)
1494                        seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1495                else
1496                        seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1497        }
1498        seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
1499                                        conf->geo.raid_disks - mddev->degraded);
1500        for (i = 0; i < conf->geo.raid_disks; i++)
1501                seq_printf(seq, "%s",
1502                              conf->mirrors[i].rdev &&
1503                              test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
1504        seq_printf(seq, "]");
1505}
1506
1507/* check if there are enough drives for
1508 * every block to appear on atleast one.
1509 * Don't consider the device numbered 'ignore'
1510 * as we might be about to remove it.
1511 */
1512static int _enough(struct r10conf *conf, struct geom *geo, int ignore)
1513{
1514        int first = 0;
1515
1516        do {
1517                int n = conf->copies;
1518                int cnt = 0;
1519                int this = first;
1520                while (n--) {
1521                        if (conf->mirrors[this].rdev &&
1522                            this != ignore)
1523                                cnt++;
1524                        this = (this+1) % geo->raid_disks;
1525                }
1526                if (cnt == 0)
1527                        return 0;
1528                first = (first + geo->near_copies) % geo->raid_disks;
1529        } while (first != 0);
1530        return 1;
1531}
1532
1533static int enough(struct r10conf *conf, int ignore)
1534{
1535        return _enough(conf, &conf->geo, ignore) &&
1536                _enough(conf, &conf->prev, ignore);
1537}
1538
1539static void error(struct mddev *mddev, struct md_rdev *rdev)
1540{
1541        char b[BDEVNAME_SIZE];
1542        struct r10conf *conf = mddev->private;
1543
1544        /*
1545         * If it is not operational, then we have already marked it as dead
1546         * else if it is the last working disks, ignore the error, let the
1547         * next level up know.
1548         * else mark the drive as failed
1549         */
1550        if (test_bit(In_sync, &rdev->flags)
1551            && !enough(conf, rdev->raid_disk))
1552                /*
1553                 * Don't fail the drive, just return an IO error.
1554                 */
1555                return;
1556        if (test_and_clear_bit(In_sync, &rdev->flags)) {
1557                unsigned long flags;
1558                spin_lock_irqsave(&conf->device_lock, flags);
1559                mddev->degraded++;
1560                spin_unlock_irqrestore(&conf->device_lock, flags);
1561                /*
1562                 * if recovery is running, make sure it aborts.
1563                 */
1564                set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1565        }
1566        set_bit(Blocked, &rdev->flags);
1567        set_bit(Faulty, &rdev->flags);
1568        set_bit(MD_CHANGE_DEVS, &mddev->flags);
1569        printk(KERN_ALERT
1570               "md/raid10:%s: Disk failure on %s, disabling device.\n"
1571               "md/raid10:%s: Operation continuing on %d devices.\n",
1572               mdname(mddev), bdevname(rdev->bdev, b),
1573               mdname(mddev), conf->geo.raid_disks - mddev->degraded);
1574}
1575
1576static void print_conf(struct r10conf *conf)
1577{
1578        int i;
1579        struct raid10_info *tmp;
1580
1581        printk(KERN_DEBUG "RAID10 conf printout:\n");
1582        if (!conf) {
1583                printk(KERN_DEBUG "(!conf)\n");
1584                return;
1585        }
1586        printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
1587                conf->geo.raid_disks);
1588
1589        for (i = 0; i < conf->geo.raid_disks; i++) {
1590                char b[BDEVNAME_SIZE];
1591                tmp = conf->mirrors + i;
1592                if (tmp->rdev)
1593                        printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1594                                i, !test_bit(In_sync, &tmp->rdev->flags),
1595                                !test_bit(Faulty, &tmp->rdev->flags),
1596                                bdevname(tmp->rdev->bdev,b));
1597        }
1598}
1599
1600static void close_sync(struct r10conf *conf)
1601{
1602        wait_barrier(conf);
1603        allow_barrier(conf);
1604
1605        mempool_destroy(conf->r10buf_pool);
1606        conf->r10buf_pool = NULL;
1607}
1608
1609static int raid10_spare_active(struct mddev *mddev)
1610{
1611        int i;
1612        struct r10conf *conf = mddev->private;
1613        struct raid10_info *tmp;
1614        int count = 0;
1615        unsigned long flags;
1616
1617        /*
1618         * Find all non-in_sync disks within the RAID10 configuration
1619         * and mark them in_sync
1620         */
1621        for (i = 0; i < conf->geo.raid_disks; i++) {
1622                tmp = conf->mirrors + i;
1623                if (tmp->replacement
1624                    && tmp->replacement->recovery_offset == MaxSector
1625                    && !test_bit(Faulty, &tmp->replacement->flags)
1626                    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
1627                        /* Replacement has just become active */
1628                        if (!tmp->rdev
1629                            || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
1630                                count++;
1631                        if (tmp->rdev) {
1632                                /* Replaced device not technically faulty,
1633                                 * but we need to be sure it gets removed
1634                                 * and never re-added.
1635                                 */
1636                                set_bit(Faulty, &tmp->rdev->flags);
1637                                sysfs_notify_dirent_safe(
1638                                        tmp->rdev->sysfs_state);
1639                        }
1640                        sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
1641                } else if (tmp->rdev
1642                           && !test_bit(Faulty, &tmp->rdev->flags)
1643                           && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1644                        count++;
1645                        sysfs_notify_dirent(tmp->rdev->sysfs_state);
1646                }
1647        }
1648        spin_lock_irqsave(&conf->device_lock, flags);
1649        mddev->degraded -= count;
1650        spin_unlock_irqrestore(&conf->device_lock, flags);
1651
1652        print_conf(conf);
1653        return count;
1654}
1655
1656
1657static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1658{
1659        struct r10conf *conf = mddev->private;
1660        int err = -EEXIST;
1661        int mirror;
1662        int first = 0;
1663        int last = conf->geo.raid_disks - 1;
1664        struct request_queue *q = bdev_get_queue(rdev->bdev);
1665
1666        if (mddev->recovery_cp < MaxSector)
1667                /* only hot-add to in-sync arrays, as recovery is
1668                 * very different from resync
1669                 */
1670                return -EBUSY;
1671        if (rdev->saved_raid_disk < 0 && !_enough(conf, &conf->prev, -1))
1672                return -EINVAL;
1673
1674        if (rdev->raid_disk >= 0)
1675                first = last = rdev->raid_disk;
1676
1677        if (q->merge_bvec_fn) {
1678                set_bit(Unmerged, &rdev->flags);
1679                mddev->merge_check_needed = 1;
1680        }
1681
1682        if (rdev->saved_raid_disk >= first &&
1683            conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1684                mirror = rdev->saved_raid_disk;
1685        else
1686                mirror = first;
1687        for ( ; mirror <= last ; mirror++) {
1688                struct raid10_info *p = &conf->mirrors[mirror];
1689                if (p->recovery_disabled == mddev->recovery_disabled)
1690                        continue;
1691                if (p->rdev) {
1692                        if (!test_bit(WantReplacement, &p->rdev->flags) ||
1693                            p->replacement != NULL)
1694                                continue;
1695                        clear_bit(In_sync, &rdev->flags);
1696                        set_bit(Replacement, &rdev->flags);
1697                        rdev->raid_disk = mirror;
1698                        err = 0;
1699                        disk_stack_limits(mddev->gendisk, rdev->bdev,
1700                                          rdev->data_offset << 9);
1701                        conf->fullsync = 1;
1702                        rcu_assign_pointer(p->replacement, rdev);
1703                        break;
1704                }
1705
1706                disk_stack_limits(mddev->gendisk, rdev->bdev,
1707                                  rdev->data_offset << 9);
1708
1709                p->head_position = 0;
1710                p->recovery_disabled = mddev->recovery_disabled - 1;
1711                rdev->raid_disk = mirror;
1712                err = 0;
1713                if (rdev->saved_raid_disk != mirror)
1714                        conf->fullsync = 1;
1715                rcu_assign_pointer(p->rdev, rdev);
1716                break;
1717        }
1718        if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
1719                /* Some requests might not have seen this new
1720                 * merge_bvec_fn.  We must wait for them to complete
1721                 * before merging the device fully.
1722                 * First we make sure any code which has tested
1723                 * our function has submitted the request, then
1724                 * we wait for all outstanding requests to complete.
1725                 */
1726                synchronize_sched();
1727                raise_barrier(conf, 0);
1728                lower_barrier(conf);
1729                clear_bit(Unmerged, &rdev->flags);
1730        }
1731        md_integrity_add_rdev(rdev, mddev);
1732        print_conf(conf);
1733        return err;
1734}
1735
1736static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
1737{
1738        struct r10conf *conf = mddev->private;
1739        int err = 0;
1740        int number = rdev->raid_disk;
1741        struct md_rdev **rdevp;
1742        struct raid10_info *p = conf->mirrors + number;
1743
1744        print_conf(conf);
1745        if (rdev == p->rdev)
1746                rdevp = &p->rdev;
1747        else if (rdev == p->replacement)
1748                rdevp = &p->replacement;
1749        else
1750                return 0;
1751
1752        if (test_bit(In_sync, &rdev->flags) ||
1753            atomic_read(&rdev->nr_pending)) {
1754                err = -EBUSY;
1755                goto abort;
1756        }
1757        /* Only remove faulty devices if recovery
1758         * is not possible.
1759         */
1760        if (!test_bit(Faulty, &rdev->flags) &&
1761            mddev->recovery_disabled != p->recovery_disabled &&
1762            (!p->replacement || p->replacement == rdev) &&
1763            number < conf->geo.raid_disks &&
1764            enough(conf, -1)) {
1765                err = -EBUSY;
1766                goto abort;
1767        }
1768        *rdevp = NULL;
1769        synchronize_rcu();
1770        if (atomic_read(&rdev->nr_pending)) {
1771                /* lost the race, try later */
1772                err = -EBUSY;
1773                *rdevp = rdev;
1774                goto abort;
1775        } else if (p->replacement) {
1776                /* We must have just cleared 'rdev' */
1777                p->rdev = p->replacement;
1778                clear_bit(Replacement, &p->replacement->flags);
1779                smp_mb(); /* Make sure other CPUs may see both as identical
1780                           * but will never see neither -- if they are careful.
1781                           */
1782                p->replacement = NULL;
1783                clear_bit(WantReplacement, &rdev->flags);
1784        } else
1785                /* We might have just remove the Replacement as faulty
1786                 * Clear the flag just in case
1787                 */
1788                clear_bit(WantReplacement, &rdev->flags);
1789
1790        err = md_integrity_register(mddev);
1791
1792abort:
1793
1794        print_conf(conf);
1795        return err;
1796}
1797
1798
1799static void end_sync_read(struct bio *bio, int error)
1800{
1801        struct r10bio *r10_bio = bio->bi_private;
1802        struct r10conf *conf = r10_bio->mddev->private;
1803        int d;
1804
1805        if (bio == r10_bio->master_bio) {
1806                /* this is a reshape read */
1807                d = r10_bio->read_slot; /* really the read dev */
1808        } else
1809                d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1810
1811        if (test_bit(BIO_UPTODATE, &bio->bi_flags))
1812                set_bit(R10BIO_Uptodate, &r10_bio->state);
1813        else
1814                /* The write handler will notice the lack of
1815                 * R10BIO_Uptodate and record any errors etc
1816                 */
1817                atomic_add(r10_bio->sectors,
1818                           &conf->mirrors[d].rdev->corrected_errors);
1819
1820        /* for reconstruct, we always reschedule after a read.
1821         * for resync, only after all reads
1822         */
1823        rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
1824        if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
1825            atomic_dec_and_test(&r10_bio->remaining)) {
1826                /* we have read all the blocks,
1827                 * do the comparison in process context in raid10d
1828                 */
1829                reschedule_retry(r10_bio);
1830        }
1831}
1832
1833static void end_sync_request(struct r10bio *r10_bio)
1834{
1835        struct mddev *mddev = r10_bio->mddev;
1836
1837        while (atomic_dec_and_test(&r10_bio->remaining)) {
1838                if (r10_bio->master_bio == NULL) {
1839                        /* the primary of several recovery bios */
1840                        sector_t s = r10_bio->sectors;
1841                        if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1842                            test_bit(R10BIO_WriteError, &r10_bio->state))
1843                                reschedule_retry(r10_bio);
1844                        else
1845                                put_buf(r10_bio);
1846                        md_done_sync(mddev, s, 1);
1847                        break;
1848                } else {
1849                        struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1850                        if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1851                            test_bit(R10BIO_WriteError, &r10_bio->state))
1852                                reschedule_retry(r10_bio);
1853                        else
1854                                put_buf(r10_bio);
1855                        r10_bio = r10_bio2;
1856                }
1857        }
1858}
1859
1860static void end_sync_write(struct bio *bio, int error)
1861{
1862        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1863        struct r10bio *r10_bio = bio->bi_private;
1864        struct mddev *mddev = r10_bio->mddev;
1865        struct r10conf *conf = mddev->private;
1866        int d;
1867        sector_t first_bad;
1868        int bad_sectors;
1869        int slot;
1870        int repl;
1871        struct md_rdev *rdev = NULL;
1872
1873        d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
1874        if (repl)
1875                rdev = conf->mirrors[d].replacement;
1876        else
1877                rdev = conf->mirrors[d].rdev;
1878
1879        if (!uptodate) {
1880                if (repl)
1881                        md_error(mddev, rdev);
1882                else {
1883                        set_bit(WriteErrorSeen, &rdev->flags);
1884                        if (!test_and_set_bit(WantReplacement, &rdev->flags))
1885                                set_bit(MD_RECOVERY_NEEDED,
1886                                        &rdev->mddev->recovery);
1887                        set_bit(R10BIO_WriteError, &r10_bio->state);
1888                }
1889        } else if (is_badblock(rdev,
1890                             r10_bio->devs[slot].addr,
1891                             r10_bio->sectors,
1892                             &first_bad, &bad_sectors))
1893                set_bit(R10BIO_MadeGood, &r10_bio->state);
1894
1895        rdev_dec_pending(rdev, mddev);
1896
1897        end_sync_request(r10_bio);
1898}
1899
1900/*
1901 * Note: sync and recover and handled very differently for raid10
1902 * This code is for resync.
1903 * For resync, we read through virtual addresses and read all blocks.
1904 * If there is any error, we schedule a write.  The lowest numbered
1905 * drive is authoritative.
1906 * However requests come for physical address, so we need to map.
1907 * For every physical address there are raid_disks/copies virtual addresses,
1908 * which is always are least one, but is not necessarly an integer.
1909 * This means that a physical address can span multiple chunks, so we may
1910 * have to submit multiple io requests for a single sync request.
1911 */
1912/*
1913 * We check if all blocks are in-sync and only write to blocks that
1914 * aren't in sync
1915 */
1916static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
1917{
1918        struct r10conf *conf = mddev->private;
1919        int i, first;
1920        struct bio *tbio, *fbio;
1921        int vcnt;
1922
1923        atomic_set(&r10_bio->remaining, 1);
1924
1925        /* find the first device with a block */
1926        for (i=0; i<conf->copies; i++)
1927                if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
1928                        break;
1929
1930        if (i == conf->copies)
1931                goto done;
1932
1933        first = i;
1934        fbio = r10_bio->devs[i].bio;
1935
1936        vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
1937        /* now find blocks with errors */
1938        for (i=0 ; i < conf->copies ; i++) {
1939                int  j, d;
1940
1941                tbio = r10_bio->devs[i].bio;
1942
1943                if (tbio->bi_end_io != end_sync_read)
1944                        continue;
1945                if (i == first)
1946                        continue;
1947                if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) {
1948                        /* We know that the bi_io_vec layout is the same for
1949                         * both 'first' and 'i', so we just compare them.
1950                         * All vec entries are PAGE_SIZE;
1951                         */
1952                        for (j = 0; j < vcnt; j++)
1953                                if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
1954                                           page_address(tbio->bi_io_vec[j].bv_page),
1955                                           fbio->bi_io_vec[j].bv_len))
1956                                        break;
1957                        if (j == vcnt)
1958                                continue;
1959                        mddev->resync_mismatches += r10_bio->sectors;
1960                        if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
1961                                /* Don't fix anything. */
1962                                continue;
1963                }
1964                /* Ok, we need to write this bio, either to correct an
1965                 * inconsistency or to correct an unreadable block.
1966                 * First we need to fixup bv_offset, bv_len and
1967                 * bi_vecs, as the read request might have corrupted these
1968                 */
1969                tbio->bi_vcnt = vcnt;
1970                tbio->bi_size = r10_bio->sectors << 9;
1971                tbio->bi_idx = 0;
1972                tbio->bi_phys_segments = 0;
1973                tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
1974                tbio->bi_flags |= 1 << BIO_UPTODATE;
1975                tbio->bi_next = NULL;
1976                tbio->bi_rw = WRITE;
1977                tbio->bi_private = r10_bio;
1978                tbio->bi_sector = r10_bio->devs[i].addr;
1979
1980                for (j=0; j < vcnt ; j++) {
1981                        tbio->bi_io_vec[j].bv_offset = 0;
1982                        tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
1983
1984                        memcpy(page_address(tbio->bi_io_vec[j].bv_page),
1985                               page_address(fbio->bi_io_vec[j].bv_page),
1986                               PAGE_SIZE);
1987                }
1988                tbio->bi_end_io = end_sync_write;
1989
1990                d = r10_bio->devs[i].devnum;
1991                atomic_inc(&conf->mirrors[d].rdev->nr_pending);
1992                atomic_inc(&r10_bio->remaining);
1993                md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
1994
1995                tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
1996                tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
1997                generic_make_request(tbio);
1998        }
1999
2000        /* Now write out to any replacement devices
2001         * that are active
2002         */
2003        for (i = 0; i < conf->copies; i++) {
2004                int j, d;
2005
2006                tbio = r10_bio->devs[i].repl_bio;
2007                if (!tbio || !tbio->bi_end_io)
2008                        continue;
2009                if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2010                    && r10_bio->devs[i].bio != fbio)
2011                        for (j = 0; j < vcnt; j++)
2012                                memcpy(page_address(tbio->bi_io_vec[j].bv_page),
2013                                       page_address(fbio->bi_io_vec[j].bv_page),
2014                                       PAGE_SIZE);
2015                d = r10_bio->devs[i].devnum;
2016                atomic_inc(&r10_bio->remaining);
2017                md_sync_acct(conf->mirrors[d].replacement->bdev,
2018                             tbio->bi_size >> 9);
2019                generic_make_request(tbio);
2020        }
2021
2022done:
2023        if (atomic_dec_and_test(&r10_bio->remaining)) {
2024                md_done_sync(mddev, r10_bio->sectors, 1);
2025                put_buf(r10_bio);
2026        }
2027}
2028
2029/*
2030 * Now for the recovery code.
2031 * Recovery happens across physical sectors.
2032 * We recover all non-is_sync drives by finding the virtual address of
2033 * each, and then choose a working drive that also has that virt address.
2034 * There is a separate r10_bio for each non-in_sync drive.
2035 * Only the first two slots are in use. The first for reading,
2036 * The second for writing.
2037 *
2038 */
2039static void fix_recovery_read_error(struct r10bio *r10_bio)
2040{
2041        /* We got a read error during recovery.
2042         * We repeat the read in smaller page-sized sections.
2043         * If a read succeeds, write it to the new device or record
2044         * a bad block if we cannot.
2045         * If a read fails, record a bad block on both old and
2046         * new devices.
2047         */
2048        struct mddev *mddev = r10_bio->mddev;
2049        struct r10conf *conf = mddev->private;
2050        struct bio *bio = r10_bio->devs[0].bio;
2051        sector_t sect = 0;
2052        int sectors = r10_bio->sectors;
2053        int idx = 0;
2054        int dr = r10_bio->devs[0].devnum;
2055        int dw = r10_bio->devs[1].devnum;
2056
2057        while (sectors) {
2058                int s = sectors;
2059                struct md_rdev *rdev;
2060                sector_t addr;
2061                int ok;
2062
2063                if (s > (PAGE_SIZE>>9))
2064                        s = PAGE_SIZE >> 9;
2065
2066                rdev = conf->mirrors[dr].rdev;
2067                addr = r10_bio->devs[0].addr + sect,
2068                ok = sync_page_io(rdev,
2069                                  addr,
2070                                  s << 9,
2071                                  bio->bi_io_vec[idx].bv_page,
2072                                  READ, false);
2073                if (ok) {
2074                        rdev = conf->mirrors[dw].rdev;
2075                        addr = r10_bio->devs[1].addr + sect;
2076                        ok = sync_page_io(rdev,
2077                                          addr,
2078                                          s << 9,
2079                                          bio->bi_io_vec[idx].bv_page,
2080                                          WRITE, false);
2081                        if (!ok) {
2082                                set_bit(WriteErrorSeen, &rdev->flags);
2083                                if (!test_and_set_bit(WantReplacement,
2084                                                      &rdev->flags))
2085                                        set_bit(MD_RECOVERY_NEEDED,
2086                                                &rdev->mddev->recovery);
2087                        }
2088                }
2089                if (!ok) {
2090                        /* We don't worry if we cannot set a bad block -
2091                         * it really is bad so there is no loss in not
2092                         * recording it yet
2093                         */
2094                        rdev_set_badblocks(rdev, addr, s, 0);
2095
2096                        if (rdev != conf->mirrors[dw].rdev) {
2097                                /* need bad block on destination too */
2098                                struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2099                                addr = r10_bio->devs[1].addr + sect;
2100                                ok = rdev_set_badblocks(rdev2, addr, s, 0);
2101                                if (!ok) {
2102                                        /* just abort the recovery */
2103                                        printk(KERN_NOTICE
2104                                               "md/raid10:%s: recovery aborted"
2105                                               " due to read error\n",
2106                                               mdname(mddev));
2107
2108                                        conf->mirrors[dw].recovery_disabled
2109                                                = mddev->recovery_disabled;
2110                                        set_bit(MD_RECOVERY_INTR,
2111                                                &mddev->recovery);
2112                                        break;
2113                                }
2114                        }
2115                }
2116
2117                sectors -= s;
2118                sect += s;
2119                idx++;
2120        }
2121}
2122
2123static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2124{
2125        struct r10conf *conf = mddev->private;
2126        int d;
2127        struct bio *wbio, *wbio2;
2128
2129        if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2130                fix_recovery_read_error(r10_bio);
2131                end_sync_request(r10_bio);
2132                return;
2133        }
2134
2135        /*
2136         * share the pages with the first bio
2137         * and submit the write request
2138         */
2139        d = r10_bio->devs[1].devnum;
2140        wbio = r10_bio->devs[1].bio;
2141        wbio2 = r10_bio->devs[1].repl_bio;
2142        if (wbio->bi_end_io) {
2143                atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2144                md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
2145                generic_make_request(wbio);
2146        }
2147        if (wbio2 && wbio2->bi_end_io) {
2148                atomic_inc(&conf->mirrors[d].replacement->nr_pending);
2149                md_sync_acct(conf->mirrors[d].replacement->bdev,
2150                             wbio2->bi_size >> 9);
2151                generic_make_request(wbio2);
2152        }
2153}
2154
2155
2156/*
2157 * Used by fix_read_error() to decay the per rdev read_errors.
2158 * We halve the read error count for every hour that has elapsed
2159 * since the last recorded read error.
2160 *
2161 */
2162static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2163{
2164        struct timespec cur_time_mon;
2165        unsigned long hours_since_last;
2166        unsigned int read_errors = atomic_read(&rdev->read_errors);
2167
2168        ktime_get_ts(&cur_time_mon);
2169
2170        if (rdev->last_read_error.tv_sec == 0 &&
2171            rdev->last_read_error.tv_nsec == 0) {
2172                /* first time we've seen a read error */
2173                rdev->last_read_error = cur_time_mon;
2174                return;
2175        }
2176
2177        hours_since_last = (cur_time_mon.tv_sec -
2178                            rdev->last_read_error.tv_sec) / 3600;
2179
2180        rdev->last_read_error = cur_time_mon;
2181
2182        /*
2183         * if hours_since_last is > the number of bits in read_errors
2184         * just set read errors to 0. We do this to avoid
2185         * overflowing the shift of read_errors by hours_since_last.
2186         */
2187        if (hours_since_last >= 8 * sizeof(read_errors))
2188                atomic_set(&rdev->read_errors, 0);
2189        else
2190                atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
2191}
2192
2193static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2194                            int sectors, struct page *page, int rw)
2195{
2196        sector_t first_bad;
2197        int bad_sectors;
2198
2199        if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
2200            && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
2201                return -1;
2202        if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
2203                /* success */
2204                return 1;
2205        if (rw == WRITE) {
2206                set_bit(WriteErrorSeen, &rdev->flags);
2207                if (!test_and_set_bit(WantReplacement, &rdev->flags))
2208                        set_bit(MD_RECOVERY_NEEDED,
2209                                &rdev->mddev->recovery);
2210        }
2211        /* need to record an error - either for the block or the device */
2212        if (!rdev_set_badblocks(rdev, sector, sectors, 0))
2213                md_error(rdev->mddev, rdev);
2214        return 0;
2215}
2216
2217/*
2218 * This is a kernel thread which:
2219 *
2220 *      1.      Retries failed read operations on working mirrors.
2221 *      2.      Updates the raid superblock when problems encounter.
2222 *      3.      Performs writes following reads for array synchronising.
2223 */
2224
2225static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2226{
2227        int sect = 0; /* Offset from r10_bio->sector */
2228        int sectors = r10_bio->sectors;
2229        struct md_rdev*rdev;
2230        int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2231        int d = r10_bio->devs[r10_bio->read_slot].devnum;
2232
2233        /* still own a reference to this rdev, so it cannot
2234         * have been cleared recently.
2235         */
2236        rdev = conf->mirrors[d].rdev;
2237
2238        if (test_bit(Faulty, &rdev->flags))
2239                /* drive has already been failed, just ignore any
2240                   more fix_read_error() attempts */
2241                return;
2242
2243        check_decay_read_errors(mddev, rdev);
2244        atomic_inc(&rdev->read_errors);
2245        if (atomic_read(&rdev->read_errors) > max_read_errors) {
2246                char b[BDEVNAME_SIZE];
2247                bdevname(rdev->bdev, b);
2248
2249                printk(KERN_NOTICE
2250                       "md/raid10:%s: %s: Raid device exceeded "
2251                       "read_error threshold [cur %d:max %d]\n",
2252                       mdname(mddev), b,
2253                       atomic_read(&rdev->read_errors), max_read_errors);
2254                printk(KERN_NOTICE
2255                       "md/raid10:%s: %s: Failing raid device\n",
2256                       mdname(mddev), b);
2257                md_error(mddev, conf->mirrors[d].rdev);
2258                r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2259                return;
2260        }
2261
2262        while(sectors) {
2263                int s = sectors;
2264                int sl = r10_bio->read_slot;
2265                int success = 0;
2266                int start;
2267
2268                if (s > (PAGE_SIZE>>9))
2269                        s = PAGE_SIZE >> 9;
2270
2271                rcu_read_lock();
2272                do {
2273                        sector_t first_bad;
2274                        int bad_sectors;
2275
2276                        d = r10_bio->devs[sl].devnum;
2277                        rdev = rcu_dereference(conf->mirrors[d].rdev);
2278                        if (rdev &&
2279                            !test_bit(Unmerged, &rdev->flags) &&
2280                            test_bit(In_sync, &rdev->flags) &&
2281                            is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
2282                                        &first_bad, &bad_sectors) == 0) {
2283                                atomic_inc(&rdev->nr_pending);
2284                                rcu_read_unlock();
2285                                success = sync_page_io(rdev,
2286                                                       r10_bio->devs[sl].addr +
2287                                                       sect,
2288                                                       s<<9,
2289                                                       conf->tmppage, READ, false);
2290                                rdev_dec_pending(rdev, mddev);
2291                                rcu_read_lock();
2292                                if (success)
2293                                        break;
2294                        }
2295                        sl++;
2296                        if (sl == conf->copies)
2297                                sl = 0;
2298                } while (!success && sl != r10_bio->read_slot);
2299                rcu_read_unlock();
2300
2301                if (!success) {
2302                        /* Cannot read from anywhere, just mark the block
2303                         * as bad on the first device to discourage future
2304                         * reads.
2305                         */
2306                        int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2307                        rdev = conf->mirrors[dn].rdev;
2308
2309                        if (!rdev_set_badblocks(
2310                                    rdev,
2311                                    r10_bio->devs[r10_bio->read_slot].addr
2312                                    + sect,
2313                                    s, 0)) {
2314                                md_error(mddev, rdev);
2315                                r10_bio->devs[r10_bio->read_slot].bio
2316                                        = IO_BLOCKED;
2317                        }
2318                        break;
2319                }
2320
2321                start = sl;
2322                /* write it back and re-read */
2323                rcu_read_lock();
2324                while (sl != r10_bio->read_slot) {
2325                        char b[BDEVNAME_SIZE];
2326
2327                        if (sl==0)
2328                                sl = conf->copies;
2329                        sl--;
2330                        d = r10_bio->devs[sl].devnum;
2331                        rdev = rcu_dereference(conf->mirrors[d].rdev);
2332                        if (!rdev ||
2333                            test_bit(Unmerged, &rdev->flags) ||
2334                            !test_bit(In_sync, &rdev->flags))
2335                                continue;
2336
2337                        atomic_inc(&rdev->nr_pending);
2338                        rcu_read_unlock();
2339                        if (r10_sync_page_io(rdev,
2340                                             r10_bio->devs[sl].addr +
2341                                             sect,
2342                                             s, conf->tmppage, WRITE)
2343                            == 0) {
2344                                /* Well, this device is dead */
2345                                printk(KERN_NOTICE
2346                                       "md/raid10:%s: read correction "
2347                                       "write failed"
2348                                       " (%d sectors at %llu on %s)\n",
2349                                       mdname(mddev), s,
2350                                       (unsigned long long)(
2351                                               sect +
2352                                               choose_data_offset(r10_bio,
2353                                                                  rdev)),
2354                                       bdevname(rdev->bdev, b));
2355                                printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2356                                       "drive\n",
2357                                       mdname(mddev),
2358                                       bdevname(rdev->bdev, b));
2359                        }
2360                        rdev_dec_pending(rdev, mddev);
2361                        rcu_read_lock();
2362                }
2363                sl = start;
2364                while (sl != r10_bio->read_slot) {
2365                        char b[BDEVNAME_SIZE];
2366
2367                        if (sl==0)
2368                                sl = conf->copies;
2369                        sl--;
2370                        d = r10_bio->devs[sl].devnum;
2371                        rdev = rcu_dereference(conf->mirrors[d].rdev);
2372                        if (!rdev ||
2373                            !test_bit(In_sync, &rdev->flags))
2374                                continue;
2375
2376                        atomic_inc(&rdev->nr_pending);
2377                        rcu_read_unlock();
2378                        switch (r10_sync_page_io(rdev,
2379                                             r10_bio->devs[sl].addr +
2380                                             sect,
2381                                             s, conf->tmppage,
2382                                                 READ)) {
2383                        case 0:
2384                                /* Well, this device is dead */
2385                                printk(KERN_NOTICE
2386                                       "md/raid10:%s: unable to read back "
2387                                       "corrected sectors"
2388                                       " (%d sectors at %llu on %s)\n",
2389                                       mdname(mddev), s,
2390                                       (unsigned long long)(
2391                                               sect +
2392                                               choose_data_offset(r10_bio, rdev)),
2393                                       bdevname(rdev->bdev, b));
2394                                printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2395                                       "drive\n",
2396                                       mdname(mddev),
2397                                       bdevname(rdev->bdev, b));
2398                                break;
2399                        case 1:
2400                                printk(KERN_INFO
2401                                       "md/raid10:%s: read error corrected"
2402                                       " (%d sectors at %llu on %s)\n",
2403                                       mdname(mddev), s,
2404                                       (unsigned long long)(
2405                                               sect +
2406                                               choose_data_offset(r10_bio, rdev)),
2407                                       bdevname(rdev->bdev, b));
2408                                atomic_add(s, &rdev->corrected_errors);
2409                        }
2410
2411                        rdev_dec_pending(rdev, mddev);
2412                        rcu_read_lock();
2413                }
2414                rcu_read_unlock();
2415
2416                sectors -= s;
2417                sect += s;
2418        }
2419}
2420
2421static void bi_complete(struct bio *bio, int error)
2422{
2423        complete((struct completion *)bio->bi_private);
2424}
2425
2426static int submit_bio_wait(int rw, struct bio *bio)
2427{
2428        struct completion event;
2429        rw |= REQ_SYNC;
2430
2431        init_completion(&event);
2432        bio->bi_private = &event;
2433        bio->bi_end_io = bi_complete;
2434        submit_bio(rw, bio);
2435        wait_for_completion(&event);
2436
2437        return test_bit(BIO_UPTODATE, &bio->bi_flags);
2438}
2439
2440static int narrow_write_error(struct r10bio *r10_bio, int i)
2441{
2442        struct bio *bio = r10_bio->master_bio;
2443        struct mddev *mddev = r10_bio->mddev;
2444        struct r10conf *conf = mddev->private;
2445        struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2446        /* bio has the data to be written to slot 'i' where
2447         * we just recently had a write error.
2448         * We repeatedly clone the bio and trim down to one block,
2449         * then try the write.  Where the write fails we record
2450         * a bad block.
2451         * It is conceivable that the bio doesn't exactly align with
2452         * blocks.  We must handle this.
2453         *
2454         * We currently own a reference to the rdev.
2455         */
2456
2457        int block_sectors;
2458        sector_t sector;
2459        int sectors;
2460        int sect_to_write = r10_bio->sectors;
2461        int ok = 1;
2462
2463        if (rdev->badblocks.shift < 0)
2464                return 0;
2465
2466        block_sectors = 1 << rdev->badblocks.shift;
2467        sector = r10_bio->sector;
2468        sectors = ((r10_bio->sector + block_sectors)
2469                   & ~(sector_t)(block_sectors - 1))
2470                - sector;
2471
2472        while (sect_to_write) {
2473                struct bio *wbio;
2474                if (sectors > sect_to_write)
2475                        sectors = sect_to_write;
2476                /* Write at 'sector' for 'sectors' */
2477                wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
2478                md_trim_bio(wbio, sector - bio->bi_sector, sectors);
2479                wbio->bi_sector = (r10_bio->devs[i].addr+
2480                                   choose_data_offset(r10_bio, rdev) +
2481                                   (sector - r10_bio->sector));
2482                wbio->bi_bdev = rdev->bdev;
2483                if (submit_bio_wait(WRITE, wbio) == 0)
2484                        /* Failure! */
2485                        ok = rdev_set_badblocks(rdev, sector,
2486                                                sectors, 0)
2487                                && ok;
2488
2489                bio_put(wbio);
2490                sect_to_write -= sectors;
2491                sector += sectors;
2492                sectors = block_sectors;
2493        }
2494        return ok;
2495}
2496
2497static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2498{
2499        int slot = r10_bio->read_slot;
2500        struct bio *bio;
2501        struct r10conf *conf = mddev->private;
2502        struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2503        char b[BDEVNAME_SIZE];
2504        unsigned long do_sync;
2505        int max_sectors;
2506
2507        /* we got a read error. Maybe the drive is bad.  Maybe just
2508         * the block and we can fix it.
2509         * We freeze all other IO, and try reading the block from
2510         * other devices.  When we find one, we re-write
2511         * and check it that fixes the read error.
2512         * This is all done synchronously while the array is
2513         * frozen.
2514         */
2515        bio = r10_bio->devs[slot].bio;
2516        bdevname(bio->bi_bdev, b);
2517        bio_put(bio);
2518        r10_bio->devs[slot].bio = NULL;
2519
2520        if (mddev->ro == 0) {
2521                freeze_array(conf);
2522                fix_read_error(conf, mddev, r10_bio);
2523                unfreeze_array(conf);
2524        } else
2525                r10_bio->devs[slot].bio = IO_BLOCKED;
2526
2527        rdev_dec_pending(rdev, mddev);
2528
2529read_more:
2530        rdev = read_balance(conf, r10_bio, &max_sectors);
2531        if (rdev == NULL) {
2532                printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
2533                       " read error for block %llu\n",
2534                       mdname(mddev), b,
2535                       (unsigned long long)r10_bio->sector);
2536                raid_end_bio_io(r10_bio);
2537                return;
2538        }
2539
2540        do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC);
2541        slot = r10_bio->read_slot;
2542        printk_ratelimited(
2543                KERN_ERR
2544                "md/raid10:%s: %s: redirecting "
2545                "sector %llu to another mirror\n",
2546                mdname(mddev),
2547                bdevname(rdev->bdev, b),
2548                (unsigned long long)r10_bio->sector);
2549        bio = bio_clone_mddev(r10_bio->master_bio,
2550                              GFP_NOIO, mddev);
2551        md_trim_bio(bio,
2552                    r10_bio->sector - bio->bi_sector,
2553                    max_sectors);
2554        r10_bio->devs[slot].bio = bio;
2555        r10_bio->devs[slot].rdev = rdev;
2556        bio->bi_sector = r10_bio->devs[slot].addr
2557                + choose_data_offset(r10_bio, rdev);
2558        bio->bi_bdev = rdev->bdev;
2559        bio->bi_rw = READ | do_sync;
2560        bio->bi_private = r10_bio;
2561        bio->bi_end_io = raid10_end_read_request;
2562        if (max_sectors < r10_bio->sectors) {
2563                /* Drat - have to split this up more */
2564                struct bio *mbio = r10_bio->master_bio;
2565                int sectors_handled =
2566                        r10_bio->sector + max_sectors
2567                        - mbio->bi_sector;
2568                r10_bio->sectors = max_sectors;
2569                spin_lock_irq(&conf->device_lock);
2570                if (mbio->bi_phys_segments == 0)
2571                        mbio->bi_phys_segments = 2;
2572                else
2573                        mbio->bi_phys_segments++;
2574                spin_unlock_irq(&conf->device_lock);
2575                generic_make_request(bio);
2576
2577                r10_bio = mempool_alloc(conf->r10bio_pool,
2578                                        GFP_NOIO);
2579                r10_bio->master_bio = mbio;
2580                r10_bio->sectors = (mbio->bi_size >> 9)
2581                        - sectors_handled;
2582                r10_bio->state = 0;
2583                set_bit(R10BIO_ReadError,
2584                        &r10_bio->state);
2585                r10_bio->mddev = mddev;
2586                r10_bio->sector = mbio->bi_sector
2587                        + sectors_handled;
2588
2589                goto read_more;
2590        } else
2591                generic_make_request(bio);
2592}
2593
2594static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2595{
2596        /* Some sort of write request has finished and it
2597         * succeeded in writing where we thought there was a
2598         * bad block.  So forget the bad block.
2599         * Or possibly if failed and we need to record
2600         * a bad block.
2601         */
2602        int m;
2603        struct md_rdev *rdev;
2604
2605        if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2606            test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2607                for (m = 0; m < conf->copies; m++) {
2608                        int dev = r10_bio->devs[m].devnum;
2609                        rdev = conf->mirrors[dev].rdev;
2610                        if (r10_bio->devs[m].bio == NULL)
2611                                continue;
2612                        if (test_bit(BIO_UPTODATE,
2613                                     &r10_bio->devs[m].bio->bi_flags)) {
2614                                rdev_clear_badblocks(
2615                                        rdev,
2616                                        r10_bio->devs[m].addr,
2617                                        r10_bio->sectors, 0);
2618                        } else {
2619                                if (!rdev_set_badblocks(
2620                                            rdev,
2621                                            r10_bio->devs[m].addr,
2622                                            r10_bio->sectors, 0))
2623                                        md_error(conf->mddev, rdev);
2624                        }
2625                        rdev = conf->mirrors[dev].replacement;
2626                        if (r10_bio->devs[m].repl_bio == NULL)
2627                                continue;
2628                        if (test_bit(BIO_UPTODATE,
2629                                     &r10_bio->devs[m].repl_bio->bi_flags)) {
2630                                rdev_clear_badblocks(
2631                                        rdev,
2632                                        r10_bio->devs[m].addr,
2633                                        r10_bio->sectors, 0);
2634                        } else {
2635                                if (!rdev_set_badblocks(
2636                                            rdev,
2637                                            r10_bio->devs[m].addr,
2638                                            r10_bio->sectors, 0))
2639                                        md_error(conf->mddev, rdev);
2640                        }
2641                }
2642                put_buf(r10_bio);
2643        } else {
2644                for (m = 0; m < conf->copies; m++) {
2645                        int dev = r10_bio->devs[m].devnum;
2646                        struct bio *bio = r10_bio->devs[m].bio;
2647                        rdev = conf->mirrors[dev].rdev;
2648                        if (bio == IO_MADE_GOOD) {
2649                                rdev_clear_badblocks(
2650                                        rdev,
2651                                        r10_bio->devs[m].addr,
2652                                        r10_bio->sectors, 0);
2653                                rdev_dec_pending(rdev, conf->mddev);
2654                        } else if (bio != NULL &&
2655                                   !test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2656                                if (!narrow_write_error(r10_bio, m)) {
2657                                        md_error(conf->mddev, rdev);
2658                                        set_bit(R10BIO_Degraded,
2659                                                &r10_bio->state);
2660                                }
2661                                rdev_dec_pending(rdev, conf->mddev);
2662                        }
2663                        bio = r10_bio->devs[m].repl_bio;
2664                        rdev = conf->mirrors[dev].replacement;
2665                        if (rdev && bio == IO_MADE_GOOD) {
2666                                rdev_clear_badblocks(
2667                                        rdev,
2668                                        r10_bio->devs[m].addr,
2669                                        r10_bio->sectors, 0);
2670                                rdev_dec_pending(rdev, conf->mddev);
2671                        }
2672                }
2673                if (test_bit(R10BIO_WriteError,
2674                             &r10_bio->state))
2675                        close_write(r10_bio);
2676                raid_end_bio_io(r10_bio);
2677        }
2678}
2679
2680static void raid10d(struct mddev *mddev)
2681{
2682        struct r10bio *r10_bio;
2683        unsigned long flags;
2684        struct r10conf *conf = mddev->private;
2685        struct list_head *head = &conf->retry_list;
2686        struct blk_plug plug;
2687
2688        md_check_recovery(mddev);
2689
2690        blk_start_plug(&plug);
2691        for (;;) {
2692
2693                flush_pending_writes(conf);
2694
2695                spin_lock_irqsave(&conf->device_lock, flags);
2696                if (list_empty(head)) {
2697                        spin_unlock_irqrestore(&conf->device_lock, flags);
2698                        break;
2699                }
2700                r10_bio = list_entry(head->prev, struct r10bio, retry_list);
2701                list_del(head->prev);
2702                conf->nr_queued--;
2703                spin_unlock_irqrestore(&conf->device_lock, flags);
2704
2705                mddev = r10_bio->mddev;
2706                conf = mddev->private;
2707                if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2708                    test_bit(R10BIO_WriteError, &r10_bio->state))
2709                        handle_write_completed(conf, r10_bio);
2710                else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
2711                        reshape_request_write(mddev, r10_bio);
2712                else if (test_bit(R10BIO_IsSync, &r10_bio->state))
2713                        sync_request_write(mddev, r10_bio);
2714                else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
2715                        recovery_request_write(mddev, r10_bio);
2716                else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2717                        handle_read_error(mddev, r10_bio);
2718                else {
2719                        /* just a partial read to be scheduled from a
2720                         * separate context
2721                         */
2722                        int slot = r10_bio->read_slot;
2723                        generic_make_request(r10_bio->devs[slot].bio);
2724                }
2725
2726                cond_resched();
2727                if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
2728                        md_check_recovery(mddev);
2729        }
2730        blk_finish_plug(&plug);
2731}
2732
2733
2734static int init_resync(struct r10conf *conf)
2735{
2736        int buffs;
2737        int i;
2738
2739        buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2740        BUG_ON(conf->r10buf_pool);
2741        conf->have_replacement = 0;
2742        for (i = 0; i < conf->geo.raid_disks; i++)
2743                if (conf->mirrors[i].replacement)
2744                        conf->have_replacement = 1;
2745        conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
2746        if (!conf->r10buf_pool)
2747                return -ENOMEM;
2748        conf->next_resync = 0;
2749        return 0;
2750}
2751
2752/*
2753 * perform a "sync" on one "block"
2754 *
2755 * We need to make sure that no normal I/O request - particularly write
2756 * requests - conflict with active sync requests.
2757 *
2758 * This is achieved by tracking pending requests and a 'barrier' concept
2759 * that can be installed to exclude normal IO requests.
2760 *
2761 * Resync and recovery are handled very differently.
2762 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
2763 *
2764 * For resync, we iterate over virtual addresses, read all copies,
2765 * and update if there are differences.  If only one copy is live,
2766 * skip it.
2767 * For recovery, we iterate over physical addresses, read a good
2768 * value for each non-in_sync drive, and over-write.
2769 *
2770 * So, for recovery we may have several outstanding complex requests for a
2771 * given address, one for each out-of-sync device.  We model this by allocating
2772 * a number of r10_bio structures, one for each out-of-sync device.
2773 * As we setup these structures, we collect all bio's together into a list
2774 * which we then process collectively to add pages, and then process again
2775 * to pass to generic_make_request.
2776 *
2777 * The r10_bio structures are linked using a borrowed master_bio pointer.
2778 * This link is counted in ->remaining.  When the r10_bio that points to NULL
2779 * has its remaining count decremented to 0, the whole complex operation
2780 * is complete.
2781 *
2782 */
2783
2784static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
2785                             int *skipped, int go_faster)
2786{
2787        struct r10conf *conf = mddev->private;
2788        struct r10bio *r10_bio;
2789        struct bio *biolist = NULL, *bio;
2790        sector_t max_sector, nr_sectors;
2791        int i;
2792        int max_sync;
2793        sector_t sync_blocks;
2794        sector_t sectors_skipped = 0;
2795        int chunks_skipped = 0;
2796        sector_t chunk_mask = conf->geo.chunk_mask;
2797
2798        if (!conf->r10buf_pool)
2799                if (init_resync(conf))
2800                        return 0;
2801
2802 skipped:
2803        max_sector = mddev->dev_sectors;
2804        if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
2805            test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
2806                max_sector = mddev->resync_max_sectors;
2807        if (sector_nr >= max_sector) {
2808                /* If we aborted, we need to abort the
2809                 * sync on the 'current' bitmap chucks (there can
2810                 * be several when recovering multiple devices).
2811                 * as we may have started syncing it but not finished.
2812                 * We can find the current address in
2813                 * mddev->curr_resync, but for recovery,
2814                 * we need to convert that to several
2815                 * virtual addresses.
2816                 */
2817                if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
2818                        end_reshape(conf);
2819                        return 0;
2820                }
2821
2822                if (mddev->curr_resync < max_sector) { /* aborted */
2823                        if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
2824                                bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2825                                                &sync_blocks, 1);
2826                        else for (i = 0; i < conf->geo.raid_disks; i++) {
2827                                sector_t sect =
2828                                        raid10_find_virt(conf, mddev->curr_resync, i);
2829                                bitmap_end_sync(mddev->bitmap, sect,
2830                                                &sync_blocks, 1);
2831                        }
2832                } else {
2833                        /* completed sync */
2834                        if ((!mddev->bitmap || conf->fullsync)
2835                            && conf->have_replacement
2836                            && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2837                                /* Completed a full sync so the replacements
2838                                 * are now fully recovered.
2839                                 */
2840                                for (i = 0; i < conf->geo.raid_disks; i++)
2841                                        if (conf->mirrors[i].replacement)
2842                                                conf->mirrors[i].replacement
2843                                                        ->recovery_offset
2844                                                        = MaxSector;
2845                        }
2846                        conf->fullsync = 0;
2847                }
2848                bitmap_close_sync(mddev->bitmap);
2849                close_sync(conf);
2850                *skipped = 1;
2851                return sectors_skipped;
2852        }
2853
2854        if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
2855                return reshape_request(mddev, sector_nr, skipped);
2856
2857        if (chunks_skipped >= conf->geo.raid_disks) {
2858                /* if there has been nothing to do on any drive,
2859                 * then there is nothing to do at all..
2860                 */
2861                *skipped = 1;
2862                return (max_sector - sector_nr) + sectors_skipped;
2863        }
2864
2865        if (max_sector > mddev->resync_max)
2866                max_sector = mddev->resync_max; /* Don't do IO beyond here */
2867
2868        /* make sure whole request will fit in a chunk - if chunks
2869         * are meaningful
2870         */
2871        if (conf->geo.near_copies < conf->geo.raid_disks &&
2872            max_sector > (sector_nr | chunk_mask))
2873                max_sector = (sector_nr | chunk_mask) + 1;
2874        /*
2875         * If there is non-resync activity waiting for us then
2876         * put in a delay to throttle resync.
2877         */
2878        if (!go_faster && conf->nr_waiting)
2879                msleep_interruptible(1000);
2880
2881        /* Again, very different code for resync and recovery.
2882         * Both must result in an r10bio with a list of bios that
2883         * have bi_end_io, bi_sector, bi_bdev set,
2884         * and bi_private set to the r10bio.
2885         * For recovery, we may actually create several r10bios
2886         * with 2 bios in each, that correspond to the bios in the main one.
2887         * In this case, the subordinate r10bios link back through a
2888         * borrowed master_bio pointer, and the counter in the master
2889         * includes a ref from each subordinate.
2890         */
2891        /* First, we decide what to do and set ->bi_end_io
2892         * To end_sync_read if we want to read, and
2893         * end_sync_write if we will want to write.
2894         */
2895
2896        max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
2897        if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2898                /* recovery... the complicated one */
2899                int j;
2900                r10_bio = NULL;
2901
2902                for (i = 0 ; i < conf->geo.raid_disks; i++) {
2903                        int still_degraded;
2904                        struct r10bio *rb2;
2905                        sector_t sect;
2906                        int must_sync;
2907                        int any_working;
2908                        struct raid10_info *mirror = &conf->mirrors[i];
2909
2910                        if ((mirror->rdev == NULL ||
2911                             test_bit(In_sync, &mirror->rdev->flags))
2912                            &&
2913                            (mirror->replacement == NULL ||
2914                             test_bit(Faulty,
2915                                      &mirror->replacement->flags)))
2916                                continue;
2917
2918                        still_degraded = 0;
2919                        /* want to reconstruct this device */
2920                        rb2 = r10_bio;
2921                        sect = raid10_find_virt(conf, sector_nr, i);
2922                        if (sect >= mddev->resync_max_sectors) {
2923                                /* last stripe is not complete - don't
2924                                 * try to recover this sector.
2925                                 */
2926                                continue;
2927                        }
2928                        /* Unless we are doing a full sync, or a replacement
2929                         * we only need to recover the block if it is set in
2930                         * the bitmap
2931                         */
2932                        must_sync = bitmap_start_sync(mddev->bitmap, sect,
2933                                                      &sync_blocks, 1);
2934                        if (sync_blocks < max_sync)
2935                                max_sync = sync_blocks;
2936                        if (!must_sync &&
2937                            mirror->replacement == NULL &&
2938                            !conf->fullsync) {
2939                                /* yep, skip the sync_blocks here, but don't assume
2940                                 * that there will never be anything to do here
2941                                 */
2942                                chunks_skipped = -1;
2943                                continue;
2944                        }
2945
2946                        r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
2947                        raise_barrier(conf, rb2 != NULL);
2948                        atomic_set(&r10_bio->remaining, 0);
2949
2950                        r10_bio->master_bio = (struct bio*)rb2;
2951                        if (rb2)
2952                                atomic_inc(&rb2->remaining);
2953                        r10_bio->mddev = mddev;
2954                        set_bit(R10BIO_IsRecover, &r10_bio->state);
2955                        r10_bio->sector = sect;
2956
2957                        raid10_find_phys(conf, r10_bio);
2958
2959                        /* Need to check if the array will still be
2960                         * degraded
2961                         */
2962                        for (j = 0; j < conf->geo.raid_disks; j++)
2963                                if (conf->mirrors[j].rdev == NULL ||
2964                                    test_bit(Faulty, &conf->mirrors[j].rdev->flags)) {
2965                                        still_degraded = 1;
2966                                        break;
2967                                }
2968
2969                        must_sync = bitmap_start_sync(mddev->bitmap, sect,
2970                                                      &sync_blocks, still_degraded);
2971
2972                        any_working = 0;
2973                        for (j=0; j<conf->copies;j++) {
2974                                int k;
2975                                int d = r10_bio->devs[j].devnum;
2976                                sector_t from_addr, to_addr;
2977                                struct md_rdev *rdev;
2978                                sector_t sector, first_bad;
2979                                int bad_sectors;
2980                                if (!conf->mirrors[d].rdev ||
2981                                    !test_bit(In_sync, &conf->mirrors[d].rdev->flags))
2982                                        continue;
2983                                /* This is where we read from */
2984                                any_working = 1;
2985                                rdev = conf->mirrors[d].rdev;
2986                                sector = r10_bio->devs[j].addr;
2987
2988                                if (is_badblock(rdev, sector, max_sync,
2989                                                &first_bad, &bad_sectors)) {
2990                                        if (first_bad > sector)
2991                                                max_sync = first_bad - sector;
2992                                        else {
2993                                                bad_sectors -= (sector
2994                                                                - first_bad);
2995                                                if (max_sync > bad_sectors)
2996                                                        max_sync = bad_sectors;
2997                                                continue;
2998                                        }
2999                                }
3000                                bio = r10_bio->devs[0].bio;
3001                                bio->bi_next = biolist;
3002                                biolist = bio;
3003                                bio->bi_private = r10_bio;
3004                                bio->bi_end_io = end_sync_read;
3005                                bio->bi_rw = READ;
3006                                from_addr = r10_bio->devs[j].addr;
3007                                bio->bi_sector = from_addr + rdev->data_offset;
3008                                bio->bi_bdev = rdev->bdev;
3009                                atomic_inc(&rdev->nr_pending);
3010                                /* and we write to 'i' (if not in_sync) */
3011
3012                                for (k=0; k<conf->copies; k++)
3013                                        if (r10_bio->devs[k].devnum == i)
3014                                                break;
3015                                BUG_ON(k == conf->copies);
3016                                to_addr = r10_bio->devs[k].addr;
3017                                r10_bio->devs[0].devnum = d;
3018                                r10_bio->devs[0].addr = from_addr;
3019                                r10_bio->devs[1].devnum = i;
3020                                r10_bio->devs[1].addr = to_addr;
3021
3022                                rdev = mirror->rdev;
3023                                if (!test_bit(In_sync, &rdev->flags)) {
3024                                        bio = r10_bio->devs[1].bio;
3025                                        bio->bi_next = biolist;
3026                                        biolist = bio;
3027                                        bio->bi_private = r10_bio;
3028                                        bio->bi_end_io = end_sync_write;
3029                                        bio->bi_rw = WRITE;
3030                                        bio->bi_sector = to_addr
3031                                                + rdev->data_offset;
3032                                        bio->bi_bdev = rdev->bdev;
3033                                        atomic_inc(&r10_bio->remaining);
3034                                } else
3035                                        r10_bio->devs[1].bio->bi_end_io = NULL;
3036
3037                                /* and maybe write to replacement */
3038                                bio = r10_bio->devs[1].repl_bio;
3039                                if (bio)
3040                                        bio->bi_end_io = NULL;
3041                                rdev = mirror->replacement;
3042                                /* Note: if rdev != NULL, then bio
3043                                 * cannot be NULL as r10buf_pool_alloc will
3044                                 * have allocated it.
3045                                 * So the second test here is pointless.
3046                                 * But it keeps semantic-checkers happy, and
3047                                 * this comment keeps human reviewers
3048                                 * happy.
3049                                 */
3050                                if (rdev == NULL || bio == NULL ||
3051                                    test_bit(Faulty, &rdev->flags))
3052                                        break;
3053                                bio->bi_next = biolist;
3054                                biolist = bio;
3055                                bio->bi_private = r10_bio;
3056                                bio->bi_end_io = end_sync_write;
3057                                bio->bi_rw = WRITE;
3058                                bio->bi_sector = to_addr + rdev->data_offset;
3059                                bio->bi_bdev = rdev->bdev;
3060                                atomic_inc(&r10_bio->remaining);
3061                                break;
3062                        }
3063                        if (j == conf->copies) {
3064                                /* Cannot recover, so abort the recovery or
3065                                 * record a bad block */
3066                                put_buf(r10_bio);
3067                                if (rb2)
3068                                        atomic_dec(&rb2->remaining);
3069                                r10_bio = rb2;
3070                                if (any_working) {
3071                                        /* problem is that there are bad blocks
3072                                         * on other device(s)
3073                                         */
3074                                        int k;
3075                                        for (k = 0; k < conf->copies; k++)
3076                                                if (r10_bio->devs[k].devnum == i)
3077                                                        break;
3078                                        if (!test_bit(In_sync,
3079                                                      &mirror->rdev->flags)
3080                                            && !rdev_set_badblocks(
3081                                                    mirror->rdev,
3082                                                    r10_bio->devs[k].addr,
3083                                                    max_sync, 0))
3084                                                any_working = 0;
3085                                        if (mirror->replacement &&
3086                                            !rdev_set_badblocks(
3087                                                    mirror->replacement,
3088                                                    r10_bio->devs[k].addr,
3089                                                    max_sync, 0))
3090                                                any_working = 0;
3091                                }
3092                                if (!any_working)  {
3093                                        if (!test_and_set_bit(MD_RECOVERY_INTR,
3094                                                              &mddev->recovery))
3095                                                printk(KERN_INFO "md/raid10:%s: insufficient "
3096                                                       "working devices for recovery.\n",
3097                                                       mdname(mddev));
3098                                        mirror->recovery_disabled
3099                                                = mddev->recovery_disabled;
3100                                }
3101                                break;
3102                        }
3103                }
3104                if (biolist == NULL) {
3105                        while (r10_bio) {
3106                                struct r10bio *rb2 = r10_bio;
3107                                r10_bio = (struct r10bio*) rb2->master_bio;
3108                                rb2->master_bio = NULL;
3109                                put_buf(rb2);
3110                        }
3111                        goto giveup;
3112                }
3113        } else {
3114                /* resync. Schedule a read for every block at this virt offset */
3115                int count = 0;
3116
3117                bitmap_cond_end_sync(mddev->bitmap, sector_nr);
3118
3119                if (!bitmap_start_sync(mddev->bitmap, sector_nr,
3120                                       &sync_blocks, mddev->degraded) &&
3121                    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
3122                                                 &mddev->recovery)) {
3123                        /* We can skip this block */
3124                        *skipped = 1;
3125                        return sync_blocks + sectors_skipped;
3126                }
3127                if (sync_blocks < max_sync)
3128                        max_sync = sync_blocks;
3129                r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3130
3131                r10_bio->mddev = mddev;
3132                atomic_set(&r10_bio->remaining, 0);
3133                raise_barrier(conf, 0);
3134                conf->next_resync = sector_nr;
3135
3136                r10_bio->master_bio = NULL;
3137                r10_bio->sector = sector_nr;
3138                set_bit(R10BIO_IsSync, &r10_bio->state);
3139                raid10_find_phys(conf, r10_bio);
3140                r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
3141
3142                for (i = 0; i < conf->copies; i++) {
3143                        int d = r10_bio->devs[i].devnum;
3144                        sector_t first_bad, sector;
3145                        int bad_sectors;
3146
3147                        if (r10_bio->devs[i].repl_bio)
3148                                r10_bio->devs[i].repl_bio->bi_end_io = NULL;
3149
3150                        bio = r10_bio->devs[i].bio;
3151                        bio->bi_end_io = NULL;
3152                        clear_bit(BIO_UPTODATE, &bio->bi_flags);
3153                        if (conf->mirrors[d].rdev == NULL ||
3154                            test_bit(Faulty, &conf->mirrors[d].rdev->flags))
3155                                continue;
3156                        sector = r10_bio->devs[i].addr;
3157                        if (is_badblock(conf->mirrors[d].rdev,
3158                                        sector, max_sync,
3159                                        &first_bad, &bad_sectors)) {
3160                                if (first_bad > sector)
3161                                        max_sync = first_bad - sector;
3162                                else {
3163                                        bad_sectors -= (sector - first_bad);
3164                                        if (max_sync > bad_sectors)
3165                                                max_sync = bad_sectors;
3166                                        continue;
3167                                }
3168                        }
3169                        atomic_inc(&conf->mirrors[d].rdev->nr_pending);
3170                        atomic_inc(&r10_bio->remaining);
3171                        bio->bi_next = biolist;
3172                        biolist = bio;
3173                        bio->bi_private = r10_bio;
3174                        bio->bi_end_io = end_sync_read;
3175                        bio->bi_rw = READ;
3176                        bio->bi_sector = sector +
3177                                conf->mirrors[d].rdev->data_offset;
3178                        bio->bi_bdev = conf->mirrors[d].rdev->bdev;
3179                        count++;
3180
3181                        if (conf->mirrors[d].replacement == NULL ||
3182                            test_bit(Faulty,
3183                                     &conf->mirrors[d].replacement->flags))
3184                                continue;
3185
3186                        /* Need to set up for writing to the replacement */
3187                        bio = r10_bio->devs[i].repl_bio;
3188                        clear_bit(BIO_UPTODATE, &bio->bi_flags);
3189
3190                        sector = r10_bio->devs[i].addr;
3191                        atomic_inc(&conf->mirrors[d].rdev->nr_pending);
3192                        bio->bi_next = biolist;
3193                        biolist = bio;
3194                        bio->bi_private = r10_bio;
3195                        bio->bi_end_io = end_sync_write;
3196                        bio->bi_rw = WRITE;
3197                        bio->bi_sector = sector +
3198                                conf->mirrors[d].replacement->data_offset;
3199                        bio->bi_bdev = conf->mirrors[d].replacement->bdev;
3200                        count++;
3201                }
3202
3203                if (count < 2) {
3204                        for (i=0; i<conf->copies; i++) {
3205                                int d = r10_bio->devs[i].devnum;
3206                                if (r10_bio->devs[i].bio->bi_end_io)
3207                                        rdev_dec_pending(conf->mirrors[d].rdev,
3208                                                         mddev);
3209                                if (r10_bio->devs[i].repl_bio &&
3210                                    r10_bio->devs[i].repl_bio->bi_end_io)
3211                                        rdev_dec_pending(
3212                                                conf->mirrors[d].replacement,
3213                                                mddev);
3214                        }
3215                        put_buf(r10_bio);
3216                        biolist = NULL;
3217                        goto giveup;
3218                }
3219        }
3220
3221        for (bio = biolist; bio ; bio=bio->bi_next) {
3222
3223                bio->bi_flags &= ~(BIO_POOL_MASK - 1);
3224                if (bio->bi_end_io)
3225                        bio->bi_flags |= 1 << BIO_UPTODATE;
3226                bio->bi_vcnt = 0;
3227                bio->bi_idx = 0;
3228                bio->bi_phys_segments = 0;
3229                bio->bi_size = 0;
3230        }
3231
3232        nr_sectors = 0;
3233        if (sector_nr + max_sync < max_sector)
3234                max_sector = sector_nr + max_sync;
3235        do {
3236                struct page *page;
3237                int len = PAGE_SIZE;
3238                if (sector_nr + (len>>9) > max_sector)
3239                        len = (max_sector - sector_nr) << 9;
3240                if (len == 0)
3241                        break;
3242                for (bio= biolist ; bio ; bio=bio->bi_next) {
3243                        struct bio *bio2;
3244                        page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
3245                        if (bio_add_page(bio, page, len, 0))
3246                                continue;
3247
3248                        /* stop here */
3249                        bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
3250                        for (bio2 = biolist;
3251                             bio2 && bio2 != bio;
3252                             bio2 = bio2->bi_next) {
3253                                /* remove last page from this bio */
3254                                bio2->bi_vcnt--;
3255                                bio2->bi_size -= len;
3256                                bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
3257                        }
3258                        goto bio_full;
3259                }
3260                nr_sectors += len>>9;
3261                sector_nr += len>>9;
3262        } while (biolist->bi_vcnt < RESYNC_PAGES);
3263 bio_full:
3264        r10_bio->sectors = nr_sectors;
3265
3266        while (biolist) {
3267                bio = biolist;
3268                biolist = biolist->bi_next;
3269
3270                bio->bi_next = NULL;
3271                r10_bio = bio->bi_private;
3272                r10_bio->sectors = nr_sectors;
3273
3274                if (bio->bi_end_io == end_sync_read) {
3275                        md_sync_acct(bio->bi_bdev, nr_sectors);
3276                        generic_make_request(bio);
3277                }
3278        }
3279
3280        if (sectors_skipped)
3281                /* pretend they weren't skipped, it makes
3282                 * no important difference in this case
3283                 */
3284                md_done_sync(mddev, sectors_skipped, 1);
3285
3286        return sectors_skipped + nr_sectors;
3287 giveup:
3288        /* There is nowhere to write, so all non-sync
3289         * drives must be failed or in resync, all drives
3290         * have a bad block, so try the next chunk...
3291         */
3292        if (sector_nr + max_sync < max_sector)
3293                max_sector = sector_nr + max_sync;
3294
3295        sectors_skipped += (max_sector - sector_nr);
3296        chunks_skipped ++;
3297        sector_nr = max_sector;
3298        goto skipped;
3299}
3300
3301static sector_t
3302raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3303{
3304        sector_t size;
3305        struct r10conf *conf = mddev->private;
3306
3307        if (!raid_disks)
3308                raid_disks = min(conf->geo.raid_disks,
3309                                 conf->prev.raid_disks);
3310        if (!sectors)
3311                sectors = conf->dev_sectors;
3312
3313        size = sectors >> conf->geo.chunk_shift;
3314        sector_div(size, conf->geo.far_copies);
3315        size = size * raid_disks;
3316        sector_div(size, conf->geo.near_copies);
3317
3318        return size << conf->geo.chunk_shift;
3319}
3320
3321static void calc_sectors(struct r10conf *conf, sector_t size)
3322{
3323        /* Calculate the number of sectors-per-device that will
3324         * actually be used, and set conf->dev_sectors and
3325         * conf->stride
3326         */
3327
3328        size = size >> conf->geo.chunk_shift;
3329        sector_div(size, conf->geo.far_copies);
3330        size = size * conf->geo.raid_disks;
3331        sector_div(size, conf->geo.near_copies);
3332        /* 'size' is now the number of chunks in the array */
3333        /* calculate "used chunks per device" */
3334        size = size * conf->copies;
3335
3336        /* We need to round up when dividing by raid_disks to
3337         * get the stride size.
3338         */
3339        size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3340
3341        conf->dev_sectors = size << conf->geo.chunk_shift;
3342
3343        if (conf->geo.far_offset)
3344                conf->geo.stride = 1 << conf->geo.chunk_shift;
3345        else {
3346                sector_div(size, conf->geo.far_copies);
3347                conf->geo.stride = size << conf->geo.chunk_shift;
3348        }
3349}
3350
3351enum geo_type {geo_new, geo_old, geo_start};
3352static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
3353{
3354        int nc, fc, fo;
3355        int layout, chunk, disks;
3356        switch (new) {
3357        case geo_old:
3358                layout = mddev->layout;
3359                chunk = mddev->chunk_sectors;
3360                disks = mddev->raid_disks - mddev->delta_disks;
3361                break;
3362        case geo_new:
3363                layout = mddev->new_layout;
3364                chunk = mddev->new_chunk_sectors;
3365                disks = mddev->raid_disks;
3366                break;
3367        default: /* avoid 'may be unused' warnings */
3368        case geo_start: /* new when starting reshape - raid_disks not
3369                         * updated yet. */
3370                layout = mddev->new_layout;
3371                chunk = mddev->new_chunk_sectors;
3372                disks = mddev->raid_disks + mddev->delta_disks;
3373                break;
3374        }
3375        if (layout >> 17)
3376                return -1;
3377        if (chunk < (PAGE_SIZE >> 9) ||
3378            !is_power_of_2(chunk))
3379                return -2;
3380        nc = layout & 255;
3381        fc = (layout >> 8) & 255;
3382        fo = layout & (1<<16);
3383        geo->raid_disks = disks;
3384        geo->near_copies = nc;
3385        geo->far_copies = fc;
3386        geo->far_offset = fo;
3387        geo->chunk_mask = chunk - 1;
3388        geo->chunk_shift = ffz(~chunk);
3389        return nc*fc;
3390}
3391
3392static struct r10conf *setup_conf(struct mddev *mddev)
3393{
3394        struct r10conf *conf = NULL;
3395        int err = -EINVAL;
3396        struct geom geo;
3397        int copies;
3398
3399        copies = setup_geo(&geo, mddev, geo_new);
3400
3401        if (copies == -2) {
3402                printk(KERN_ERR "md/raid10:%s: chunk size must be "
3403                       "at least PAGE_SIZE(%ld) and be a power of 2.\n",
3404                       mdname(mddev), PAGE_SIZE);
3405                goto out;
3406        }
3407
3408        if (copies < 2 || copies > mddev->raid_disks) {
3409                printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
3410                       mdname(mddev), mddev->new_layout);
3411                goto out;
3412        }
3413
3414        err = -ENOMEM;
3415        conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
3416        if (!conf)
3417                goto out;
3418
3419        /* FIXME calc properly */
3420        conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks +
3421                                                            max(0,mddev->delta_disks)),
3422                                GFP_KERNEL);
3423        if (!conf->mirrors)
3424                goto out;
3425
3426        conf->tmppage = alloc_page(GFP_KERNEL);
3427        if (!conf->tmppage)
3428                goto out;
3429
3430        conf->geo = geo;
3431        conf->copies = copies;
3432        conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
3433                                           r10bio_pool_free, conf);
3434        if (!conf->r10bio_pool)
3435                goto out;
3436
3437        calc_sectors(conf, mddev->dev_sectors);
3438        if (mddev->reshape_position == MaxSector) {
3439                conf->prev = conf->geo;
3440                conf->reshape_progress = MaxSector;
3441        } else {
3442                if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {
3443                        err = -EINVAL;
3444                        goto out;
3445                }
3446                conf->reshape_progress = mddev->reshape_position;
3447                if (conf->prev.far_offset)
3448                        conf->prev.stride = 1 << conf->prev.chunk_shift;
3449                else
3450                        /* far_copies must be 1 */
3451                        conf->prev.stride = conf->dev_sectors;
3452        }
3453        spin_lock_init(&conf->device_lock);
3454        INIT_LIST_HEAD(&conf->retry_list);
3455
3456        spin_lock_init(&conf->resync_lock);
3457        init_waitqueue_head(&conf->wait_barrier);
3458
3459        conf->thread = md_register_thread(raid10d, mddev, "raid10");
3460        if (!conf->thread)
3461                goto out;
3462
3463        conf->mddev = mddev;
3464        return conf;
3465
3466 out:
3467        if (err == -ENOMEM)
3468                printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
3469                       mdname(mddev));
3470        if (conf) {
3471                if (conf->r10bio_pool)
3472                        mempool_destroy(conf->r10bio_pool);
3473                kfree(conf->mirrors);
3474                safe_put_page(conf->tmppage);
3475                kfree(conf);
3476        }
3477        return ERR_PTR(err);
3478}
3479
3480static int run(struct mddev *mddev)
3481{
3482        struct r10conf *conf;
3483        int i, disk_idx, chunk_size;
3484        struct raid10_info *disk;
3485        struct md_rdev *rdev;
3486        sector_t size;
3487        sector_t min_offset_diff = 0;
3488        int first = 1;
3489
3490        if (mddev->private == NULL) {
3491                conf = setup_conf(mddev);
3492                if (IS_ERR(conf))
3493                        return PTR_ERR(conf);
3494                mddev->private = conf;
3495        }
3496        conf = mddev->private;
3497        if (!conf)
3498                goto out;
3499
3500        mddev->thread = conf->thread;
3501        conf->thread = NULL;
3502
3503        chunk_size = mddev->chunk_sectors << 9;
3504        if (mddev->queue) {
3505                blk_queue_io_min(mddev->queue, chunk_size);
3506                if (conf->geo.raid_disks % conf->geo.near_copies)
3507                        blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
3508                else
3509                        blk_queue_io_opt(mddev->queue, chunk_size *
3510                                         (conf->geo.raid_disks / conf->geo.near_copies));
3511        }
3512
3513        rdev_for_each(rdev, mddev) {
3514                long long diff;
3515                struct request_queue *q;
3516
3517                disk_idx = rdev->raid_disk;
3518                if (disk_idx < 0)
3519                        continue;
3520                if (disk_idx >= conf->geo.raid_disks &&
3521                    disk_idx >= conf->prev.raid_disks)
3522                        continue;
3523                disk = conf->mirrors + disk_idx;
3524
3525                if (test_bit(Replacement, &rdev->flags)) {
3526                        if (disk->replacement)
3527                                goto out_free_conf;
3528                        disk->replacement = rdev;
3529                } else {
3530                        if (disk->rdev)
3531                                goto out_free_conf;
3532                        disk->rdev = rdev;
3533                }
3534                q = bdev_get_queue(rdev->bdev);
3535                if (q->merge_bvec_fn)
3536                        mddev->merge_check_needed = 1;
3537                diff = (rdev->new_data_offset - rdev->data_offset);
3538                if (!mddev->reshape_backwards)
3539                        diff = -diff;
3540                if (diff < 0)
3541                        diff = 0;
3542                if (first || diff < min_offset_diff)
3543                        min_offset_diff = diff;
3544
3545                if (mddev->gendisk)
3546                        disk_stack_limits(mddev->gendisk, rdev->bdev,
3547                                          rdev->data_offset << 9);
3548
3549                disk->head_position = 0;
3550        }
3551
3552        /* need to check that every block has at least one working mirror */
3553        if (!enough(conf, -1)) {
3554                printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n",
3555                       mdname(mddev));
3556                goto out_free_conf;
3557        }
3558
3559        if (conf->reshape_progress != MaxSector) {
3560                /* must ensure that shape change is supported */
3561                if (conf->geo.far_copies != 1 &&
3562                    conf->geo.far_offset == 0)
3563                        goto out_free_conf;
3564                if (conf->prev.far_copies != 1 &&
3565                    conf->geo.far_offset == 0)
3566                        goto out_free_conf;
3567        }
3568
3569        mddev->degraded = 0;
3570        for (i = 0;
3571             i < conf->geo.raid_disks
3572                     || i < conf->prev.raid_disks;
3573             i++) {
3574
3575                disk = conf->mirrors + i;
3576
3577                if (!disk->rdev && disk->replacement) {
3578                        /* The replacement is all we have - use it */
3579                        disk->rdev = disk->replacement;
3580                        disk->replacement = NULL;
3581                        clear_bit(Replacement, &disk->rdev->flags);
3582                }
3583
3584                if (!disk->rdev ||
3585                    !test_bit(In_sync, &disk->rdev->flags)) {
3586                        disk->head_position = 0;
3587                        mddev->degraded++;
3588                        if (disk->rdev)
3589                                conf->fullsync = 1;
3590                }
3591                disk->recovery_disabled = mddev->recovery_disabled - 1;
3592        }
3593
3594        if (mddev->recovery_cp != MaxSector)
3595                printk(KERN_NOTICE "md/raid10:%s: not clean"
3596                       " -- starting background reconstruction\n",
3597                       mdname(mddev));
3598        printk(KERN_INFO
3599                "md/raid10:%s: active with %d out of %d devices\n",
3600                mdname(mddev), conf->geo.raid_disks - mddev->degraded,
3601                conf->geo.raid_disks);
3602        /*
3603         * Ok, everything is just fine now
3604         */
3605        mddev->dev_sectors = conf->dev_sectors;
3606        size = raid10_size(mddev, 0, 0);
3607        md_set_array_sectors(mddev, size);
3608        mddev->resync_max_sectors = size;
3609
3610        if (mddev->queue) {
3611                int stripe = conf->geo.raid_disks *
3612                        ((mddev->chunk_sectors << 9) / PAGE_SIZE);
3613                mddev->queue->backing_dev_info.congested_fn = raid10_congested;
3614                mddev->queue->backing_dev_info.congested_data = mddev;
3615
3616                /* Calculate max read-ahead size.
3617                 * We need to readahead at least twice a whole stripe....
3618                 * maybe...
3619                 */
3620                stripe /= conf->geo.near_copies;
3621                if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
3622                        mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
3623                blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec);
3624        }
3625
3626
3627        if (md_integrity_register(mddev))
3628                goto out_free_conf;
3629
3630        if (conf->reshape_progress != MaxSector) {
3631                unsigned long before_length, after_length;
3632
3633                before_length = ((1 << conf->prev.chunk_shift) *
3634                                 conf->prev.far_copies);
3635                after_length = ((1 << conf->geo.chunk_shift) *
3636                                conf->geo.far_copies);
3637
3638                if (max(before_length, after_length) > min_offset_diff) {
3639                        /* This cannot work */
3640                        printk("md/raid10: offset difference not enough to continue reshape\n");
3641                        goto out_free_conf;
3642                }
3643                conf->offset_diff = min_offset_diff;
3644
3645                conf->reshape_safe = conf->reshape_progress;
3646                clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
3647                clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
3648                set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
3649                set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
3650                mddev->sync_thread = md_register_thread(md_do_sync, mddev,
3651                                                        "reshape");
3652        }
3653
3654        return 0;
3655
3656out_free_conf:
3657        md_unregister_thread(&mddev->thread);
3658        if (conf->r10bio_pool)
3659                mempool_destroy(conf->r10bio_pool);
3660        safe_put_page(conf->tmppage);
3661        kfree(conf->mirrors);
3662        kfree(conf);
3663        mddev->private = NULL;
3664out:
3665        return -EIO;
3666}
3667
3668static int stop(struct mddev *mddev)
3669{
3670        struct r10conf *conf = mddev->private;
3671
3672        raise_barrier(conf, 0);
3673        lower_barrier(conf);
3674
3675        md_unregister_thread(&mddev->thread);
3676        if (mddev->queue)
3677                /* the unplug fn references 'conf'*/
3678                blk_sync_queue(mddev->queue);
3679
3680        if (conf->r10bio_pool)
3681                mempool_destroy(conf->r10bio_pool);
3682        kfree(conf->mirrors);
3683        kfree(conf);
3684        mddev->private = NULL;
3685        return 0;
3686}
3687
3688static void raid10_quiesce(struct mddev *mddev, int state)
3689{
3690        struct r10conf *conf = mddev->private;
3691
3692        switch(state) {
3693        case 1:
3694                raise_barrier(conf, 0);
3695                break;
3696        case 0:
3697                lower_barrier(conf);
3698                break;
3699        }
3700}
3701
3702static int raid10_resize(struct mddev *mddev, sector_t sectors)
3703{
3704        /* Resize of 'far' arrays is not supported.
3705         * For 'near' and 'offset' arrays we can set the
3706         * number of sectors used to be an appropriate multiple
3707         * of the chunk size.
3708         * For 'offset', this is far_copies*chunksize.
3709         * For 'near' the multiplier is the LCM of
3710         * near_copies and raid_disks.
3711         * So if far_copies > 1 && !far_offset, fail.
3712         * Else find LCM(raid_disks, near_copy)*far_copies and
3713         * multiply by chunk_size.  Then round to this number.
3714         * This is mostly done by raid10_size()
3715         */
3716        struct r10conf *conf = mddev->private;
3717        sector_t oldsize, size;
3718
3719        if (mddev->reshape_position != MaxSector)
3720                return -EBUSY;
3721
3722        if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
3723                return -EINVAL;
3724
3725        oldsize = raid10_size(mddev, 0, 0);
3726        size = raid10_size(mddev, sectors, 0);
3727        if (mddev->external_size &&
3728            mddev->array_sectors > size)
3729                return -EINVAL;
3730        if (mddev->bitmap) {
3731                int ret = bitmap_resize(mddev->bitmap, size, 0, 0);
3732                if (ret)
3733                        return ret;
3734        }
3735        md_set_array_sectors(mddev, size);
3736        set_capacity(mddev->gendisk, mddev->array_sectors);
3737        revalidate_disk(mddev->gendisk);
3738        if (sectors > mddev->dev_sectors &&
3739            mddev->recovery_cp > oldsize) {
3740                mddev->recovery_cp = oldsize;
3741                set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3742        }
3743        calc_sectors(conf, sectors);
3744        mddev->dev_sectors = conf->dev_sectors;
3745        mddev->resync_max_sectors = size;
3746        return 0;
3747}
3748
3749static void *raid10_takeover_raid0(struct mddev *mddev)
3750{
3751        struct md_rdev *rdev;
3752        struct r10conf *conf;
3753
3754        if (mddev->degraded > 0) {
3755                printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n",
3756                       mdname(mddev));
3757                return ERR_PTR(-EINVAL);
3758        }
3759
3760        /* Set new parameters */
3761        mddev->new_level = 10;
3762        /* new layout: far_copies = 1, near_copies = 2 */
3763        mddev->new_layout = (1<<8) + 2;
3764        mddev->new_chunk_sectors = mddev->chunk_sectors;
3765        mddev->delta_disks = mddev->raid_disks;
3766        mddev->raid_disks *= 2;
3767        /* make sure it will be not marked as dirty */
3768        mddev->recovery_cp = MaxSector;
3769
3770        conf = setup_conf(mddev);
3771        if (!IS_ERR(conf)) {
3772                rdev_for_each(rdev, mddev)
3773                        if (rdev->raid_disk >= 0)
3774                                rdev->new_raid_disk = rdev->raid_disk * 2;
3775                conf->barrier = 1;
3776        }
3777
3778        return conf;
3779}
3780
3781static void *raid10_takeover(struct mddev *mddev)
3782{
3783        struct r0conf *raid0_conf;
3784
3785        /* raid10 can take over:
3786         *  raid0 - providing it has only two drives
3787         */
3788        if (mddev->level == 0) {
3789                /* for raid0 takeover only one zone is supported */
3790                raid0_conf = mddev->private;
3791                if (raid0_conf->nr_strip_zones > 1) {
3792                        printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0"
3793                               " with more than one zone.\n",
3794                               mdname(mddev));
3795                        return ERR_PTR(-EINVAL);
3796                }
3797                return raid10_takeover_raid0(mddev);
3798        }
3799        return ERR_PTR(-EINVAL);
3800}
3801
3802static int raid10_check_reshape(struct mddev *mddev)
3803{
3804        /* Called when there is a request to change
3805         * - layout (to ->new_layout)
3806         * - chunk size (to ->new_chunk_sectors)
3807         * - raid_disks (by delta_disks)
3808         * or when trying to restart a reshape that was ongoing.
3809         *
3810         * We need to validate the request and possibly allocate
3811         * space if that might be an issue later.
3812         *
3813         * Currently we reject any reshape of a 'far' mode array,
3814         * allow chunk size to change if new is generally acceptable,
3815         * allow raid_disks to increase, and allow
3816         * a switch between 'near' mode and 'offset' mode.
3817         */
3818        struct r10conf *conf = mddev->private;
3819        struct geom geo;
3820
3821        if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
3822                return -EINVAL;
3823
3824        if (setup_geo(&geo, mddev, geo_start) != conf->copies)
3825                /* mustn't change number of copies */
3826                return -EINVAL;
3827        if (geo.far_copies > 1 && !geo.far_offset)
3828                /* Cannot switch to 'far' mode */
3829                return -EINVAL;
3830
3831        if (mddev->array_sectors & geo.chunk_mask)
3832                        /* not factor of array size */
3833                        return -EINVAL;
3834
3835        if (!enough(conf, -1))
3836                return -EINVAL;
3837
3838        kfree(conf->mirrors_new);
3839        conf->mirrors_new = NULL;
3840        if (mddev->delta_disks > 0) {
3841                /* allocate new 'mirrors' list */
3842                conf->mirrors_new = kzalloc(
3843                        sizeof(struct raid10_info)
3844                        *(mddev->raid_disks +
3845                          mddev->delta_disks),
3846                        GFP_KERNEL);
3847                if (!conf->mirrors_new)
3848                        return -ENOMEM;
3849        }
3850        return 0;
3851}
3852
3853/*
3854 * Need to check if array has failed when deciding whether to:
3855 *  - start an array
3856 *  - remove non-faulty devices
3857 *  - add a spare
3858 *  - allow a reshape
3859 * This determination is simple when no reshape is happening.
3860 * However if there is a reshape, we need to carefully check
3861 * both the before and after sections.
3862 * This is because some failed devices may only affect one
3863 * of the two sections, and some non-in_sync devices may
3864 * be insync in the section most affected by failed devices.
3865 */
3866static int calc_degraded(struct r10conf *conf)
3867{
3868        int degraded, degraded2;
3869        int i;
3870
3871        rcu_read_lock();
3872        degraded = 0;
3873        /* 'prev' section first */
3874        for (i = 0; i < conf->prev.raid_disks; i++) {
3875                struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
3876                if (!rdev || test_bit(Faulty, &rdev->flags))
3877                        degraded++;
3878                else if (!test_bit(In_sync, &rdev->flags))
3879                        /* When we can reduce the number of devices in
3880                         * an array, this might not contribute to
3881                         * 'degraded'.  It does now.
3882                         */
3883                        degraded++;
3884        }
3885        rcu_read_unlock();
3886        if (conf->geo.raid_disks == conf->prev.raid_disks)
3887                return degraded;
3888        rcu_read_lock();
3889        degraded2 = 0;
3890        for (i = 0; i < conf->geo.raid_disks; i++) {
3891                struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
3892                if (!rdev || test_bit(Faulty, &rdev->flags))
3893                        degraded2++;
3894                else if (!test_bit(In_sync, &rdev->flags)) {
3895                        /* If reshape is increasing the number of devices,
3896                         * this section has already been recovered, so
3897                         * it doesn't contribute to degraded.
3898                         * else it does.
3899                         */
3900                        if (conf->geo.raid_disks <= conf->prev.raid_disks)
3901                                degraded2++;
3902                }
3903        }
3904        rcu_read_unlock();
3905        if (degraded2 > degraded)
3906                return degraded2;
3907        return degraded;
3908}
3909
3910static int raid10_start_reshape(struct mddev *mddev)
3911{
3912        /* A 'reshape' has been requested. This commits
3913         * the various 'new' fields and sets MD_RECOVER_RESHAPE
3914         * This also checks if there are enough spares and adds them
3915         * to the array.
3916         * We currently require enough spares to make the final
3917         * array non-degraded.  We also require that the difference
3918         * between old and new data_offset - on each device - is
3919         * enough that we never risk over-writing.
3920         */
3921
3922        unsigned long before_length, after_length;
3923        sector_t min_offset_diff = 0;
3924        int first = 1;
3925        struct geom new;
3926        struct r10conf *conf = mddev->private;
3927        struct md_rdev *rdev;
3928        int spares = 0;
3929        int ret;
3930
3931        if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
3932                return -EBUSY;
3933
3934        if (setup_geo(&new, mddev, geo_start) != conf->copies)
3935                return -EINVAL;
3936
3937        before_length = ((1 << conf->prev.chunk_shift) *
3938                         conf->prev.far_copies);
3939        after_length = ((1 << conf->geo.chunk_shift) *
3940                        conf->geo.far_copies);
3941
3942        rdev_for_each(rdev, mddev) {
3943                if (!test_bit(In_sync, &rdev->flags)
3944                    && !test_bit(Faulty, &rdev->flags))
3945                        spares++;
3946                if (rdev->raid_disk >= 0) {
3947                        long long diff = (rdev->new_data_offset
3948                                          - rdev->data_offset);
3949                        if (!mddev->reshape_backwards)
3950                                diff = -diff;
3951                        if (diff < 0)
3952                                diff = 0;
3953                        if (first || diff < min_offset_diff)
3954                                min_offset_diff = diff;
3955                }
3956        }
3957
3958        if (max(before_length, after_length) > min_offset_diff)
3959                return -EINVAL;
3960
3961        if (spares < mddev->delta_disks)
3962                return -EINVAL;
3963
3964        conf->offset_diff = min_offset_diff;
3965        spin_lock_irq(&conf->device_lock);
3966        if (conf->mirrors_new) {
3967                memcpy(conf->mirrors_new, conf->mirrors,
3968                       sizeof(struct raid10_info)*conf->prev.raid_disks);
3969                smp_mb();
3970                kfree(conf->mirrors_old); /* FIXME and elsewhere */
3971                conf->mirrors_old = conf->mirrors;
3972                conf->mirrors = conf->mirrors_new;
3973                conf->mirrors_new = NULL;
3974        }
3975        setup_geo(&conf->geo, mddev, geo_start);
3976        smp_mb();
3977        if (mddev->reshape_backwards) {
3978                sector_t size = raid10_size(mddev, 0, 0);
3979                if (size < mddev->array_sectors) {
3980                        spin_unlock_irq(&conf->device_lock);
3981                        printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n",
3982                               mdname(mddev));
3983                        return -EINVAL;
3984                }
3985                mddev->resync_max_sectors = size;
3986                conf->reshape_progress = size;
3987        } else
3988                conf->reshape_progress = 0;
3989        spin_unlock_irq(&conf->device_lock);
3990
3991        if (mddev->delta_disks && mddev->bitmap) {
3992                ret = bitmap_resize(mddev->bitmap,
3993                                    raid10_size(mddev, 0,
3994                                                conf->geo.raid_disks),
3995                                    0, 0);
3996                if (ret)
3997                        goto abort;
3998        }
3999        if (mddev->delta_disks > 0) {
4000                rdev_for_each(rdev, mddev)
4001                        if (rdev->raid_disk < 0 &&
4002                            !test_bit(Faulty, &rdev->flags)) {
4003                                if (raid10_add_disk(mddev, rdev) == 0) {
4004                                        if (rdev->raid_disk >=
4005                                            conf->prev.raid_disks)
4006                                                set_bit(In_sync, &rdev->flags);
4007                                        else
4008                                                rdev->recovery_offset = 0;
4009
4010                                        if (sysfs_link_rdev(mddev, rdev))
4011                                                /* Failure here  is OK */;
4012                                }
4013                        } else if (rdev->raid_disk >= conf->prev.raid_disks
4014                                   && !test_bit(Faulty, &rdev->flags)) {
4015                                /* This is a spare that was manually added */
4016                                set_bit(In_sync, &rdev->flags);
4017                        }
4018        }
4019        /* When a reshape changes the number of devices,
4020         * ->degraded is measured against the larger of the
4021         * pre and  post numbers.
4022         */
4023        spin_lock_irq(&conf->device_lock);
4024        mddev->degraded = calc_degraded(conf);
4025        spin_unlock_irq(&conf->device_lock);
4026        mddev->raid_disks = conf->geo.raid_disks;
4027        mddev->reshape_position = conf->reshape_progress;
4028        set_bit(MD_CHANGE_DEVS, &mddev->flags);
4029
4030        clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
4031        clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4032        set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
4033        set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
4034
4035        mddev->sync_thread = md_register_thread(md_do_sync, mddev,
4036                                                "reshape");
4037        if (!mddev->sync_thread) {
4038                ret = -EAGAIN;
4039                goto abort;
4040        }
4041        conf->reshape_checkpoint = jiffies;
4042        md_wakeup_thread(mddev->sync_thread);
4043        md_new_event(mddev);
4044        return 0;
4045
4046abort:
4047        mddev->recovery = 0;
4048        spin_lock_irq(&conf->device_lock);
4049        conf->geo = conf->prev;
4050        mddev->raid_disks = conf->geo.raid_disks;
4051        rdev_for_each(rdev, mddev)
4052                rdev->new_data_offset = rdev->data_offset;
4053        smp_wmb();
4054        conf->reshape_progress = MaxSector;
4055        mddev->reshape_position = MaxSector;
4056        spin_unlock_irq(&conf->device_lock);
4057        return ret;
4058}
4059
4060/* Calculate the last device-address that could contain
4061 * any block from the chunk that includes the array-address 's'
4062 * and report the next address.
4063 * i.e. the address returned will be chunk-aligned and after
4064 * any data that is in the chunk containing 's'.
4065 */
4066static sector_t last_dev_address(sector_t s, struct geom *geo)
4067{
4068        s = (s | geo->chunk_mask) + 1;
4069        s >>= geo->chunk_shift;
4070        s *= geo->near_copies;
4071        s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
4072        s *= geo->far_copies;
4073        s <<= geo->chunk_shift;
4074        return s;
4075}
4076
4077/* Calculate the first device-address that could contain
4078 * any block from the chunk that includes the array-address 's'.
4079 * This too will be the start of a chunk
4080 */
4081static sector_t first_dev_address(sector_t s, struct geom *geo)
4082{
4083        s >>= geo->chunk_shift;
4084        s *= geo->near_copies;
4085        sector_div(s, geo->raid_disks);
4086        s *= geo->far_copies;
4087        s <<= geo->chunk_shift;
4088        return s;
4089}
4090
4091static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
4092                                int *skipped)
4093{
4094        /* We simply copy at most one chunk (smallest of old and new)
4095         * at a time, possibly less if that exceeds RESYNC_PAGES,
4096         * or we hit a bad block or something.
4097         * This might mean we pause for normal IO in the middle of
4098         * a chunk, but that is not a problem was mddev->reshape_position
4099         * can record any location.
4100         *
4101         * If we will want to write to a location that isn't
4102         * yet recorded as 'safe' (i.e. in metadata on disk) then
4103         * we need to flush all reshape requests and update the metadata.
4104         *
4105         * When reshaping forwards (e.g. to more devices), we interpret
4106         * 'safe' as the earliest block which might not have been copied
4107         * down yet.  We divide this by previous stripe size and multiply
4108         * by previous stripe length to get lowest device offset that we
4109         * cannot write to yet.
4110         * We interpret 'sector_nr' as an address that we want to write to.
4111         * From this we use last_device_address() to find where we might
4112         * write to, and first_device_address on the  'safe' position.
4113         * If this 'next' write position is after the 'safe' position,
4114         * we must update the metadata to increase the 'safe' position.
4115         *
4116         * When reshaping backwards, we round in the opposite direction
4117         * and perform the reverse test:  next write position must not be
4118         * less than current safe position.
4119         *
4120         * In all this the minimum difference in data offsets
4121         * (conf->offset_diff - always positive) allows a bit of slack,
4122         * so next can be after 'safe', but not by more than offset_disk
4123         *
4124         * We need to prepare all the bios here before we start any IO
4125         * to ensure the size we choose is acceptable to all devices.
4126         * The means one for each copy for write-out and an extra one for
4127         * read-in.
4128         * We store the read-in bio in ->master_bio and the others in
4129         * ->devs[x].bio and ->devs[x].repl_bio.
4130         */
4131        struct r10conf *conf = mddev->private;
4132        struct r10bio *r10_bio;
4133        sector_t next, safe, last;
4134        int max_sectors;
4135        int nr_sectors;
4136        int s;
4137        struct md_rdev *rdev;
4138        int need_flush = 0;
4139        struct bio *blist;
4140        struct bio *bio, *read_bio;
4141        int sectors_done = 0;
4142
4143        if (sector_nr == 0) {
4144                /* If restarting in the middle, skip the initial sectors */
4145                if (mddev->reshape_backwards &&
4146                    conf->reshape_progress < raid10_size(mddev, 0, 0)) {
4147                        sector_nr = (raid10_size(mddev, 0, 0)
4148                                     - conf->reshape_progress);
4149                } else if (!mddev->reshape_backwards &&
4150                           conf->reshape_progress > 0)
4151                        sector_nr = conf->reshape_progress;
4152                if (sector_nr) {
4153                        mddev->curr_resync_completed = sector_nr;
4154                        sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4155                        *skipped = 1;
4156                        return sector_nr;
4157                }
4158        }
4159
4160        /* We don't use sector_nr to track where we are up to
4161         * as that doesn't work well for ->reshape_backwards.
4162         * So just use ->reshape_progress.
4163         */
4164        if (mddev->reshape_backwards) {
4165                /* 'next' is the earliest device address that we might
4166                 * write to for this chunk in the new layout
4167                 */
4168                next = first_dev_address(conf->reshape_progress - 1,
4169                                         &conf->geo);
4170
4171                /* 'safe' is the last device address that we might read from
4172                 * in the old layout after a restart
4173                 */
4174                safe = last_dev_address(conf->reshape_safe - 1,
4175                                        &conf->prev);
4176
4177                if (next + conf->offset_diff < safe)
4178                        need_flush = 1;
4179
4180                last = conf->reshape_progress - 1;
4181                sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
4182                                               & conf->prev.chunk_mask);
4183                if (sector_nr + RESYNC_BLOCK_SIZE/512 < last)
4184                        sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512;
4185        } else {
4186                /* 'next' is after the last device address that we
4187                 * might write to for this chunk in the new layout
4188                 */
4189                next = last_dev_address(conf->reshape_progress, &conf->geo);
4190
4191                /* 'safe' is the earliest device address that we might
4192                 * read from in the old layout after a restart
4193                 */
4194                safe = first_dev_address(conf->reshape_safe, &conf->prev);
4195
4196                /* Need to update metadata if 'next' might be beyond 'safe'
4197                 * as that would possibly corrupt data
4198                 */
4199                if (next > safe + conf->offset_diff)
4200                        need_flush = 1;
4201
4202                sector_nr = conf->reshape_progress;
4203                last  = sector_nr | (conf->geo.chunk_mask
4204                                     & conf->prev.chunk_mask);
4205
4206                if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last)
4207                        last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1;
4208        }
4209
4210        if (need_flush ||
4211            time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4212                /* Need to update reshape_position in metadata */
4213                wait_barrier(conf);
4214                mddev->reshape_position = conf->reshape_progress;
4215                if (mddev->reshape_backwards)
4216                        mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
4217                                - conf->reshape_progress;
4218                else
4219                        mddev->curr_resync_completed = conf->reshape_progress;
4220                conf->reshape_checkpoint = jiffies;
4221                set_bit(MD_CHANGE_DEVS, &mddev->flags);
4222                md_wakeup_thread(mddev->thread);
4223                wait_event(mddev->sb_wait, mddev->flags == 0 ||
4224                           kthread_should_stop());
4225                conf->reshape_safe = mddev->reshape_position;
4226                allow_barrier(conf);
4227        }
4228
4229read_more:
4230        /* Now schedule reads for blocks from sector_nr to last */
4231        r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
4232        raise_barrier(conf, sectors_done != 0);
4233        atomic_set(&r10_bio->remaining, 0);
4234        r10_bio->mddev = mddev;
4235        r10_bio->sector = sector_nr;
4236        set_bit(R10BIO_IsReshape, &r10_bio->state);
4237        r10_bio->sectors = last - sector_nr + 1;
4238        rdev = read_balance(conf, r10_bio, &max_sectors);
4239        BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));
4240
4241        if (!rdev) {
4242                /* Cannot read from here, so need to record bad blocks
4243                 * on all the target devices.
4244                 */
4245                // FIXME
4246                set_bit(MD_RECOVERY_INTR, &mddev->recovery);
4247                return sectors_done;
4248        }
4249
4250        read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);
4251
4252        read_bio->bi_bdev = rdev->bdev;
4253        read_bio->bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
4254                               + rdev->data_offset);
4255        read_bio->bi_private = r10_bio;
4256        read_bio->bi_end_io = end_sync_read;
4257        read_bio->bi_rw = READ;
4258        read_bio->bi_flags &= ~(BIO_POOL_MASK - 1);
4259        read_bio->bi_flags |= 1 << BIO_UPTODATE;
4260        read_bio->bi_vcnt = 0;
4261        read_bio->bi_idx = 0;
4262        read_bio->bi_size = 0;
4263        r10_bio->master_bio = read_bio;
4264        r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;
4265
4266        /* Now find the locations in the new layout */
4267        __raid10_find_phys(&conf->geo, r10_bio);
4268
4269        blist = read_bio;
4270        read_bio->bi_next = NULL;
4271
4272        for (s = 0; s < conf->copies*2; s++) {
4273                struct bio *b;
4274                int d = r10_bio->devs[s/2].devnum;
4275                struct md_rdev *rdev2;
4276                if (s&1) {
4277                        rdev2 = conf->mirrors[d].replacement;
4278                        b = r10_bio->devs[s/2].repl_bio;
4279                } else {
4280                        rdev2 = conf->mirrors[d].rdev;
4281                        b = r10_bio->devs[s/2].bio;
4282                }
4283                if (!rdev2 || test_bit(Faulty, &rdev2->flags))
4284                        continue;
4285                b->bi_bdev = rdev2->bdev;
4286                b->bi_sector = r10_bio->devs[s/2].addr + rdev2->new_data_offset;
4287                b->bi_private = r10_bio;
4288                b->bi_end_io = end_reshape_write;
4289                b->bi_rw = WRITE;
4290                b->bi_flags &= ~(BIO_POOL_MASK - 1);
4291                b->bi_flags |= 1 << BIO_UPTODATE;
4292                b->bi_next = blist;
4293                b->bi_vcnt = 0;
4294                b->bi_idx = 0;
4295                b->bi_size = 0;
4296                blist = b;
4297        }
4298
4299        /* Now add as many pages as possible to all of these bios. */
4300
4301        nr_sectors = 0;
4302        for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
4303                struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page;
4304                int len = (max_sectors - s) << 9;
4305                if (len > PAGE_SIZE)
4306                        len = PAGE_SIZE;
4307                for (bio = blist; bio ; bio = bio->bi_next) {
4308                        struct bio *bio2;
4309                        if (bio_add_page(bio, page, len, 0))
4310                                continue;
4311
4312                        /* Didn't fit, must stop */
4313                        for (bio2 = blist;
4314                             bio2 && bio2 != bio;
4315                             bio2 = bio2->bi_next) {
4316                                /* Remove last page from this bio */
4317                                bio2->bi_vcnt--;
4318                                bio2->bi_size -= len;
4319                                bio2->bi_flags &= ~(1<<BIO_SEG_VALID);
4320                        }
4321                        goto bio_full;
4322                }
4323                sector_nr += len >> 9;
4324                nr_sectors += len >> 9;
4325        }
4326bio_full:
4327        r10_bio->sectors = nr_sectors;
4328
4329        /* Now submit the read */
4330        md_sync_acct(read_bio->bi_bdev, r10_bio->sectors);
4331        atomic_inc(&r10_bio->remaining);
4332        read_bio->bi_next = NULL;
4333        generic_make_request(read_bio);
4334        sector_nr += nr_sectors;
4335        sectors_done += nr_sectors;
4336        if (sector_nr <= last)
4337                goto read_more;
4338
4339        /* Now that we have done the whole section we can
4340         * update reshape_progress
4341         */
4342        if (mddev->reshape_backwards)
4343                conf->reshape_progress -= sectors_done;
4344        else
4345                conf->reshape_progress += sectors_done;
4346
4347        return sectors_done;
4348}
4349
4350static void end_reshape_request(struct r10bio *r10_bio);
4351static int handle_reshape_read_error(struct mddev *mddev,
4352                                     struct r10bio *r10_bio);
4353static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
4354{
4355        /* Reshape read completed.  Hopefully we have a block
4356         * to write out.
4357         * If we got a read error then we do sync 1-page reads from
4358         * elsewhere until we find the data - or give up.
4359         */
4360        struct r10conf *conf = mddev->private;
4361        int s;
4362
4363        if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
4364                if (handle_reshape_read_error(mddev, r10_bio) < 0) {
4365                        /* Reshape has been aborted */
4366                        md_done_sync(mddev, r10_bio->sectors, 0);
4367                        return;
4368                }
4369
4370        /* We definitely have the data in the pages, schedule the
4371         * writes.
4372         */
4373        atomic_set(&r10_bio->remaining, 1);
4374        for (s = 0; s < conf->copies*2; s++) {
4375                struct bio *b;
4376                int d = r10_bio->devs[s/2].devnum;
4377                struct md_rdev *rdev;
4378                if (s&1) {
4379                        rdev = conf->mirrors[d].replacement;
4380                        b = r10_bio->devs[s/2].repl_bio;
4381                } else {
4382                        rdev = conf->mirrors[d].rdev;
4383                        b = r10_bio->devs[s/2].bio;
4384                }
4385                if (!rdev || test_bit(Faulty, &rdev->flags))
4386                        continue;
4387                atomic_inc(&rdev->nr_pending);
4388                md_sync_acct(b->bi_bdev, r10_bio->sectors);
4389                atomic_inc(&r10_bio->remaining);
4390                b->bi_next = NULL;
4391                generic_make_request(b);
4392        }
4393        end_reshape_request(r10_bio);
4394}
4395
4396static void end_reshape(struct r10conf *conf)
4397{
4398        if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
4399                return;
4400
4401        spin_lock_irq(&conf->device_lock);
4402        conf->prev = conf->geo;
4403        md_finish_reshape(conf->mddev);
4404        smp_wmb();
4405        conf->reshape_progress = MaxSector;
4406        spin_unlock_irq(&conf->device_lock);
4407
4408        /* read-ahead size must cover two whole stripes, which is
4409         * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
4410         */
4411        if (conf->mddev->queue) {
4412                int stripe = conf->geo.raid_disks *
4413                        ((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
4414                stripe /= conf->geo.near_copies;
4415                if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
4416                        conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
4417        }
4418        conf->fullsync = 0;
4419}
4420
4421
4422static int handle_reshape_read_error(struct mddev *mddev,
4423                                     struct r10bio *r10_bio)
4424{
4425        /* Use sync reads to get the blocks from somewhere else */
4426        int sectors = r10_bio->sectors;
4427        struct r10conf *conf = mddev->private;
4428        struct {
4429                struct r10bio r10_bio;
4430                struct r10dev devs[conf->copies];
4431        } on_stack;
4432        struct r10bio *r10b = &on_stack.r10_bio;
4433        int slot = 0;
4434        int idx = 0;
4435        struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec;
4436
4437        r10b->sector = r10_bio->sector;
4438        __raid10_find_phys(&conf->prev, r10b);
4439
4440        while (sectors) {
4441                int s = sectors;
4442                int success = 0;
4443                int first_slot = slot;
4444
4445                if (s > (PAGE_SIZE >> 9))
4446                        s = PAGE_SIZE >> 9;
4447
4448                while (!success) {
4449                        int d = r10b->devs[slot].devnum;
4450                        struct md_rdev *rdev = conf->mirrors[d].rdev;
4451                        sector_t addr;
4452                        if (rdev == NULL ||
4453                            test_bit(Faulty, &rdev->flags) ||
4454                            !test_bit(In_sync, &rdev->flags))
4455                                goto failed;
4456
4457                        addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
4458                        success = sync_page_io(rdev,
4459                                               addr,
4460                                               s << 9,
4461                                               bvec[idx].bv_page,
4462                                               READ, false);
4463                        if (success)
4464                                break;
4465                failed:
4466                        slot++;
4467                        if (slot >= conf->copies)
4468                                slot = 0;
4469                        if (slot == first_slot)
4470                                break;
4471                }
4472                if (!success) {
4473                        /* couldn't read this block, must give up */
4474                        set_bit(MD_RECOVERY_INTR,
4475                                &mddev->recovery);
4476                        return -EIO;
4477                }
4478                sectors -= s;
4479                idx++;
4480        }
4481        return 0;
4482}
4483
4484static void end_reshape_write(struct bio *bio, int error)
4485{
4486        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
4487        struct r10bio *r10_bio = bio->bi_private;
4488        struct mddev *mddev = r10_bio->mddev;
4489        struct r10conf *conf = mddev->private;
4490        int d;
4491        int slot;
4492        int repl;
4493        struct md_rdev *rdev = NULL;
4494
4495        d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
4496        if (repl)
4497                rdev = conf->mirrors[d].replacement;
4498        if (!rdev) {
4499                smp_mb();
4500                rdev = conf->mirrors[d].rdev;
4501        }
4502
4503        if (!uptodate) {
4504                /* FIXME should record badblock */
4505                md_error(mddev, rdev);
4506        }
4507
4508        rdev_dec_pending(rdev, mddev);
4509        end_reshape_request(r10_bio);
4510}
4511
4512static void end_reshape_request(struct r10bio *r10_bio)
4513{
4514        if (!atomic_dec_and_test(&r10_bio->remaining))
4515                return;
4516        md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);
4517        bio_put(r10_bio->master_bio);
4518        put_buf(r10_bio);
4519}
4520
4521static void raid10_finish_reshape(struct mddev *mddev)
4522{
4523        struct r10conf *conf = mddev->private;
4524
4525        if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
4526                return;
4527
4528        if (mddev->delta_disks > 0) {
4529                sector_t size = raid10_size(mddev, 0, 0);
4530                md_set_array_sectors(mddev, size);
4531                if (mddev->recovery_cp > mddev->resync_max_sectors) {
4532                        mddev->recovery_cp = mddev->resync_max_sectors;
4533                        set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
4534                }
4535                mddev->resync_max_sectors = size;
4536                set_capacity(mddev->gendisk, mddev->array_sectors);
4537                revalidate_disk(mddev->gendisk);
4538        } else {
4539                int d;
4540                for (d = conf->geo.raid_disks ;
4541                     d < conf->geo.raid_disks - mddev->delta_disks;
4542                     d++) {
4543                        struct md_rdev *rdev = conf->mirrors[d].rdev;
4544                        if (rdev)
4545                                clear_bit(In_sync, &rdev->flags);
4546                        rdev = conf->mirrors[d].replacement;
4547                        if (rdev)
4548                                clear_bit(In_sync, &rdev->flags);
4549                }
4550        }
4551        mddev->layout = mddev->new_layout;
4552        mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
4553        mddev->reshape_position = MaxSector;
4554        mddev->delta_disks = 0;
4555        mddev->reshape_backwards = 0;
4556}
4557
4558static struct md_personality raid10_personality =
4559{
4560        .name           = "raid10",
4561        .level          = 10,
4562        .owner          = THIS_MODULE,
4563        .make_request   = make_request,
4564        .run            = run,
4565        .stop           = stop,
4566        .status         = status,
4567        .error_handler  = error,
4568        .hot_add_disk   = raid10_add_disk,
4569        .hot_remove_disk= raid10_remove_disk,
4570        .spare_active   = raid10_spare_active,
4571        .sync_request   = sync_request,
4572        .quiesce        = raid10_quiesce,
4573        .size           = raid10_size,
4574        .resize         = raid10_resize,
4575        .takeover       = raid10_takeover,
4576        .check_reshape  = raid10_check_reshape,
4577        .start_reshape  = raid10_start_reshape,
4578        .finish_reshape = raid10_finish_reshape,
4579};
4580
4581static int __init raid_init(void)
4582{
4583        return register_md_personality(&raid10_personality);
4584}
4585
4586static void raid_exit(void)
4587{
4588        unregister_md_personality(&raid10_personality);
4589}
4590
4591module_init(raid_init);
4592module_exit(raid_exit);
4593MODULE_LICENSE("GPL");
4594MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
4595MODULE_ALIAS("md-personality-9"); /* RAID10 */
4596MODULE_ALIAS("md-raid10");
4597MODULE_ALIAS("md-level-10");
4598
4599module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);
4600
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.