linux/drivers/md/raid1.c
<<
>>
Prefs
   1/*
   2 * raid1.c : Multiple Devices driver for Linux
   3 *
   4 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
   5 *
   6 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
   7 *
   8 * RAID-1 management functions.
   9 *
  10 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000
  11 *
  12 * Fixes to reconstruction by Jakob Ƙstergaard" <jakob@ostenfeld.dk>
  13 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>
  14 *
  15 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support
  16 * bitmapped intelligence in resync:
  17 *
  18 *      - bitmap marked during normal i/o
  19 *      - bitmap used to skip nondirty blocks during sync
  20 *
  21 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology:
  22 * - persistent bitmap code
  23 *
  24 * This program is free software; you can redistribute it and/or modify
  25 * it under the terms of the GNU General Public License as published by
  26 * the Free Software Foundation; either version 2, or (at your option)
  27 * any later version.
  28 *
  29 * You should have received a copy of the GNU General Public License
  30 * (for example /usr/src/linux/COPYING); if not, write to the Free
  31 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  32 */
  33
  34#include "dm-bio-list.h"
  35#include <linux/raid/raid1.h>
  36#include <linux/raid/bitmap.h>
  37
  38#define DEBUG 0
  39#if DEBUG
  40#define PRINTK(x...) printk(x)
  41#else
  42#define PRINTK(x...)
  43#endif
  44
  45/*
  46 * Number of guaranteed r1bios in case of extreme VM load:
  47 */
  48#define NR_RAID1_BIOS 256
  49
  50
  51static void unplug_slaves(mddev_t *mddev);
  52
  53static void allow_barrier(conf_t *conf);
  54static void lower_barrier(conf_t *conf);
  55
  56static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
  57{
  58        struct pool_info *pi = data;
  59        r1bio_t *r1_bio;
  60        int size = offsetof(r1bio_t, bios[pi->raid_disks]);
  61
  62        /* allocate a r1bio with room for raid_disks entries in the bios array */
  63        r1_bio = kzalloc(size, gfp_flags);
  64        if (!r1_bio)
  65                unplug_slaves(pi->mddev);
  66
  67        return r1_bio;
  68}
  69
  70static void r1bio_pool_free(void *r1_bio, void *data)
  71{
  72        kfree(r1_bio);
  73}
  74
  75#define RESYNC_BLOCK_SIZE (64*1024)
  76//#define RESYNC_BLOCK_SIZE PAGE_SIZE
  77#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
  78#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
  79#define RESYNC_WINDOW (2048*1024)
  80
  81static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data)
  82{
  83        struct pool_info *pi = data;
  84        struct page *page;
  85        r1bio_t *r1_bio;
  86        struct bio *bio;
  87        int i, j;
  88
  89        r1_bio = r1bio_pool_alloc(gfp_flags, pi);
  90        if (!r1_bio) {
  91                unplug_slaves(pi->mddev);
  92                return NULL;
  93        }
  94
  95        /*
  96         * Allocate bios : 1 for reading, n-1 for writing
  97         */
  98        for (j = pi->raid_disks ; j-- ; ) {
  99                bio = bio_alloc(gfp_flags, RESYNC_PAGES);
 100                if (!bio)
 101                        goto out_free_bio;
 102                r1_bio->bios[j] = bio;
 103        }
 104        /*
 105         * Allocate RESYNC_PAGES data pages and attach them to
 106         * the first bio.
 107         * If this is a user-requested check/repair, allocate
 108         * RESYNC_PAGES for each bio.
 109         */
 110        if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery))
 111                j = pi->raid_disks;
 112        else
 113                j = 1;
 114        while(j--) {
 115                bio = r1_bio->bios[j];
 116                for (i = 0; i < RESYNC_PAGES; i++) {
 117                        page = alloc_page(gfp_flags);
 118                        if (unlikely(!page))
 119                                goto out_free_pages;
 120
 121                        bio->bi_io_vec[i].bv_page = page;
 122                }
 123        }
 124        /* If not user-requests, copy the page pointers to all bios */
 125        if (!test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) {
 126                for (i=0; i<RESYNC_PAGES ; i++)
 127                        for (j=1; j<pi->raid_disks; j++)
 128                                r1_bio->bios[j]->bi_io_vec[i].bv_page =
 129                                        r1_bio->bios[0]->bi_io_vec[i].bv_page;
 130        }
 131
 132        r1_bio->master_bio = NULL;
 133
 134        return r1_bio;
 135
 136out_free_pages:
 137        for (i=0; i < RESYNC_PAGES ; i++)
 138                for (j=0 ; j < pi->raid_disks; j++)
 139                        safe_put_page(r1_bio->bios[j]->bi_io_vec[i].bv_page);
 140        j = -1;
 141out_free_bio:
 142        while ( ++j < pi->raid_disks )
 143                bio_put(r1_bio->bios[j]);
 144        r1bio_pool_free(r1_bio, data);
 145        return NULL;
 146}
 147
 148static void r1buf_pool_free(void *__r1_bio, void *data)
 149{
 150        struct pool_info *pi = data;
 151        int i,j;
 152        r1bio_t *r1bio = __r1_bio;
 153
 154        for (i = 0; i < RESYNC_PAGES; i++)
 155                for (j = pi->raid_disks; j-- ;) {
 156                        if (j == 0 ||
 157                            r1bio->bios[j]->bi_io_vec[i].bv_page !=
 158                            r1bio->bios[0]->bi_io_vec[i].bv_page)
 159                                safe_put_page(r1bio->bios[j]->bi_io_vec[i].bv_page);
 160                }
 161        for (i=0 ; i < pi->raid_disks; i++)
 162                bio_put(r1bio->bios[i]);
 163
 164        r1bio_pool_free(r1bio, data);
 165}
 166
 167static void put_all_bios(conf_t *conf, r1bio_t *r1_bio)
 168{
 169        int i;
 170
 171        for (i = 0; i < conf->raid_disks; i++) {
 172                struct bio **bio = r1_bio->bios + i;
 173                if (*bio && *bio != IO_BLOCKED)
 174                        bio_put(*bio);
 175                *bio = NULL;
 176        }
 177}
 178
 179static void free_r1bio(r1bio_t *r1_bio)
 180{
 181        conf_t *conf = mddev_to_conf(r1_bio->mddev);
 182
 183        /*
 184         * Wake up any possible resync thread that waits for the device
 185         * to go idle.
 186         */
 187        allow_barrier(conf);
 188
 189        put_all_bios(conf, r1_bio);
 190        mempool_free(r1_bio, conf->r1bio_pool);
 191}
 192
 193static void put_buf(r1bio_t *r1_bio)
 194{
 195        conf_t *conf = mddev_to_conf(r1_bio->mddev);
 196        int i;
 197
 198        for (i=0; i<conf->raid_disks; i++) {
 199                struct bio *bio = r1_bio->bios[i];
 200                if (bio->bi_end_io)
 201                        rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev);
 202        }
 203
 204        mempool_free(r1_bio, conf->r1buf_pool);
 205
 206        lower_barrier(conf);
 207}
 208
 209static void reschedule_retry(r1bio_t *r1_bio)
 210{
 211        unsigned long flags;
 212        mddev_t *mddev = r1_bio->mddev;
 213        conf_t *conf = mddev_to_conf(mddev);
 214
 215        spin_lock_irqsave(&conf->device_lock, flags);
 216        list_add(&r1_bio->retry_list, &conf->retry_list);
 217        conf->nr_queued ++;
 218        spin_unlock_irqrestore(&conf->device_lock, flags);
 219
 220        wake_up(&conf->wait_barrier);
 221        md_wakeup_thread(mddev->thread);
 222}
 223
 224/*
 225 * raid_end_bio_io() is called when we have finished servicing a mirrored
 226 * operation and are ready to return a success/failure code to the buffer
 227 * cache layer.
 228 */
 229static void raid_end_bio_io(r1bio_t *r1_bio)
 230{
 231        struct bio *bio = r1_bio->master_bio;
 232
 233        /* if nobody has done the final endio yet, do it now */
 234        if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
 235                PRINTK(KERN_DEBUG "raid1: sync end %s on sectors %llu-%llu\n",
 236                        (bio_data_dir(bio) == WRITE) ? "write" : "read",
 237                        (unsigned long long) bio->bi_sector,
 238                        (unsigned long long) bio->bi_sector +
 239                                (bio->bi_size >> 9) - 1);
 240
 241                bio_endio(bio,
 242                        test_bit(R1BIO_Uptodate, &r1_bio->state) ? 0 : -EIO);
 243        }
 244        free_r1bio(r1_bio);
 245}
 246
 247/*
 248 * Update disk head position estimator based on IRQ completion info.
 249 */
 250static inline void update_head_pos(int disk, r1bio_t *r1_bio)
 251{
 252        conf_t *conf = mddev_to_conf(r1_bio->mddev);
 253
 254        conf->mirrors[disk].head_position =
 255                r1_bio->sector + (r1_bio->sectors);
 256}
 257
 258static void raid1_end_read_request(struct bio *bio, int error)
 259{
 260        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 261        r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
 262        int mirror;
 263        conf_t *conf = mddev_to_conf(r1_bio->mddev);
 264
 265        mirror = r1_bio->read_disk;
 266        /*
 267         * this branch is our 'one mirror IO has finished' event handler:
 268         */
 269        update_head_pos(mirror, r1_bio);
 270
 271        if (uptodate)
 272                set_bit(R1BIO_Uptodate, &r1_bio->state);
 273        else {
 274                /* If all other devices have failed, we want to return
 275                 * the error upwards rather than fail the last device.
 276                 * Here we redefine "uptodate" to mean "Don't want to retry"
 277                 */
 278                unsigned long flags;
 279                spin_lock_irqsave(&conf->device_lock, flags);
 280                if (r1_bio->mddev->degraded == conf->raid_disks ||
 281                    (r1_bio->mddev->degraded == conf->raid_disks-1 &&
 282                     !test_bit(Faulty, &conf->mirrors[mirror].rdev->flags)))
 283                        uptodate = 1;
 284                spin_unlock_irqrestore(&conf->device_lock, flags);
 285        }
 286
 287        if (uptodate)
 288                raid_end_bio_io(r1_bio);
 289        else {
 290                /*
 291                 * oops, read error:
 292                 */
 293                char b[BDEVNAME_SIZE];
 294                if (printk_ratelimit())
 295                        printk(KERN_ERR "raid1: %s: rescheduling sector %llu\n",
 296                               bdevname(conf->mirrors[mirror].rdev->bdev,b), (unsigned long long)r1_bio->sector);
 297                reschedule_retry(r1_bio);
 298        }
 299
 300        rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
 301}
 302
 303static void raid1_end_write_request(struct bio *bio, int error)
 304{
 305        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
 306        r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
 307        int mirror, behind = test_bit(R1BIO_BehindIO, &r1_bio->state);
 308        conf_t *conf = mddev_to_conf(r1_bio->mddev);
 309        struct bio *to_put = NULL;
 310
 311
 312        for (mirror = 0; mirror < conf->raid_disks; mirror++)
 313                if (r1_bio->bios[mirror] == bio)
 314                        break;
 315
 316        if (error == -EOPNOTSUPP && test_bit(R1BIO_Barrier, &r1_bio->state)) {
 317                set_bit(BarriersNotsupp, &conf->mirrors[mirror].rdev->flags);
 318                set_bit(R1BIO_BarrierRetry, &r1_bio->state);
 319                r1_bio->mddev->barriers_work = 0;
 320                /* Don't rdev_dec_pending in this branch - keep it for the retry */
 321        } else {
 322                /*
 323                 * this branch is our 'one mirror IO has finished' event handler:
 324                 */
 325                r1_bio->bios[mirror] = NULL;
 326                to_put = bio;
 327                if (!uptodate) {
 328                        md_error(r1_bio->mddev, conf->mirrors[mirror].rdev);
 329                        /* an I/O failed, we can't clear the bitmap */
 330                        set_bit(R1BIO_Degraded, &r1_bio->state);
 331                } else
 332                        /*
 333                         * Set R1BIO_Uptodate in our master bio, so that
 334                         * we will return a good error code for to the higher
 335                         * levels even if IO on some other mirrored buffer fails.
 336                         *
 337                         * The 'master' represents the composite IO operation to
 338                         * user-side. So if something waits for IO, then it will
 339                         * wait for the 'master' bio.
 340                         */
 341                        set_bit(R1BIO_Uptodate, &r1_bio->state);
 342
 343                update_head_pos(mirror, r1_bio);
 344
 345                if (behind) {
 346                        if (test_bit(WriteMostly, &conf->mirrors[mirror].rdev->flags))
 347                                atomic_dec(&r1_bio->behind_remaining);
 348
 349                        /* In behind mode, we ACK the master bio once the I/O has safely
 350                         * reached all non-writemostly disks. Setting the Returned bit
 351                         * ensures that this gets done only once -- we don't ever want to
 352                         * return -EIO here, instead we'll wait */
 353
 354                        if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) &&
 355                            test_bit(R1BIO_Uptodate, &r1_bio->state)) {
 356                                /* Maybe we can return now */
 357                                if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
 358                                        struct bio *mbio = r1_bio->master_bio;
 359                                        PRINTK(KERN_DEBUG "raid1: behind end write sectors %llu-%llu\n",
 360                                               (unsigned long long) mbio->bi_sector,
 361                                               (unsigned long long) mbio->bi_sector +
 362                                               (mbio->bi_size >> 9) - 1);
 363                                        bio_endio(mbio, 0);
 364                                }
 365                        }
 366                }
 367                rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
 368        }
 369        /*
 370         *
 371         * Let's see if all mirrored write operations have finished
 372         * already.
 373         */
 374        if (atomic_dec_and_test(&r1_bio->remaining)) {
 375                if (test_bit(R1BIO_BarrierRetry, &r1_bio->state))
 376                        reschedule_retry(r1_bio);
 377                else {
 378                        /* it really is the end of this request */
 379                        if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
 380                                /* free extra copy of the data pages */
 381                                int i = bio->bi_vcnt;
 382                                while (i--)
 383                                        safe_put_page(bio->bi_io_vec[i].bv_page);
 384                        }
 385                        /* clear the bitmap if all writes complete successfully */
 386                        bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector,
 387                                        r1_bio->sectors,
 388                                        !test_bit(R1BIO_Degraded, &r1_bio->state),
 389                                        behind);
 390                        md_write_end(r1_bio->mddev);
 391                        raid_end_bio_io(r1_bio);
 392                }
 393        }
 394
 395        if (to_put)
 396                bio_put(to_put);
 397}
 398
 399
 400/*
 401 * This routine returns the disk from which the requested read should
 402 * be done. There is a per-array 'next expected sequential IO' sector
 403 * number - if this matches on the next IO then we use the last disk.
 404 * There is also a per-disk 'last know head position' sector that is
 405 * maintained from IRQ contexts, both the normal and the resync IO
 406 * completion handlers update this position correctly. If there is no
 407 * perfect sequential match then we pick the disk whose head is closest.
 408 *
 409 * If there are 2 mirrors in the same 2 devices, performance degrades
 410 * because position is mirror, not device based.
 411 *
 412 * The rdev for the device selected will have nr_pending incremented.
 413 */
 414static int read_balance(conf_t *conf, r1bio_t *r1_bio)
 415{
 416        const unsigned long this_sector = r1_bio->sector;
 417        int new_disk = conf->last_used, disk = new_disk;
 418        int wonly_disk = -1;
 419        const int sectors = r1_bio->sectors;
 420        sector_t new_distance, current_distance;
 421        mdk_rdev_t *rdev;
 422
 423        rcu_read_lock();
 424        /*
 425         * Check if we can balance. We can balance on the whole
 426         * device if no resync is going on, or below the resync window.
 427         * We take the first readable disk when above the resync window.
 428         */
 429 retry:
 430        if (conf->mddev->recovery_cp < MaxSector &&
 431            (this_sector + sectors >= conf->next_resync)) {
 432                /* Choose the first operation device, for consistancy */
 433                new_disk = 0;
 434
 435                for (rdev = rcu_dereference(conf->mirrors[new_disk].rdev);
 436                     r1_bio->bios[new_disk] == IO_BLOCKED ||
 437                     !rdev || !test_bit(In_sync, &rdev->flags)
 438                             || test_bit(WriteMostly, &rdev->flags);
 439                     rdev = rcu_dereference(conf->mirrors[++new_disk].rdev)) {
 440
 441                        if (rdev && test_bit(In_sync, &rdev->flags) &&
 442                                r1_bio->bios[new_disk] != IO_BLOCKED)
 443                                wonly_disk = new_disk;
 444
 445                        if (new_disk == conf->raid_disks - 1) {
 446                                new_disk = wonly_disk;
 447                                break;
 448                        }
 449                }
 450                goto rb_out;
 451        }
 452
 453
 454        /* make sure the disk is operational */
 455        for (rdev = rcu_dereference(conf->mirrors[new_disk].rdev);
 456             r1_bio->bios[new_disk] == IO_BLOCKED ||
 457             !rdev || !test_bit(In_sync, &rdev->flags) ||
 458                     test_bit(WriteMostly, &rdev->flags);
 459             rdev = rcu_dereference(conf->mirrors[new_disk].rdev)) {
 460
 461                if (rdev && test_bit(In_sync, &rdev->flags) &&
 462                    r1_bio->bios[new_disk] != IO_BLOCKED)
 463                        wonly_disk = new_disk;
 464
 465                if (new_disk <= 0)
 466                        new_disk = conf->raid_disks;
 467                new_disk--;
 468                if (new_disk == disk) {
 469                        new_disk = wonly_disk;
 470                        break;
 471                }
 472        }
 473
 474        if (new_disk < 0)
 475                goto rb_out;
 476
 477        disk = new_disk;
 478        /* now disk == new_disk == starting point for search */
 479
 480        /*
 481         * Don't change to another disk for sequential reads:
 482         */
 483        if (conf->next_seq_sect == this_sector)
 484                goto rb_out;
 485        if (this_sector == conf->mirrors[new_disk].head_position)
 486                goto rb_out;
 487
 488        current_distance = abs(this_sector - conf->mirrors[disk].head_position);
 489
 490        /* Find the disk whose head is closest */
 491
 492        do {
 493                if (disk <= 0)
 494                        disk = conf->raid_disks;
 495                disk--;
 496
 497                rdev = rcu_dereference(conf->mirrors[disk].rdev);
 498
 499                if (!rdev || r1_bio->bios[disk] == IO_BLOCKED ||
 500                    !test_bit(In_sync, &rdev->flags) ||
 501                    test_bit(WriteMostly, &rdev->flags))
 502                        continue;
 503
 504                if (!atomic_read(&rdev->nr_pending)) {
 505                        new_disk = disk;
 506                        break;
 507                }
 508                new_distance = abs(this_sector - conf->mirrors[disk].head_position);
 509                if (new_distance < current_distance) {
 510                        current_distance = new_distance;
 511                        new_disk = disk;
 512                }
 513        } while (disk != conf->last_used);
 514
 515 rb_out:
 516
 517
 518        if (new_disk >= 0) {
 519                rdev = rcu_dereference(conf->mirrors[new_disk].rdev);
 520                if (!rdev)
 521                        goto retry;
 522                atomic_inc(&rdev->nr_pending);
 523                if (!test_bit(In_sync, &rdev->flags)) {
 524                        /* cannot risk returning a device that failed
 525                         * before we inc'ed nr_pending
 526                         */
 527                        rdev_dec_pending(rdev, conf->mddev);
 528                        goto retry;
 529                }
 530                conf->next_seq_sect = this_sector + sectors;
 531                conf->last_used = new_disk;
 532        }
 533        rcu_read_unlock();
 534
 535        return new_disk;
 536}
 537
 538static void unplug_slaves(mddev_t *mddev)
 539{
 540        conf_t *conf = mddev_to_conf(mddev);
 541        int i;
 542
 543        rcu_read_lock();
 544        for (i=0; i<mddev->raid_disks; i++) {
 545                mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
 546                if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) {
 547                        struct request_queue *r_queue = bdev_get_queue(rdev->bdev);
 548
 549                        atomic_inc(&rdev->nr_pending);
 550                        rcu_read_unlock();
 551
 552                        blk_unplug(r_queue);
 553
 554                        rdev_dec_pending(rdev, mddev);
 555                        rcu_read_lock();
 556                }
 557        }
 558        rcu_read_unlock();
 559}
 560
 561static void raid1_unplug(struct request_queue *q)
 562{
 563        mddev_t *mddev = q->queuedata;
 564
 565        unplug_slaves(mddev);
 566        md_wakeup_thread(mddev->thread);
 567}
 568
 569static int raid1_congested(void *data, int bits)
 570{
 571        mddev_t *mddev = data;
 572        conf_t *conf = mddev_to_conf(mddev);
 573        int i, ret = 0;
 574
 575        rcu_read_lock();
 576        for (i = 0; i < mddev->raid_disks; i++) {
 577                mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
 578                if (rdev && !test_bit(Faulty, &rdev->flags)) {
 579                        struct request_queue *q = bdev_get_queue(rdev->bdev);
 580
 581                        /* Note the '|| 1' - when read_balance prefers
 582                         * non-congested targets, it can be removed
 583                         */
 584                        if ((bits & (1<<BDI_write_congested)) || 1)
 585                                ret |= bdi_congested(&q->backing_dev_info, bits);
 586                        else
 587                                ret &= bdi_congested(&q->backing_dev_info, bits);
 588                }
 589        }
 590        rcu_read_unlock();
 591        return ret;
 592}
 593
 594
 595static int flush_pending_writes(conf_t *conf)
 596{
 597        /* Any writes that have been queued but are awaiting
 598         * bitmap updates get flushed here.
 599         * We return 1 if any requests were actually submitted.
 600         */
 601        int rv = 0;
 602
 603        spin_lock_irq(&conf->device_lock);
 604
 605        if (conf->pending_bio_list.head) {
 606                struct bio *bio;
 607                bio = bio_list_get(&conf->pending_bio_list);
 608                blk_remove_plug(conf->mddev->queue);
 609                spin_unlock_irq(&conf->device_lock);
 610                /* flush any pending bitmap writes to
 611                 * disk before proceeding w/ I/O */
 612                bitmap_unplug(conf->mddev->bitmap);
 613
 614                while (bio) { /* submit pending writes */
 615                        struct bio *next = bio->bi_next;
 616                        bio->bi_next = NULL;
 617                        generic_make_request(bio);
 618                        bio = next;
 619                }
 620                rv = 1;
 621        } else
 622                spin_unlock_irq(&conf->device_lock);
 623        return rv;
 624}
 625
 626/* Barriers....
 627 * Sometimes we need to suspend IO while we do something else,
 628 * either some resync/recovery, or reconfigure the array.
 629 * To do this we raise a 'barrier'.
 630 * The 'barrier' is a counter that can be raised multiple times
 631 * to count how many activities are happening which preclude
 632 * normal IO.
 633 * We can only raise the barrier if there is no pending IO.
 634 * i.e. if nr_pending == 0.
 635 * We choose only to raise the barrier if no-one is waiting for the
 636 * barrier to go down.  This means that as soon as an IO request
 637 * is ready, no other operations which require a barrier will start
 638 * until the IO request has had a chance.
 639 *
 640 * So: regular IO calls 'wait_barrier'.  When that returns there
 641 *    is no backgroup IO happening,  It must arrange to call
 642 *    allow_barrier when it has finished its IO.
 643 * backgroup IO calls must call raise_barrier.  Once that returns
 644 *    there is no normal IO happeing.  It must arrange to call
 645 *    lower_barrier when the particular background IO completes.
 646 */
 647#define RESYNC_DEPTH 32
 648
 649static void raise_barrier(conf_t *conf)
 650{
 651        spin_lock_irq(&conf->resync_lock);
 652
 653        /* Wait until no block IO is waiting */
 654        wait_event_lock_irq(conf->wait_barrier, !conf->nr_waiting,
 655                            conf->resync_lock,
 656                            raid1_unplug(conf->mddev->queue));
 657
 658        /* block any new IO from starting */
 659        conf->barrier++;
 660
 661        /* No wait for all pending IO to complete */
 662        wait_event_lock_irq(conf->wait_barrier,
 663                            !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
 664                            conf->resync_lock,
 665                            raid1_unplug(conf->mddev->queue));
 666
 667        spin_unlock_irq(&conf->resync_lock);
 668}
 669
 670static void lower_barrier(conf_t *conf)
 671{
 672        unsigned long flags;
 673        spin_lock_irqsave(&conf->resync_lock, flags);
 674        conf->barrier--;
 675        spin_unlock_irqrestore(&conf->resync_lock, flags);
 676        wake_up(&conf->wait_barrier);
 677}
 678
 679static void wait_barrier(conf_t *conf)
 680{
 681        spin_lock_irq(&conf->resync_lock);
 682        if (conf->barrier) {
 683                conf->nr_waiting++;
 684                wait_event_lock_irq(conf->wait_barrier, !conf->barrier,
 685                                    conf->resync_lock,
 686                                    raid1_unplug(conf->mddev->queue));
 687                conf->nr_waiting--;
 688        }
 689        conf->nr_pending++;
 690        spin_unlock_irq(&conf->resync_lock);
 691}
 692
 693static void allow_barrier(conf_t *conf)
 694{
 695        unsigned long flags;
 696        spin_lock_irqsave(&conf->resync_lock, flags);
 697        conf->nr_pending--;
 698        spin_unlock_irqrestore(&conf->resync_lock, flags);
 699        wake_up(&conf->wait_barrier);
 700}
 701
 702static void freeze_array(conf_t *conf)
 703{
 704        /* stop syncio and normal IO and wait for everything to
 705         * go quite.
 706         * We increment barrier and nr_waiting, and then
 707         * wait until nr_pending match nr_queued+1
 708         * This is called in the context of one normal IO request
 709         * that has failed. Thus any sync request that might be pending
 710         * will be blocked by nr_pending, and we need to wait for
 711         * pending IO requests to complete or be queued for re-try.
 712         * Thus the number queued (nr_queued) plus this request (1)
 713         * must match the number of pending IOs (nr_pending) before
 714         * we continue.
 715         */
 716        spin_lock_irq(&conf->resync_lock);
 717        conf->barrier++;
 718        conf->nr_waiting++;
 719        wait_event_lock_irq(conf->wait_barrier,
 720                            conf->nr_pending == conf->nr_queued+1,
 721                            conf->resync_lock,
 722                            ({ flush_pending_writes(conf);
 723                               raid1_unplug(conf->mddev->queue); }));
 724        spin_unlock_irq(&conf->resync_lock);
 725}
 726static void unfreeze_array(conf_t *conf)
 727{
 728        /* reverse the effect of the freeze */
 729        spin_lock_irq(&conf->resync_lock);
 730        conf->barrier--;
 731        conf->nr_waiting--;
 732        wake_up(&conf->wait_barrier);
 733        spin_unlock_irq(&conf->resync_lock);
 734}
 735
 736
 737/* duplicate the data pages for behind I/O */
 738static struct page **alloc_behind_pages(struct bio *bio)
 739{
 740        int i;
 741        struct bio_vec *bvec;
 742        struct page **pages = kzalloc(bio->bi_vcnt * sizeof(struct page *),
 743                                        GFP_NOIO);
 744        if (unlikely(!pages))
 745                goto do_sync_io;
 746
 747        bio_for_each_segment(bvec, bio, i) {
 748                pages[i] = alloc_page(GFP_NOIO);
 749                if (unlikely(!pages[i]))
 750                        goto do_sync_io;
 751                memcpy(kmap(pages[i]) + bvec->bv_offset,
 752                        kmap(bvec->bv_page) + bvec->bv_offset, bvec->bv_len);
 753                kunmap(pages[i]);
 754                kunmap(bvec->bv_page);
 755        }
 756
 757        return pages;
 758
 759do_sync_io:
 760        if (pages)
 761                for (i = 0; i < bio->bi_vcnt && pages[i]; i++)
 762                        put_page(pages[i]);
 763        kfree(pages);
 764        PRINTK("%dB behind alloc failed, doing sync I/O\n", bio->bi_size);
 765        return NULL;
 766}
 767
 768static int make_request(struct request_queue *q, struct bio * bio)
 769{
 770        mddev_t *mddev = q->queuedata;
 771        conf_t *conf = mddev_to_conf(mddev);
 772        mirror_info_t *mirror;
 773        r1bio_t *r1_bio;
 774        struct bio *read_bio;
 775        int i, targets = 0, disks;
 776        struct bitmap *bitmap;
 777        unsigned long flags;
 778        struct bio_list bl;
 779        struct page **behind_pages = NULL;
 780        const int rw = bio_data_dir(bio);
 781        const int do_sync = bio_sync(bio);
 782        int do_barriers;
 783        mdk_rdev_t *blocked_rdev;
 784
 785        /*
 786         * Register the new request and wait if the reconstruction
 787         * thread has put up a bar for new requests.
 788         * Continue immediately if no resync is active currently.
 789         * We test barriers_work *after* md_write_start as md_write_start
 790         * may cause the first superblock write, and that will check out
 791         * if barriers work.
 792         */
 793
 794        md_write_start(mddev, bio); /* wait on superblock update early */
 795
 796        if (unlikely(!mddev->barriers_work && bio_barrier(bio))) {
 797                if (rw == WRITE)
 798                        md_write_end(mddev);
 799                bio_endio(bio, -EOPNOTSUPP);
 800                return 0;
 801        }
 802
 803        wait_barrier(conf);
 804
 805        bitmap = mddev->bitmap;
 806
 807        disk_stat_inc(mddev->gendisk, ios[rw]);
 808        disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bio));
 809
 810        /*
 811         * make_request() can abort the operation when READA is being
 812         * used and no empty request is available.
 813         *
 814         */
 815        r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
 816
 817        r1_bio->master_bio = bio;
 818        r1_bio->sectors = bio->bi_size >> 9;
 819        r1_bio->state = 0;
 820        r1_bio->mddev = mddev;
 821        r1_bio->sector = bio->bi_sector;
 822
 823        if (rw == READ) {
 824                /*
 825                 * read balancing logic:
 826                 */
 827                int rdisk = read_balance(conf, r1_bio);
 828
 829                if (rdisk < 0) {
 830                        /* couldn't find anywhere to read from */
 831                        raid_end_bio_io(r1_bio);
 832                        return 0;
 833                }
 834                mirror = conf->mirrors + rdisk;
 835
 836                r1_bio->read_disk = rdisk;
 837
 838                read_bio = bio_clone(bio, GFP_NOIO);
 839
 840                r1_bio->bios[rdisk] = read_bio;
 841
 842                read_bio->bi_sector = r1_bio->sector + mirror->rdev->data_offset;
 843                read_bio->bi_bdev = mirror->rdev->bdev;
 844                read_bio->bi_end_io = raid1_end_read_request;
 845                read_bio->bi_rw = READ | do_sync;
 846                read_bio->bi_private = r1_bio;
 847
 848                generic_make_request(read_bio);
 849                return 0;
 850        }
 851
 852        /*
 853         * WRITE:
 854         */
 855        /* first select target devices under spinlock and
 856         * inc refcount on their rdev.  Record them by setting
 857         * bios[x] to bio
 858         */
 859        disks = conf->raid_disks;
 860#if 0
 861        { static int first=1;
 862        if (first) printk("First Write sector %llu disks %d\n",
 863                          (unsigned long long)r1_bio->sector, disks);
 864        first = 0;
 865        }
 866#endif
 867 retry_write:
 868        blocked_rdev = NULL;
 869        rcu_read_lock();
 870        for (i = 0;  i < disks; i++) {
 871                mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
 872                if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
 873                        atomic_inc(&rdev->nr_pending);
 874                        blocked_rdev = rdev;
 875                        break;
 876                }
 877                if (rdev && !test_bit(Faulty, &rdev->flags)) {
 878                        atomic_inc(&rdev->nr_pending);
 879                        if (test_bit(Faulty, &rdev->flags)) {
 880                                rdev_dec_pending(rdev, mddev);
 881                                r1_bio->bios[i] = NULL;
 882                        } else
 883                                r1_bio->bios[i] = bio;
 884                        targets++;
 885                } else
 886                        r1_bio->bios[i] = NULL;
 887        }
 888        rcu_read_unlock();
 889
 890        if (unlikely(blocked_rdev)) {
 891                /* Wait for this device to become unblocked */
 892                int j;
 893
 894                for (j = 0; j < i; j++)
 895                        if (r1_bio->bios[j])
 896                                rdev_dec_pending(conf->mirrors[j].rdev, mddev);
 897
 898                allow_barrier(conf);
 899                md_wait_for_blocked_rdev(blocked_rdev, mddev);
 900                wait_barrier(conf);
 901                goto retry_write;
 902        }
 903
 904        BUG_ON(targets == 0); /* we never fail the last device */
 905
 906        if (targets < conf->raid_disks) {
 907                /* array is degraded, we will not clear the bitmap
 908                 * on I/O completion (see raid1_end_write_request) */
 909                set_bit(R1BIO_Degraded, &r1_bio->state);
 910        }
 911
 912        /* do behind I/O ? */
 913        if (bitmap &&
 914            atomic_read(&bitmap->behind_writes) < bitmap->max_write_behind &&
 915            (behind_pages = alloc_behind_pages(bio)) != NULL)
 916                set_bit(R1BIO_BehindIO, &r1_bio->state);
 917
 918        atomic_set(&r1_bio->remaining, 0);
 919        atomic_set(&r1_bio->behind_remaining, 0);
 920
 921        do_barriers = bio_barrier(bio);
 922        if (do_barriers)
 923                set_bit(R1BIO_Barrier, &r1_bio->state);
 924
 925        bio_list_init(&bl);
 926        for (i = 0; i < disks; i++) {
 927                struct bio *mbio;
 928                if (!r1_bio->bios[i])
 929                        continue;
 930
 931                mbio = bio_clone(bio, GFP_NOIO);
 932                r1_bio->bios[i] = mbio;
 933
 934                mbio->bi_sector = r1_bio->sector + conf->mirrors[i].rdev->data_offset;
 935                mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
 936                mbio->bi_end_io = raid1_end_write_request;
 937                mbio->bi_rw = WRITE | do_barriers | do_sync;
 938                mbio->bi_private = r1_bio;
 939
 940                if (behind_pages) {
 941                        struct bio_vec *bvec;
 942                        int j;
 943
 944                        /* Yes, I really want the '__' version so that
 945                         * we clear any unused pointer in the io_vec, rather
 946                         * than leave them unchanged.  This is important
 947                         * because when we come to free the pages, we won't
 948                         * know the originial bi_idx, so we just free
 949                         * them all
 950                         */
 951                        __bio_for_each_segment(bvec, mbio, j, 0)
 952                                bvec->bv_page = behind_pages[j];
 953                        if (test_bit(WriteMostly, &conf->mirrors[i].rdev->flags))
 954                                atomic_inc(&r1_bio->behind_remaining);
 955                }
 956
 957                atomic_inc(&r1_bio->remaining);
 958
 959                bio_list_add(&bl, mbio);
 960        }
 961        kfree(behind_pages); /* the behind pages are attached to the bios now */
 962
 963        bitmap_startwrite(bitmap, bio->bi_sector, r1_bio->sectors,
 964                                test_bit(R1BIO_BehindIO, &r1_bio->state));
 965        spin_lock_irqsave(&conf->device_lock, flags);
 966        bio_list_merge(&conf->pending_bio_list, &bl);
 967        bio_list_init(&bl);
 968
 969        blk_plug_device(mddev->queue);
 970        spin_unlock_irqrestore(&conf->device_lock, flags);
 971
 972        /* In case raid1d snuck into freeze_array */
 973        wake_up(&conf->wait_barrier);
 974
 975        if (do_sync)
 976                md_wakeup_thread(mddev->thread);
 977#if 0
 978        while ((bio = bio_list_pop(&bl)) != NULL)
 979                generic_make_request(bio);
 980#endif
 981
 982        return 0;
 983}
 984
 985static void status(struct seq_file *seq, mddev_t *mddev)
 986{
 987        conf_t *conf = mddev_to_conf(mddev);
 988        int i;
 989
 990        seq_printf(seq, " [%d/%d] [", conf->raid_disks,
 991                   conf->raid_disks - mddev->degraded);
 992        rcu_read_lock();
 993        for (i = 0; i < conf->raid_disks; i++) {
 994                mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
 995                seq_printf(seq, "%s",
 996                           rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
 997        }
 998        rcu_read_unlock();
 999        seq_printf(seq, "]");
1000}
1001
1002
1003static void error(mddev_t *mddev, mdk_rdev_t *rdev)
1004{
1005        char b[BDEVNAME_SIZE];
1006        conf_t *conf = mddev_to_conf(mddev);
1007
1008        /*
1009         * If it is not operational, then we have already marked it as dead
1010         * else if it is the last working disks, ignore the error, let the
1011         * next level up know.
1012         * else mark the drive as failed
1013         */
1014        if (test_bit(In_sync, &rdev->flags)
1015            && (conf->raid_disks - mddev->degraded) == 1)
1016                /*
1017                 * Don't fail the drive, act as though we were just a
1018                 * normal single drive
1019                 */
1020                return;
1021        if (test_and_clear_bit(In_sync, &rdev->flags)) {
1022                unsigned long flags;
1023                spin_lock_irqsave(&conf->device_lock, flags);
1024                mddev->degraded++;
1025                set_bit(Faulty, &rdev->flags);
1026                spin_unlock_irqrestore(&conf->device_lock, flags);
1027                /*
1028                 * if recovery is running, make sure it aborts.
1029                 */
1030                set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1031        } else
1032                set_bit(Faulty, &rdev->flags);
1033        set_bit(MD_CHANGE_DEVS, &mddev->flags);
1034        printk(KERN_ALERT "raid1: Disk failure on %s, disabling device.\n"
1035                "raid1: Operation continuing on %d devices.\n",
1036                bdevname(rdev->bdev,b), conf->raid_disks - mddev->degraded);
1037}
1038
1039static void print_conf(conf_t *conf)
1040{
1041        int i;
1042
1043        printk("RAID1 conf printout:\n");
1044        if (!conf) {
1045                printk("(!conf)\n");
1046                return;
1047        }
1048        printk(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
1049                conf->raid_disks);
1050
1051        rcu_read_lock();
1052        for (i = 0; i < conf->raid_disks; i++) {
1053                char b[BDEVNAME_SIZE];
1054                mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev);
1055                if (rdev)
1056                        printk(" disk %d, wo:%d, o:%d, dev:%s\n",
1057                               i, !test_bit(In_sync, &rdev->flags),
1058                               !test_bit(Faulty, &rdev->flags),
1059                               bdevname(rdev->bdev,b));
1060        }
1061        rcu_read_unlock();
1062}
1063
1064static void close_sync(conf_t *conf)
1065{
1066        wait_barrier(conf);
1067        allow_barrier(conf);
1068
1069        mempool_destroy(conf->r1buf_pool);
1070        conf->r1buf_pool = NULL;
1071}
1072
1073static int raid1_spare_active(mddev_t *mddev)
1074{
1075        int i;
1076        conf_t *conf = mddev->private;
1077
1078        /*
1079         * Find all failed disks within the RAID1 configuration 
1080         * and mark them readable.
1081         * Called under mddev lock, so rcu protection not needed.
1082         */
1083        for (i = 0; i < conf->raid_disks; i++) {
1084                mdk_rdev_t *rdev = conf->mirrors[i].rdev;
1085                if (rdev
1086                    && !test_bit(Faulty, &rdev->flags)
1087                    && !test_and_set_bit(In_sync, &rdev->flags)) {
1088                        unsigned long flags;
1089                        spin_lock_irqsave(&conf->device_lock, flags);
1090                        mddev->degraded--;
1091                        spin_unlock_irqrestore(&conf->device_lock, flags);
1092                }
1093        }
1094
1095        print_conf(conf);
1096        return 0;
1097}
1098
1099
1100static int raid1_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
1101{
1102        conf_t *conf = mddev->private;
1103        int err = -EEXIST;
1104        int mirror = 0;
1105        mirror_info_t *p;
1106        int first = 0;
1107        int last = mddev->raid_disks - 1;
1108
1109        if (rdev->raid_disk >= 0)
1110                first = last = rdev->raid_disk;
1111
1112        for (mirror = first; mirror <= last; mirror++)
1113                if ( !(p=conf->mirrors+mirror)->rdev) {
1114
1115                        blk_queue_stack_limits(mddev->queue,
1116                                               rdev->bdev->bd_disk->queue);
1117                        /* as we don't honour merge_bvec_fn, we must never risk
1118                         * violating it, so limit ->max_sector to one PAGE, as
1119                         * a one page request is never in violation.
1120                         */
1121                        if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
1122                            mddev->queue->max_sectors > (PAGE_SIZE>>9))
1123                                blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
1124
1125                        p->head_position = 0;
1126                        rdev->raid_disk = mirror;
1127                        err = 0;
1128                        /* As all devices are equivalent, we don't need a full recovery
1129                         * if this was recently any drive of the array
1130                         */
1131                        if (rdev->saved_raid_disk < 0)
1132                                conf->fullsync = 1;
1133                        rcu_assign_pointer(p->rdev, rdev);
1134                        break;
1135                }
1136
1137        print_conf(conf);
1138        return err;
1139}
1140
1141static int raid1_remove_disk(mddev_t *mddev, int number)
1142{
1143        conf_t *conf = mddev->private;
1144        int err = 0;
1145        mdk_rdev_t *rdev;
1146        mirror_info_t *p = conf->mirrors+ number;
1147
1148        print_conf(conf);
1149        rdev = p->rdev;
1150        if (rdev) {
1151                if (test_bit(In_sync, &rdev->flags) ||
1152                    atomic_read(&rdev->nr_pending)) {
1153                        err = -EBUSY;
1154                        goto abort;
1155                }
1156                /* Only remove non-faulty devices is recovery
1157                 * is not possible.
1158                 */
1159                if (!test_bit(Faulty, &rdev->flags) &&
1160                    mddev->degraded < conf->raid_disks) {
1161                        err = -EBUSY;
1162                        goto abort;
1163                }
1164                p->rdev = NULL;
1165                synchronize_rcu();
1166                if (atomic_read(&rdev->nr_pending)) {
1167                        /* lost the race, try later */
1168                        err = -EBUSY;
1169                        p->rdev = rdev;
1170                }
1171        }
1172abort:
1173
1174        print_conf(conf);
1175        return err;
1176}
1177
1178
1179static void end_sync_read(struct bio *bio, int error)
1180{
1181        r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
1182        int i;
1183
1184        for (i=r1_bio->mddev->raid_disks; i--; )
1185                if (r1_bio->bios[i] == bio)
1186                        break;
1187        BUG_ON(i < 0);
1188        update_head_pos(i, r1_bio);
1189        /*
1190         * we have read a block, now it needs to be re-written,
1191         * or re-read if the read failed.
1192         * We don't do much here, just schedule handling by raid1d
1193         */
1194        if (test_bit(BIO_UPTODATE, &bio->bi_flags))
1195                set_bit(R1BIO_Uptodate, &r1_bio->state);
1196
1197        if (atomic_dec_and_test(&r1_bio->remaining))
1198                reschedule_retry(r1_bio);
1199}
1200
1201static void end_sync_write(struct bio *bio, int error)
1202{
1203        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1204        r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
1205        mddev_t *mddev = r1_bio->mddev;
1206        conf_t *conf = mddev_to_conf(mddev);
1207        int i;
1208        int mirror=0;
1209
1210        for (i = 0; i < conf->raid_disks; i++)
1211                if (r1_bio->bios[i] == bio) {
1212                        mirror = i;
1213                        break;
1214                }
1215        if (!uptodate) {
1216                int sync_blocks = 0;
1217                sector_t s = r1_bio->sector;
1218                long sectors_to_go = r1_bio->sectors;
1219                /* make sure these bits doesn't get cleared. */
1220                do {
1221                        bitmap_end_sync(mddev->bitmap, s,
1222                                        &sync_blocks, 1);
1223                        s += sync_blocks;
1224                        sectors_to_go -= sync_blocks;
1225                } while (sectors_to_go > 0);
1226                md_error(mddev, conf->mirrors[mirror].rdev);
1227        }
1228
1229        update_head_pos(mirror, r1_bio);
1230
1231        if (atomic_dec_and_test(&r1_bio->remaining)) {
1232                md_done_sync(mddev, r1_bio->sectors, uptodate);
1233                put_buf(r1_bio);
1234        }
1235}
1236
1237static void sync_request_write(mddev_t *mddev, r1bio_t *r1_bio)
1238{
1239        conf_t *conf = mddev_to_conf(mddev);
1240        int i;
1241        int disks = conf->raid_disks;
1242        struct bio *bio, *wbio;
1243
1244        bio = r1_bio->bios[r1_bio->read_disk];
1245
1246
1247        if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
1248                /* We have read all readable devices.  If we haven't
1249                 * got the block, then there is no hope left.
1250                 * If we have, then we want to do a comparison
1251                 * and skip the write if everything is the same.
1252                 * If any blocks failed to read, then we need to
1253                 * attempt an over-write
1254                 */
1255                int primary;
1256                if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) {
1257                        for (i=0; i<mddev->raid_disks; i++)
1258                                if (r1_bio->bios[i]->bi_end_io == end_sync_read)
1259                                        md_error(mddev, conf->mirrors[i].rdev);
1260
1261                        md_done_sync(mddev, r1_bio->sectors, 1);
1262                        put_buf(r1_bio);
1263                        return;
1264                }
1265                for (primary=0; primary<mddev->raid_disks; primary++)
1266                        if (r1_bio->bios[primary]->bi_end_io == end_sync_read &&
1267                            test_bit(BIO_UPTODATE, &r1_bio->bios[primary]->bi_flags)) {
1268                                r1_bio->bios[primary]->bi_end_io = NULL;
1269                                rdev_dec_pending(conf->mirrors[primary].rdev, mddev);
1270                                break;
1271                        }
1272                r1_bio->read_disk = primary;
1273                for (i=0; i<mddev->raid_disks; i++)
1274                        if (r1_bio->bios[i]->bi_end_io == end_sync_read) {
1275                                int j;
1276                                int vcnt = r1_bio->sectors >> (PAGE_SHIFT- 9);
1277                                struct bio *pbio = r1_bio->bios[primary];
1278                                struct bio *sbio = r1_bio->bios[i];
1279
1280                                if (test_bit(BIO_UPTODATE, &sbio->bi_flags)) {
1281                                        for (j = vcnt; j-- ; ) {
1282                                                struct page *p, *s;
1283                                                p = pbio->bi_io_vec[j].bv_page;
1284                                                s = sbio->bi_io_vec[j].bv_page;
1285                                                if (memcmp(page_address(p),
1286                                                           page_address(s),
1287                                                           PAGE_SIZE))
1288                                                        break;
1289                                        }
1290                                } else
1291                                        j = 0;
1292                                if (j >= 0)
1293                                        mddev->resync_mismatches += r1_bio->sectors;
1294                                if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)
1295                                              && test_bit(BIO_UPTODATE, &sbio->bi_flags))) {
1296                                        sbio->bi_end_io = NULL;
1297                                        rdev_dec_pending(conf->mirrors[i].rdev, mddev);
1298                                } else {
1299                                        /* fixup the bio for reuse */
1300                                        int size;
1301                                        sbio->bi_vcnt = vcnt;
1302                                        sbio->bi_size = r1_bio->sectors << 9;
1303                                        sbio->bi_idx = 0;
1304                                        sbio->bi_phys_segments = 0;
1305                                        sbio->bi_hw_segments = 0;
1306                                        sbio->bi_hw_front_size = 0;
1307                                        sbio->bi_hw_back_size = 0;
1308                                        sbio->bi_flags &= ~(BIO_POOL_MASK - 1);
1309                                        sbio->bi_flags |= 1 << BIO_UPTODATE;
1310                                        sbio->bi_next = NULL;
1311                                        sbio->bi_sector = r1_bio->sector +
1312                                                conf->mirrors[i].rdev->data_offset;
1313                                        sbio->bi_bdev = conf->mirrors[i].rdev->bdev;
1314                                        size = sbio->bi_size;
1315                                        for (j = 0; j < vcnt ; j++) {
1316                                                struct bio_vec *bi;
1317                                                bi = &sbio->bi_io_vec[j];
1318                                                bi->bv_offset = 0;
1319                                                if (size > PAGE_SIZE)
1320                                                        bi->bv_len = PAGE_SIZE;
1321                                                else
1322                                                        bi->bv_len = size;
1323                                                size -= PAGE_SIZE;
1324                                                memcpy(page_address(bi->bv_page),
1325                                                       page_address(pbio->bi_io_vec[j].bv_page),
1326                                                       PAGE_SIZE);
1327                                        }
1328
1329                                }
1330                        }
1331        }
1332        if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) {
1333                /* ouch - failed to read all of that.
1334                 * Try some synchronous reads of other devices to get
1335                 * good data, much like with normal read errors.  Only
1336                 * read into the pages we already have so we don't
1337                 * need to re-issue the read request.
1338                 * We don't need to freeze the array, because being in an
1339                 * active sync request, there is no normal IO, and
1340                 * no overlapping syncs.
1341                 */
1342                sector_t sect = r1_bio->sector;
1343                int sectors = r1_bio->sectors;
1344                int idx = 0;
1345
1346                while(sectors) {
1347                        int s = sectors;
1348                        int d = r1_bio->read_disk;
1349                        int success = 0;
1350                        mdk_rdev_t *rdev;
1351
1352                        if (s > (PAGE_SIZE>>9))
1353                                s = PAGE_SIZE >> 9;
1354                        do {
1355                                if (r1_bio->bios[d]->bi_end_io == end_sync_read) {
1356                                        /* No rcu protection needed here devices
1357                                         * can only be removed when no resync is
1358                                         * active, and resync is currently active
1359                                         */
1360                                        rdev = conf->mirrors[d].rdev;
1361                                        if (sync_page_io(rdev->bdev,
1362                                                         sect + rdev->data_offset,
1363                                                         s<<9,
1364                                                         bio->bi_io_vec[idx].bv_page,
1365                                                         READ)) {
1366                                                success = 1;
1367                                                break;
1368                                        }
1369                                }
1370                                d++;
1371                                if (d == conf->raid_disks)
1372                                        d = 0;
1373                        } while (!success && d != r1_bio->read_disk);
1374
1375                        if (success) {
1376                                int start = d;
1377                                /* write it back and re-read */
1378                                set_bit(R1BIO_Uptodate, &r1_bio->state);
1379                                while (d != r1_bio->read_disk) {
1380                                        if (d == 0)
1381                                                d = conf->raid_disks;
1382                                        d--;
1383                                        if (r1_bio->bios[d]->bi_end_io != end_sync_read)
1384                                                continue;
1385                                        rdev = conf->mirrors[d].rdev;
1386                                        atomic_add(s, &rdev->corrected_errors);
1387                                        if (sync_page_io(rdev->bdev,
1388                                                         sect + rdev->data_offset,
1389                                                         s<<9,
1390                                                         bio->bi_io_vec[idx].bv_page,
1391                                                         WRITE) == 0)
1392                                                md_error(mddev, rdev);
1393                                }
1394                                d = start;
1395                                while (d != r1_bio->read_disk) {
1396                                        if (d == 0)
1397                                                d = conf->raid_disks;
1398                                        d--;
1399                                        if (r1_bio->bios[d]->bi_end_io != end_sync_read)
1400                                                continue;
1401                                        rdev = conf->mirrors[d].rdev;
1402                                        if (sync_page_io(rdev->bdev,
1403                                                         sect + rdev->data_offset,
1404                                                         s<<9,
1405                                                         bio->bi_io_vec[idx].bv_page,
1406                                                         READ) == 0)
1407                                                md_error(mddev, rdev);
1408                                }
1409                        } else {
1410                                char b[BDEVNAME_SIZE];
1411                                /* Cannot read from anywhere, array is toast */
1412                                md_error(mddev, conf->mirrors[r1_bio->read_disk].rdev);
1413                                printk(KERN_ALERT "raid1: %s: unrecoverable I/O read error"
1414                                       " for block %llu\n",
1415                                       bdevname(bio->bi_bdev,b),
1416                                       (unsigned long long)r1_bio->sector);
1417                                md_done_sync(mddev, r1_bio->sectors, 0);
1418                                put_buf(r1_bio);
1419                                return;
1420                        }
1421                        sectors -= s;
1422                        sect += s;
1423                        idx ++;
1424                }
1425        }
1426
1427        /*
1428         * schedule writes
1429         */
1430        atomic_set(&r1_bio->remaining, 1);
1431        for (i = 0; i < disks ; i++) {
1432                wbio = r1_bio->bios[i];
1433                if (wbio->bi_end_io == NULL ||
1434                    (wbio->bi_end_io == end_sync_read &&
1435                     (i == r1_bio->read_disk ||
1436                      !test_bit(MD_RECOVERY_SYNC, &mddev->recovery))))
1437                        continue;
1438
1439                wbio->bi_rw = WRITE;
1440                wbio->bi_end_io = end_sync_write;
1441                atomic_inc(&r1_bio->remaining);
1442                md_sync_acct(conf->mirrors[i].rdev->bdev, wbio->bi_size >> 9);
1443
1444                generic_make_request(wbio);
1445        }
1446
1447        if (atomic_dec_and_test(&r1_bio->remaining)) {
1448                /* if we're here, all write(s) have completed, so clean up */
1449                md_done_sync(mddev, r1_bio->sectors, 1);
1450                put_buf(r1_bio);
1451        }
1452}
1453
1454/*
1455 * This is a kernel thread which:
1456 *
1457 *      1.      Retries failed read operations on working mirrors.
1458 *      2.      Updates the raid superblock when problems encounter.
1459 *      3.      Performs writes following reads for array syncronising.
1460 */
1461
1462static void fix_read_error(conf_t *conf, int read_disk,
1463                           sector_t sect, int sectors)
1464{
1465        mddev_t *mddev = conf->mddev;
1466        while(sectors) {
1467                int s = sectors;
1468                int d = read_disk;
1469                int success = 0;
1470                int start;
1471                mdk_rdev_t *rdev;
1472
1473                if (s > (PAGE_SIZE>>9))
1474                        s = PAGE_SIZE >> 9;
1475
1476                do {
1477                        /* Note: no rcu protection needed here
1478                         * as this is synchronous in the raid1d thread
1479                         * which is the thread that might remove
1480                         * a device.  If raid1d ever becomes multi-threaded....
1481                         */
1482                        rdev = conf->mirrors[d].rdev;
1483                        if (rdev &&
1484                            test_bit(In_sync, &rdev->flags) &&
1485                            sync_page_io(rdev->bdev,
1486                                         sect + rdev->data_offset,
1487                                         s<<9,
1488                                         conf->tmppage, READ))
1489                                success = 1;
1490                        else {
1491                                d++;
1492                                if (d == conf->raid_disks)
1493                                        d = 0;
1494                        }
1495                } while (!success && d != read_disk);
1496
1497                if (!success) {
1498                        /* Cannot read from anywhere -- bye bye array */
1499                        md_error(mddev, conf->mirrors[read_disk].rdev);
1500                        break;
1501                }
1502                /* write it back and re-read */
1503                start = d;
1504                while (d != read_disk) {
1505                        if (d==0)
1506                                d = conf->raid_disks;
1507                        d--;
1508                        rdev = conf->mirrors[d].rdev;
1509                        if (rdev &&
1510                            test_bit(In_sync, &rdev->flags)) {
1511                                if (sync_page_io(rdev->bdev,
1512                                                 sect + rdev->data_offset,
1513                                                 s<<9, conf->tmppage, WRITE)
1514                                    == 0)
1515                                        /* Well, this device is dead */
1516                                        md_error(mddev, rdev);
1517                        }
1518                }
1519                d = start;
1520                while (d != read_disk) {
1521                        char b[BDEVNAME_SIZE];
1522                        if (d==0)
1523                                d = conf->raid_disks;
1524                        d--;
1525                        rdev = conf->mirrors[d].rdev;
1526                        if (rdev &&
1527                            test_bit(In_sync, &rdev->flags)) {
1528                                if (sync_page_io(rdev->bdev,
1529                                                 sect + rdev->data_offset,
1530                                                 s<<9, conf->tmppage, READ)
1531                                    == 0)
1532                                        /* Well, this device is dead */
1533                                        md_error(mddev, rdev);
1534                                else {
1535                                        atomic_add(s, &rdev->corrected_errors);
1536                                        printk(KERN_INFO
1537                                               "raid1:%s: read error corrected "
1538                                               "(%d sectors at %llu on %s)\n",
1539                                               mdname(mddev), s,
1540                                               (unsigned long long)(sect +
1541                                                   rdev->data_offset),
1542                                               bdevname(rdev->bdev, b));
1543                                }
1544                        }
1545                }
1546                sectors -= s;
1547                sect += s;
1548        }
1549}
1550
1551static void raid1d(mddev_t *mddev)
1552{
1553        r1bio_t *r1_bio;
1554        struct bio *bio;
1555        unsigned long flags;
1556        conf_t *conf = mddev_to_conf(mddev);
1557        struct list_head *head = &conf->retry_list;
1558        int unplug=0;
1559        mdk_rdev_t *rdev;
1560
1561        md_check_recovery(mddev);
1562        
1563        for (;;) {
1564                char b[BDEVNAME_SIZE];
1565
1566                unplug += flush_pending_writes(conf);
1567
1568                spin_lock_irqsave(&conf->device_lock, flags);
1569                if (list_empty(head)) {
1570                        spin_unlock_irqrestore(&conf->device_lock, flags);
1571                        break;
1572                }
1573                r1_bio = list_entry(head->prev, r1bio_t, retry_list);
1574                list_del(head->prev);
1575                conf->nr_queued--;
1576                spin_unlock_irqrestore(&conf->device_lock, flags);
1577
1578                mddev = r1_bio->mddev;
1579                conf = mddev_to_conf(mddev);
1580                if (test_bit(R1BIO_IsSync, &r1_bio->state)) {
1581                        sync_request_write(mddev, r1_bio);
1582                        unplug = 1;
1583                } else if (test_bit(R1BIO_BarrierRetry, &r1_bio->state)) {
1584                        /* some requests in the r1bio were BIO_RW_BARRIER
1585                         * requests which failed with -EOPNOTSUPP.  Hohumm..
1586                         * Better resubmit without the barrier.
1587                         * We know which devices to resubmit for, because
1588                         * all others have had their bios[] entry cleared.
1589                         * We already have a nr_pending reference on these rdevs.
1590                         */
1591                        int i;
1592                        const int do_sync = bio_sync(r1_bio->master_bio);
1593                        clear_bit(R1BIO_BarrierRetry, &r1_bio->state);
1594                        clear_bit(R1BIO_Barrier, &r1_bio->state);
1595                        for (i=0; i < conf->raid_disks; i++)
1596                                if (r1_bio->bios[i])
1597                                        atomic_inc(&r1_bio->remaining);
1598                        for (i=0; i < conf->raid_disks; i++)
1599                                if (r1_bio->bios[i]) {
1600                                        struct bio_vec *bvec;
1601                                        int j;
1602
1603                                        bio = bio_clone(r1_bio->master_bio, GFP_NOIO);
1604                                        /* copy pages from the failed bio, as
1605                                         * this might be a write-behind device */
1606                                        __bio_for_each_segment(bvec, bio, j, 0)
1607                                                bvec->bv_page = bio_iovec_idx(r1_bio->bios[i], j)->bv_page;
1608                                        bio_put(r1_bio->bios[i]);
1609                                        bio->bi_sector = r1_bio->sector +
1610                                                conf->mirrors[i].rdev->data_offset;
1611                                        bio->bi_bdev = conf->mirrors[i].rdev->bdev;
1612                                        bio->bi_end_io = raid1_end_write_request;
1613                                        bio->bi_rw = WRITE | do_sync;
1614                                        bio->bi_private = r1_bio;
1615                                        r1_bio->bios[i] = bio;
1616                                        generic_make_request(bio);
1617                                }
1618                } else {
1619                        int disk;
1620
1621                        /* we got a read error. Maybe the drive is bad.  Maybe just
1622                         * the block and we can fix it.
1623                         * We freeze all other IO, and try reading the block from
1624                         * other devices.  When we find one, we re-write
1625                         * and check it that fixes the read error.
1626                         * This is all done synchronously while the array is
1627                         * frozen
1628                         */
1629                        if (mddev->ro == 0) {
1630                                freeze_array(conf);
1631                                fix_read_error(conf, r1_bio->read_disk,
1632                                               r1_bio->sector,
1633                                               r1_bio->sectors);
1634                                unfreeze_array(conf);
1635                        }
1636
1637                        bio = r1_bio->bios[r1_bio->read_disk];
1638                        if ((disk=read_balance(conf, r1_bio)) == -1) {
1639                                printk(KERN_ALERT "raid1: %s: unrecoverable I/O"
1640                                       " read error for block %llu\n",
1641                                       bdevname(bio->bi_bdev,b),
1642                                       (unsigned long long)r1_bio->sector);
1643                                raid_end_bio_io(r1_bio);
1644                        } else {
1645                                const int do_sync = bio_sync(r1_bio->master_bio);
1646                                r1_bio->bios[r1_bio->read_disk] =
1647                                        mddev->ro ? IO_BLOCKED : NULL;
1648                                r1_bio->read_disk = disk;
1649                                bio_put(bio);
1650                                bio = bio_clone(r1_bio->master_bio, GFP_NOIO);
1651                                r1_bio->bios[r1_bio->read_disk] = bio;
1652                                rdev = conf->mirrors[disk].rdev;
1653                                if (printk_ratelimit())
1654                                        printk(KERN_ERR "raid1: %s: redirecting sector %llu to"
1655                                               " another mirror\n",
1656                                               bdevname(rdev->bdev,b),
1657                                               (unsigned long long)r1_bio->sector);
1658                                bio->bi_sector = r1_bio->sector + rdev->data_offset;
1659                                bio->bi_bdev = rdev->bdev;
1660                                bio->bi_end_io = raid1_end_read_request;
1661                                bio->bi_rw = READ | do_sync;
1662                                bio->bi_private = r1_bio;
1663                                unplug = 1;
1664                                generic_make_request(bio);
1665                        }
1666                }
1667        }
1668        if (unplug)
1669                unplug_slaves(mddev);
1670}
1671
1672
1673static int init_resync(conf_t *conf)
1674{
1675        int buffs;
1676
1677        buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
1678        BUG_ON(conf->r1buf_pool);
1679        conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free,
1680                                          conf->poolinfo);
1681        if (!conf->r1buf_pool)
1682                return -ENOMEM;
1683        conf->next_resync = 0;
1684        return 0;
1685}
1686
1687/*
1688 * perform a "sync" on one "block"
1689 *
1690 * We need to make sure that no normal I/O request - particularly write
1691 * requests - conflict with active sync requests.
1692 *
1693 * This is achieved by tracking pending requests and a 'barrier' concept
1694 * that can be installed to exclude normal IO requests.
1695 */
1696
1697static sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
1698{
1699        conf_t *conf = mddev_to_conf(mddev);
1700        r1bio_t *r1_bio;
1701        struct bio *bio;
1702        sector_t max_sector, nr_sectors;
1703        int disk = -1;
1704        int i;
1705        int wonly = -1;
1706        int write_targets = 0, read_targets = 0;
1707        int sync_blocks;
1708        int still_degraded = 0;
1709
1710        if (!conf->r1buf_pool)
1711        {
1712/*
1713                printk("sync start - bitmap %p\n", mddev->bitmap);
1714*/
1715                if (init_resync(conf))
1716                        return 0;
1717        }
1718
1719        max_sector = mddev->size << 1;
1720        if (sector_nr >= max_sector) {
1721                /* If we aborted, we need to abort the
1722                 * sync on the 'current' bitmap chunk (there will
1723                 * only be one in raid1 resync.
1724                 * We can find the current addess in mddev->curr_resync
1725                 */
1726                if (mddev->curr_resync < max_sector) /* aborted */
1727                        bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
1728                                                &sync_blocks, 1);
1729                else /* completed sync */
1730                        conf->fullsync = 0;
1731
1732                bitmap_close_sync(mddev->bitmap);
1733                close_sync(conf);
1734                return 0;
1735        }
1736
1737        if (mddev->bitmap == NULL &&
1738            mddev->recovery_cp == MaxSector &&
1739            !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
1740            conf->fullsync == 0) {
1741                *skipped = 1;
1742                return max_sector - sector_nr;
1743        }
1744        /* before building a request, check if we can skip these blocks..
1745         * This call the bitmap_start_sync doesn't actually record anything
1746         */
1747        if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
1748            !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
1749                /* We can skip this block, and probably several more */
1750                *skipped = 1;
1751                return sync_blocks;
1752        }
1753        /*
1754         * If there is non-resync activity waiting for a turn,
1755         * and resync is going fast enough,
1756         * then let it though before starting on this new sync request.
1757         */
1758        if (!go_faster && conf->nr_waiting)
1759                msleep_interruptible(1000);
1760
1761        bitmap_cond_end_sync(mddev->bitmap, sector_nr);
1762        raise_barrier(conf);
1763
1764        conf->next_resync = sector_nr;
1765
1766        r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
1767        rcu_read_lock();
1768        /*
1769         * If we get a correctably read error during resync or recovery,
1770         * we might want to read from a different device.  So we
1771         * flag all drives that could conceivably be read from for READ,
1772         * and any others (which will be non-In_sync devices) for WRITE.
1773         * If a read fails, we try reading from something else for which READ
1774         * is OK.
1775         */
1776
1777        r1_bio->mddev = mddev;
1778        r1_bio->sector = sector_nr;
1779        r1_bio->state = 0;
1780        set_bit(R1BIO_IsSync, &r1_bio->state);
1781
1782        for (i=0; i < conf->raid_disks; i++) {
1783                mdk_rdev_t *rdev;
1784                bio = r1_bio->bios[i];
1785
1786                /* take from bio_init */
1787                bio->bi_next = NULL;
1788                bio->bi_flags |= 1 << BIO_UPTODATE;
1789                bio->bi_rw = READ;
1790                bio->bi_vcnt = 0;
1791                bio->bi_idx = 0;
1792                bio->bi_phys_segments = 0;
1793                bio->bi_hw_segments = 0;
1794                bio->bi_size = 0;
1795                bio->bi_end_io = NULL;
1796                bio->bi_private = NULL;
1797
1798                rdev = rcu_dereference(conf->mirrors[i].rdev);
1799                if (rdev == NULL ||
1800                           test_bit(Faulty, &rdev->flags)) {
1801                        still_degraded = 1;
1802                        continue;
1803                } else if (!test_bit(In_sync, &rdev->flags)) {
1804                        bio->bi_rw = WRITE;
1805                        bio->bi_end_io = end_sync_write;
1806                        write_targets ++;
1807                } else {
1808                        /* may need to read from here */
1809                        bio->bi_rw = READ;
1810                        bio->bi_end_io = end_sync_read;
1811                        if (test_bit(WriteMostly, &rdev->flags)) {
1812                                if (wonly < 0)
1813                                        wonly = i;
1814                        } else {
1815                                if (disk < 0)
1816                                        disk = i;
1817                        }
1818                        read_targets++;
1819                }
1820                atomic_inc(&rdev->nr_pending);
1821                bio->bi_sector = sector_nr + rdev->data_offset;
1822                bio->bi_bdev = rdev->bdev;
1823                bio->bi_private = r1_bio;
1824        }
1825        rcu_read_unlock();
1826        if (disk < 0)
1827                disk = wonly;
1828        r1_bio->read_disk = disk;
1829
1830        if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0)
1831                /* extra read targets are also write targets */
1832                write_targets += read_targets-1;
1833
1834        if (write_targets == 0 || read_targets == 0) {
1835                /* There is nowhere to write, so all non-sync
1836                 * drives must be failed - so we are finished
1837                 */
1838                sector_t rv = max_sector - sector_nr;
1839                *skipped = 1;
1840                put_buf(r1_bio);
1841                return rv;
1842        }
1843
1844        if (max_sector > mddev->resync_max)
1845                max_sector = mddev->resync_max; /* Don't do IO beyond here */
1846        nr_sectors = 0;
1847        sync_blocks = 0;
1848        do {
1849                struct page *page;
1850                int len = PAGE_SIZE;
1851                if (sector_nr + (len>>9) > max_sector)
1852                        len = (max_sector - sector_nr) << 9;
1853                if (len == 0)
1854                        break;
1855                if (sync_blocks == 0) {
1856                        if (!bitmap_start_sync(mddev->bitmap, sector_nr,
1857                                               &sync_blocks, still_degraded) &&
1858                            !conf->fullsync &&
1859                            !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
1860                                break;
1861                        BUG_ON(sync_blocks < (PAGE_SIZE>>9));
1862                        if (len > (sync_blocks<<9))
1863                                len = sync_blocks<<9;
1864                }
1865
1866                for (i=0 ; i < conf->raid_disks; i++) {
1867                        bio = r1_bio->bios[i];
1868                        if (bio->bi_end_io) {
1869                                page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
1870                                if (bio_add_page(bio, page, len, 0) == 0) {
1871                                        /* stop here */
1872                                        bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
1873                                        while (i > 0) {
1874                                                i--;
1875                                                bio = r1_bio->bios[i];
1876                                                if (bio->bi_end_io==NULL)
1877                                                        continue;
1878                                                /* remove last page from this bio */
1879                                                bio->bi_vcnt--;
1880                                                bio->bi_size -= len;
1881                                                bio->bi_flags &= ~(1<< BIO_SEG_VALID);
1882                                        }
1883                                        goto bio_full;
1884                                }
1885                        }
1886                }
1887                nr_sectors += len>>9;
1888                sector_nr += len>>9;
1889                sync_blocks -= (len>>9);
1890        } while (r1_bio->bios[disk]->bi_vcnt < RESYNC_PAGES);
1891 bio_full:
1892        r1_bio->sectors = nr_sectors;
1893
1894        /* For a user-requested sync, we read all readable devices and do a
1895         * compare
1896         */
1897        if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
1898                atomic_set(&r1_bio->remaining, read_targets);
1899                for (i=0; i<conf->raid_disks; i++) {
1900                        bio = r1_bio->bios[i];
1901                        if (bio->bi_end_io == end_sync_read) {
1902                                md_sync_acct(bio->bi_bdev, nr_sectors);
1903                                generic_make_request(bio);
1904                        }
1905                }
1906        } else {
1907                atomic_set(&r1_bio->remaining, 1);
1908                bio = r1_bio->bios[r1_bio->read_disk];
1909                md_sync_acct(bio->bi_bdev, nr_sectors);
1910                generic_make_request(bio);
1911
1912        }
1913        return nr_sectors;
1914}
1915
1916static int run(mddev_t *mddev)
1917{
1918        conf_t *conf;
1919        int i, j, disk_idx;
1920        mirror_info_t *disk;
1921        mdk_rdev_t *rdev;
1922        struct list_head *tmp;
1923
1924        if (mddev->level != 1) {
1925                printk("raid1: %s: raid level not set to mirroring (%d)\n",
1926                       mdname(mddev), mddev->level);
1927                goto out;
1928        }
1929        if (mddev->reshape_position != MaxSector) {
1930                printk("raid1: %s: reshape_position set but not supported\n",
1931                       mdname(mddev));
1932                goto out;
1933        }
1934        /*
1935         * copy the already verified devices into our private RAID1
1936         * bookkeeping area. [whatever we allocate in run(),
1937         * should be freed in stop()]
1938         */
1939        conf = kzalloc(sizeof(conf_t), GFP_KERNEL);
1940        mddev->private = conf;
1941        if (!conf)
1942                goto out_no_mem;
1943
1944        conf->mirrors = kzalloc(sizeof(struct mirror_info)*mddev->raid_disks,
1945                                 GFP_KERNEL);
1946        if (!conf->mirrors)
1947                goto out_no_mem;
1948
1949        conf->tmppage = alloc_page(GFP_KERNEL);
1950        if (!conf->tmppage)
1951                goto out_no_mem;
1952
1953        conf->poolinfo = kmalloc(sizeof(*conf->poolinfo), GFP_KERNEL);
1954        if (!conf->poolinfo)
1955                goto out_no_mem;
1956        conf->poolinfo->mddev = mddev;
1957        conf->poolinfo->raid_disks = mddev->raid_disks;
1958        conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
1959                                          r1bio_pool_free,
1960                                          conf->poolinfo);
1961        if (!conf->r1bio_pool)
1962                goto out_no_mem;
1963
1964        spin_lock_init(&conf->device_lock);
1965        mddev->queue->queue_lock = &conf->device_lock;
1966
1967        rdev_for_each(rdev, tmp, mddev) {
1968                disk_idx = rdev->raid_disk;
1969                if (disk_idx >= mddev->raid_disks
1970                    || disk_idx < 0)
1971                        continue;
1972                disk = conf->mirrors + disk_idx;
1973
1974                disk->rdev = rdev;
1975
1976                blk_queue_stack_limits(mddev->queue,
1977                                       rdev->bdev->bd_disk->queue);
1978                /* as we don't honour merge_bvec_fn, we must never risk
1979                 * violating it, so limit ->max_sector to one PAGE, as
1980                 * a one page request is never in violation.
1981                 */
1982                if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
1983                    mddev->queue->max_sectors > (PAGE_SIZE>>9))
1984                        blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
1985
1986                disk->head_position = 0;
1987        }
1988        conf->raid_disks = mddev->raid_disks;
1989        conf->mddev = mddev;
1990        INIT_LIST_HEAD(&conf->retry_list);
1991
1992        spin_lock_init(&conf->resync_lock);
1993        init_waitqueue_head(&conf->wait_barrier);
1994
1995        bio_list_init(&conf->pending_bio_list);
1996        bio_list_init(&conf->flushing_bio_list);
1997
1998
1999        mddev->degraded = 0;
2000        for (i = 0; i < conf->raid_disks; i++) {
2001
2002                disk = conf->mirrors + i;
2003
2004                if (!disk->rdev ||
2005                    !test_bit(In_sync, &disk->rdev->flags)) {
2006                        disk->head_position = 0;
2007                        mddev->degraded++;
2008                        if (disk->rdev)
2009                                conf->fullsync = 1;
2010                }
2011        }
2012        if (mddev->degraded == conf->raid_disks) {
2013                printk(KERN_ERR "raid1: no operational mirrors for %s\n",
2014                        mdname(mddev));
2015                goto out_free_conf;
2016        }
2017        if (conf->raid_disks - mddev->degraded == 1)
2018                mddev->recovery_cp = MaxSector;
2019
2020        /*
2021         * find the first working one and use it as a starting point
2022         * to read balancing.
2023         */
2024        for (j = 0; j < conf->raid_disks &&
2025                     (!conf->mirrors[j].rdev ||
2026                      !test_bit(In_sync, &conf->mirrors[j].rdev->flags)) ; j++)
2027                /* nothing */;
2028        conf->last_used = j;
2029
2030
2031        mddev->thread = md_register_thread(raid1d, mddev, "%s_raid1");
2032        if (!mddev->thread) {
2033                printk(KERN_ERR
2034                       "raid1: couldn't allocate thread for %s\n",
2035                       mdname(mddev));
2036                goto out_free_conf;
2037        }
2038
2039        printk(KERN_INFO 
2040                "raid1: raid set %s active with %d out of %d mirrors\n",
2041                mdname(mddev), mddev->raid_disks - mddev->degraded, 
2042                mddev->raid_disks);
2043        /*
2044         * Ok, everything is just fine now
2045         */
2046        mddev->array_sectors = mddev->size * 2;
2047
2048        mddev->queue->unplug_fn = raid1_unplug;
2049        mddev->queue->backing_dev_info.congested_fn = raid1_congested;
2050        mddev->queue->backing_dev_info.congested_data = mddev;
2051
2052        return 0;
2053
2054out_no_mem:
2055        printk(KERN_ERR "raid1: couldn't allocate memory for %s\n",
2056               mdname(mddev));
2057
2058out_free_conf:
2059        if (conf) {
2060                if (conf->r1bio_pool)
2061                        mempool_destroy(conf->r1bio_pool);
2062                kfree(conf->mirrors);
2063                safe_put_page(conf->tmppage);
2064                kfree(conf->poolinfo);
2065                kfree(conf);
2066                mddev->private = NULL;
2067        }
2068out:
2069        return -EIO;
2070}
2071
2072static int stop(mddev_t *mddev)
2073{
2074        conf_t *conf = mddev_to_conf(mddev);
2075        struct bitmap *bitmap = mddev->bitmap;
2076        int behind_wait = 0;
2077
2078        /* wait for behind writes to complete */
2079        while (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
2080                behind_wait++;
2081                printk(KERN_INFO "raid1: behind writes in progress on device %s, waiting to stop (%d)\n", mdname(mddev), behind_wait);
2082                set_current_state(TASK_UNINTERRUPTIBLE);
2083                schedule_timeout(HZ); /* wait a second */
2084                /* need to kick something here to make sure I/O goes? */
2085        }
2086
2087        md_unregister_thread(mddev->thread);
2088        mddev->thread = NULL;
2089        blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
2090        if (conf->r1bio_pool)
2091                mempool_destroy(conf->r1bio_pool);
2092        kfree(conf->mirrors);
2093        kfree(conf->poolinfo);
2094        kfree(conf);
2095        mddev->private = NULL;
2096        return 0;
2097}
2098
2099static int raid1_resize(mddev_t *mddev, sector_t sectors)
2100{
2101        /* no resync is happening, and there is enough space
2102         * on all devices, so we can resize.
2103         * We need to make sure resync covers any new space.
2104         * If the array is shrinking we should possibly wait until
2105         * any io in the removed space completes, but it hardly seems
2106         * worth it.
2107         */
2108        mddev->array_sectors = sectors;
2109        set_capacity(mddev->gendisk, mddev->array_sectors);
2110        mddev->changed = 1;
2111        if (mddev->array_sectors / 2 > mddev->size &&
2112            mddev->recovery_cp == MaxSector) {
2113                mddev->recovery_cp = mddev->size << 1;
2114                set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
2115        }
2116        mddev->size = mddev->array_sectors / 2;
2117        mddev->resync_max_sectors = sectors;
2118        return 0;
2119}
2120
2121static int raid1_reshape(mddev_t *mddev)
2122{
2123        /* We need to:
2124         * 1/ resize the r1bio_pool
2125         * 2/ resize conf->mirrors
2126         *
2127         * We allocate a new r1bio_pool if we can.
2128         * Then raise a device barrier and wait until all IO stops.
2129         * Then resize conf->mirrors and swap in the new r1bio pool.
2130         *
2131         * At the same time, we "pack" the devices so that all the missing
2132         * devices have the higher raid_disk numbers.
2133         */
2134        mempool_t *newpool, *oldpool;
2135        struct pool_info *newpoolinfo;
2136        mirror_info_t *newmirrors;
2137        conf_t *conf = mddev_to_conf(mddev);
2138        int cnt, raid_disks;
2139        unsigned long flags;
2140        int d, d2, err;
2141
2142        /* Cannot change chunk_size, layout, or level */
2143        if (mddev->chunk_size != mddev->new_chunk ||
2144            mddev->layout != mddev->new_layout ||
2145            mddev->level != mddev->new_level) {
2146                mddev->new_chunk = mddev->chunk_size;
2147                mddev->new_layout = mddev->layout;
2148                mddev->new_level = mddev->level;
2149                return -EINVAL;
2150        }
2151
2152        err = md_allow_write(mddev);
2153        if (err)
2154                return err;
2155
2156        raid_disks = mddev->raid_disks + mddev->delta_disks;
2157
2158        if (raid_disks < conf->raid_disks) {
2159                cnt=0;
2160                for (d= 0; d < conf->raid_disks; d++)
2161                        if (conf->mirrors[d].rdev)
2162                                cnt++;
2163                if (cnt > raid_disks)
2164                        return -EBUSY;
2165        }
2166
2167        newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);
2168        if (!newpoolinfo)
2169                return -ENOMEM;
2170        newpoolinfo->mddev = mddev;
2171        newpoolinfo->raid_disks = raid_disks;
2172
2173        newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
2174                                 r1bio_pool_free, newpoolinfo);
2175        if (!newpool) {
2176                kfree(newpoolinfo);
2177                return -ENOMEM;
2178        }
2179        newmirrors = kzalloc(sizeof(struct mirror_info) * raid_disks, GFP_KERNEL);
2180        if (!newmirrors) {
2181                kfree(newpoolinfo);
2182                mempool_destroy(newpool);
2183                return -ENOMEM;
2184        }
2185
2186        raise_barrier(conf);
2187
2188        /* ok, everything is stopped */
2189        oldpool = conf->r1bio_pool;
2190        conf->r1bio_pool = newpool;
2191
2192        for (d = d2 = 0; d < conf->raid_disks; d++) {
2193                mdk_rdev_t *rdev = conf->mirrors[d].rdev;
2194                if (rdev && rdev->raid_disk != d2) {
2195                        char nm[20];
2196                        sprintf(nm, "rd%d", rdev->raid_disk);
2197                        sysfs_remove_link(&mddev->kobj, nm);
2198                        rdev->raid_disk = d2;
2199                        sprintf(nm, "rd%d", rdev->raid_disk);
2200                        sysfs_remove_link(&mddev->kobj, nm);
2201                        if (sysfs_create_link(&mddev->kobj,
2202                                              &rdev->kobj, nm))
2203                                printk(KERN_WARNING
2204                                       "md/raid1: cannot register "
2205                                       "%s for %s\n",
2206                                       nm, mdname(mddev));
2207                }
2208                if (rdev)
2209                        newmirrors[d2++].rdev = rdev;
2210        }
2211        kfree(conf->mirrors);
2212        conf->mirrors = newmirrors;
2213        kfree(conf->poolinfo);
2214        conf->poolinfo = newpoolinfo;
2215
2216        spin_lock_irqsave(&conf->device_lock, flags);
2217        mddev->degraded += (raid_disks - conf->raid_disks);
2218        spin_unlock_irqrestore(&conf->device_lock, flags);
2219        conf->raid_disks = mddev->raid_disks = raid_disks;
2220        mddev->delta_disks = 0;
2221
2222        conf->last_used = 0; /* just make sure it is in-range */
2223        lower_barrier(conf);
2224
2225        set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
2226        md_wakeup_thread(mddev->thread);
2227
2228        mempool_destroy(oldpool);
2229        return 0;
2230}
2231
2232static void raid1_quiesce(mddev_t *mddev, int state)
2233{
2234        conf_t *conf = mddev_to_conf(mddev);
2235
2236        switch(state) {
2237        case 1:
2238                raise_barrier(conf);
2239                break;
2240        case 0:
2241                lower_barrier(conf);
2242                break;
2243        }
2244}
2245
2246
2247static struct mdk_personality raid1_personality =
2248{
2249        .name           = "raid1",
2250        .level          = 1,
2251        .owner          = THIS_MODULE,
2252        .make_request   = make_request,
2253        .run            = run,
2254        .stop           = stop,
2255        .status         = status,
2256        .error_handler  = error,
2257        .hot_add_disk   = raid1_add_disk,
2258        .hot_remove_disk= raid1_remove_disk,
2259        .spare_active   = raid1_spare_active,
2260        .sync_request   = sync_request,
2261        .resize         = raid1_resize,
2262        .check_reshape  = raid1_reshape,
2263        .quiesce        = raid1_quiesce,
2264};
2265
2266static int __init raid_init(void)
2267{
2268        return register_md_personality(&raid1_personality);
2269}
2270
2271static void raid_exit(void)
2272{
2273        unregister_md_personality(&raid1_personality);
2274}
2275
2276module_init(raid_init);
2277module_exit(raid_exit);
2278MODULE_LICENSE("GPL");
2279MODULE_ALIAS("md-personality-3"); /* RAID1 */
2280MODULE_ALIAS("md-raid1");
2281MODULE_ALIAS("md-level-1");
2282
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.