linux/block/elevator.c
<<
>>
Prefs
   1/*
   2 *  Block device elevator/IO-scheduler.
   3 *
   4 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
   5 *
   6 * 30042000 Jens Axboe <axboe@kernel.dk> :
   7 *
   8 * Split the elevator a bit so that it is possible to choose a different
   9 * one or even write a new "plug in". There are three pieces:
  10 * - elevator_fn, inserts a new request in the queue list
  11 * - elevator_merge_fn, decides whether a new buffer can be merged with
  12 *   an existing request
  13 * - elevator_dequeue_fn, called when a request is taken off the active list
  14 *
  15 * 20082000 Dave Jones <davej@suse.de> :
  16 * Removed tests for max-bomb-segments, which was breaking elvtune
  17 *  when run without -bN
  18 *
  19 * Jens:
  20 * - Rework again to work with bio instead of buffer_heads
  21 * - loose bi_dev comparisons, partition handling is right now
  22 * - completely modularize elevator setup and teardown
  23 *
  24 */
  25#include <linux/kernel.h>
  26#include <linux/fs.h>
  27#include <linux/blkdev.h>
  28#include <linux/elevator.h>
  29#include <linux/bio.h>
  30#include <linux/module.h>
  31#include <linux/slab.h>
  32#include <linux/init.h>
  33#include <linux/compiler.h>
  34#include <linux/blktrace_api.h>
  35#include <linux/hash.h>
  36#include <linux/uaccess.h>
  37#include <linux/pm_runtime.h>
  38
  39#include <trace/events/block.h>
  40
  41#include "blk.h"
  42#include "blk-cgroup.h"
  43
  44static DEFINE_SPINLOCK(elv_list_lock);
  45static LIST_HEAD(elv_list);
  46
  47/*
  48 * Merge hash stuff.
  49 */
  50#define rq_hash_key(rq)         (blk_rq_pos(rq) + blk_rq_sectors(rq))
  51
  52/*
  53 * Query io scheduler to see if the current process issuing bio may be
  54 * merged with rq.
  55 */
  56static int elv_iosched_allow_merge(struct request *rq, struct bio *bio)
  57{
  58        struct request_queue *q = rq->q;
  59        struct elevator_queue *e = q->elevator;
  60
  61        if (e->type->ops.elevator_allow_merge_fn)
  62                return e->type->ops.elevator_allow_merge_fn(q, rq, bio);
  63
  64        return 1;
  65}
  66
  67/*
  68 * can we safely merge with this request?
  69 */
  70bool elv_rq_merge_ok(struct request *rq, struct bio *bio)
  71{
  72        if (!blk_rq_merge_ok(rq, bio))
  73                return 0;
  74
  75        if (!elv_iosched_allow_merge(rq, bio))
  76                return 0;
  77
  78        return 1;
  79}
  80EXPORT_SYMBOL(elv_rq_merge_ok);
  81
  82static struct elevator_type *elevator_find(const char *name)
  83{
  84        struct elevator_type *e;
  85
  86        list_for_each_entry(e, &elv_list, list) {
  87                if (!strcmp(e->elevator_name, name))
  88                        return e;
  89        }
  90
  91        return NULL;
  92}
  93
  94static void elevator_put(struct elevator_type *e)
  95{
  96        module_put(e->elevator_owner);
  97}
  98
  99static struct elevator_type *elevator_get(const char *name, bool try_loading)
 100{
 101        struct elevator_type *e;
 102
 103        spin_lock(&elv_list_lock);
 104
 105        e = elevator_find(name);
 106        if (!e && try_loading) {
 107                spin_unlock(&elv_list_lock);
 108                request_module("%s-iosched", name);
 109                spin_lock(&elv_list_lock);
 110                e = elevator_find(name);
 111        }
 112
 113        if (e && !try_module_get(e->elevator_owner))
 114                e = NULL;
 115
 116        spin_unlock(&elv_list_lock);
 117
 118        return e;
 119}
 120
 121static char chosen_elevator[ELV_NAME_MAX];
 122
 123static int __init elevator_setup(char *str)
 124{
 125        /*
 126         * Be backwards-compatible with previous kernels, so users
 127         * won't get the wrong elevator.
 128         */
 129        strncpy(chosen_elevator, str, sizeof(chosen_elevator) - 1);
 130        return 1;
 131}
 132
 133__setup("elevator=", elevator_setup);
 134
 135/* called during boot to load the elevator chosen by the elevator param */
 136void __init load_default_elevator_module(void)
 137{
 138        struct elevator_type *e;
 139
 140        if (!chosen_elevator[0])
 141                return;
 142
 143        spin_lock(&elv_list_lock);
 144        e = elevator_find(chosen_elevator);
 145        spin_unlock(&elv_list_lock);
 146
 147        if (!e)
 148                request_module("%s-iosched", chosen_elevator);
 149}
 150
 151static struct kobj_type elv_ktype;
 152
 153struct elevator_queue *elevator_alloc(struct request_queue *q,
 154                                  struct elevator_type *e)
 155{
 156        struct elevator_queue *eq;
 157
 158        eq = kmalloc_node(sizeof(*eq), GFP_KERNEL | __GFP_ZERO, q->node);
 159        if (unlikely(!eq))
 160                goto err;
 161
 162        eq->type = e;
 163        kobject_init(&eq->kobj, &elv_ktype);
 164        mutex_init(&eq->sysfs_lock);
 165        hash_init(eq->hash);
 166
 167        return eq;
 168err:
 169        kfree(eq);
 170        elevator_put(e);
 171        return NULL;
 172}
 173EXPORT_SYMBOL(elevator_alloc);
 174
 175static void elevator_release(struct kobject *kobj)
 176{
 177        struct elevator_queue *e;
 178
 179        e = container_of(kobj, struct elevator_queue, kobj);
 180        elevator_put(e->type);
 181        kfree(e);
 182}
 183
 184int elevator_init(struct request_queue *q, char *name)
 185{
 186        struct elevator_type *e = NULL;
 187        int err;
 188
 189        if (unlikely(q->elevator))
 190                return 0;
 191
 192        INIT_LIST_HEAD(&q->queue_head);
 193        q->last_merge = NULL;
 194        q->end_sector = 0;
 195        q->boundary_rq = NULL;
 196
 197        if (name) {
 198                e = elevator_get(name, true);
 199                if (!e)
 200                        return -EINVAL;
 201        }
 202
 203        /*
 204         * Use the default elevator specified by config boot param or
 205         * config option.  Don't try to load modules as we could be running
 206         * off async and request_module() isn't allowed from async.
 207         */
 208        if (!e && *chosen_elevator) {
 209                e = elevator_get(chosen_elevator, false);
 210                if (!e)
 211                        printk(KERN_ERR "I/O scheduler %s not found\n",
 212                                                        chosen_elevator);
 213        }
 214
 215        if (!e) {
 216                e = elevator_get(CONFIG_DEFAULT_IOSCHED, false);
 217                if (!e) {
 218                        printk(KERN_ERR
 219                                "Default I/O scheduler not found. " \
 220                                "Using noop.\n");
 221                        e = elevator_get("noop", false);
 222                }
 223        }
 224
 225        err = e->ops.elevator_init_fn(q, e);
 226        return 0;
 227}
 228EXPORT_SYMBOL(elevator_init);
 229
 230void elevator_exit(struct elevator_queue *e)
 231{
 232        mutex_lock(&e->sysfs_lock);
 233        if (e->type->ops.elevator_exit_fn)
 234                e->type->ops.elevator_exit_fn(e);
 235        mutex_unlock(&e->sysfs_lock);
 236
 237        kobject_put(&e->kobj);
 238}
 239EXPORT_SYMBOL(elevator_exit);
 240
 241static inline void __elv_rqhash_del(struct request *rq)
 242{
 243        hash_del(&rq->hash);
 244}
 245
 246static void elv_rqhash_del(struct request_queue *q, struct request *rq)
 247{
 248        if (ELV_ON_HASH(rq))
 249                __elv_rqhash_del(rq);
 250}
 251
 252static void elv_rqhash_add(struct request_queue *q, struct request *rq)
 253{
 254        struct elevator_queue *e = q->elevator;
 255
 256        BUG_ON(ELV_ON_HASH(rq));
 257        hash_add(e->hash, &rq->hash, rq_hash_key(rq));
 258}
 259
 260static void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
 261{
 262        __elv_rqhash_del(rq);
 263        elv_rqhash_add(q, rq);
 264}
 265
 266static struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
 267{
 268        struct elevator_queue *e = q->elevator;
 269        struct hlist_node *next;
 270        struct request *rq;
 271
 272        hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {
 273                BUG_ON(!ELV_ON_HASH(rq));
 274
 275                if (unlikely(!rq_mergeable(rq))) {
 276                        __elv_rqhash_del(rq);
 277                        continue;
 278                }
 279
 280                if (rq_hash_key(rq) == offset)
 281                        return rq;
 282        }
 283
 284        return NULL;
 285}
 286
 287/*
 288 * RB-tree support functions for inserting/lookup/removal of requests
 289 * in a sorted RB tree.
 290 */
 291void elv_rb_add(struct rb_root *root, struct request *rq)
 292{
 293        struct rb_node **p = &root->rb_node;
 294        struct rb_node *parent = NULL;
 295        struct request *__rq;
 296
 297        while (*p) {
 298                parent = *p;
 299                __rq = rb_entry(parent, struct request, rb_node);
 300
 301                if (blk_rq_pos(rq) < blk_rq_pos(__rq))
 302                        p = &(*p)->rb_left;
 303                else if (blk_rq_pos(rq) >= blk_rq_pos(__rq))
 304                        p = &(*p)->rb_right;
 305        }
 306
 307        rb_link_node(&rq->rb_node, parent, p);
 308        rb_insert_color(&rq->rb_node, root);
 309}
 310EXPORT_SYMBOL(elv_rb_add);
 311
 312void elv_rb_del(struct rb_root *root, struct request *rq)
 313{
 314        BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
 315        rb_erase(&rq->rb_node, root);
 316        RB_CLEAR_NODE(&rq->rb_node);
 317}
 318EXPORT_SYMBOL(elv_rb_del);
 319
 320struct request *elv_rb_find(struct rb_root *root, sector_t sector)
 321{
 322        struct rb_node *n = root->rb_node;
 323        struct request *rq;
 324
 325        while (n) {
 326                rq = rb_entry(n, struct request, rb_node);
 327
 328                if (sector < blk_rq_pos(rq))
 329                        n = n->rb_left;
 330                else if (sector > blk_rq_pos(rq))
 331                        n = n->rb_right;
 332                else
 333                        return rq;
 334        }
 335
 336        return NULL;
 337}
 338EXPORT_SYMBOL(elv_rb_find);
 339
 340/*
 341 * Insert rq into dispatch queue of q.  Queue lock must be held on
 342 * entry.  rq is sort instead into the dispatch queue. To be used by
 343 * specific elevators.
 344 */
 345void elv_dispatch_sort(struct request_queue *q, struct request *rq)
 346{
 347        sector_t boundary;
 348        struct list_head *entry;
 349        int stop_flags;
 350
 351        if (q->last_merge == rq)
 352                q->last_merge = NULL;
 353
 354        elv_rqhash_del(q, rq);
 355
 356        q->nr_sorted--;
 357
 358        boundary = q->end_sector;
 359        stop_flags = REQ_SOFTBARRIER | REQ_STARTED;
 360        list_for_each_prev(entry, &q->queue_head) {
 361                struct request *pos = list_entry_rq(entry);
 362
 363                if ((rq->cmd_flags & REQ_DISCARD) !=
 364                    (pos->cmd_flags & REQ_DISCARD))
 365                        break;
 366                if (rq_data_dir(rq) != rq_data_dir(pos))
 367                        break;
 368                if (pos->cmd_flags & stop_flags)
 369                        break;
 370                if (blk_rq_pos(rq) >= boundary) {
 371                        if (blk_rq_pos(pos) < boundary)
 372                                continue;
 373                } else {
 374                        if (blk_rq_pos(pos) >= boundary)
 375                                break;
 376                }
 377                if (blk_rq_pos(rq) >= blk_rq_pos(pos))
 378                        break;
 379        }
 380
 381        list_add(&rq->queuelist, entry);
 382}
 383EXPORT_SYMBOL(elv_dispatch_sort);
 384
 385/*
 386 * Insert rq into dispatch queue of q.  Queue lock must be held on
 387 * entry.  rq is added to the back of the dispatch queue. To be used by
 388 * specific elevators.
 389 */
 390void elv_dispatch_add_tail(struct request_queue *q, struct request *rq)
 391{
 392        if (q->last_merge == rq)
 393                q->last_merge = NULL;
 394
 395        elv_rqhash_del(q, rq);
 396
 397        q->nr_sorted--;
 398
 399        q->end_sector = rq_end_sector(rq);
 400        q->boundary_rq = rq;
 401        list_add_tail(&rq->queuelist, &q->queue_head);
 402}
 403EXPORT_SYMBOL(elv_dispatch_add_tail);
 404
 405int elv_merge(struct request_queue *q, struct request **req, struct bio *bio)
 406{
 407        struct elevator_queue *e = q->elevator;
 408        struct request *__rq;
 409        int ret;
 410
 411        /*
 412         * Levels of merges:
 413         *      nomerges:  No merges at all attempted
 414         *      noxmerges: Only simple one-hit cache try
 415         *      merges:    All merge tries attempted
 416         */
 417        if (blk_queue_nomerges(q))
 418                return ELEVATOR_NO_MERGE;
 419
 420        /*
 421         * First try one-hit cache.
 422         */
 423        if (q->last_merge && elv_rq_merge_ok(q->last_merge, bio)) {
 424                ret = blk_try_merge(q->last_merge, bio);
 425                if (ret != ELEVATOR_NO_MERGE) {
 426                        *req = q->last_merge;
 427                        return ret;
 428                }
 429        }
 430
 431        if (blk_queue_noxmerges(q))
 432                return ELEVATOR_NO_MERGE;
 433
 434        /*
 435         * See if our hash lookup can find a potential backmerge.
 436         */
 437        __rq = elv_rqhash_find(q, bio->bi_sector);
 438        if (__rq && elv_rq_merge_ok(__rq, bio)) {
 439                *req = __rq;
 440                return ELEVATOR_BACK_MERGE;
 441        }
 442
 443        if (e->type->ops.elevator_merge_fn)
 444                return e->type->ops.elevator_merge_fn(q, req, bio);
 445
 446        return ELEVATOR_NO_MERGE;
 447}
 448
 449/*
 450 * Attempt to do an insertion back merge. Only check for the case where
 451 * we can append 'rq' to an existing request, so we can throw 'rq' away
 452 * afterwards.
 453 *
 454 * Returns true if we merged, false otherwise
 455 */
 456static bool elv_attempt_insert_merge(struct request_queue *q,
 457                                     struct request *rq)
 458{
 459        struct request *__rq;
 460        bool ret;
 461
 462        if (blk_queue_nomerges(q))
 463                return false;
 464
 465        /*
 466         * First try one-hit cache.
 467         */
 468        if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))
 469                return true;
 470
 471        if (blk_queue_noxmerges(q))
 472                return false;
 473
 474        ret = false;
 475        /*
 476         * See if our hash lookup can find a potential backmerge.
 477         */
 478        while (1) {
 479                __rq = elv_rqhash_find(q, blk_rq_pos(rq));
 480                if (!__rq || !blk_attempt_req_merge(q, __rq, rq))
 481                        break;
 482
 483                /* The merged request could be merged with others, try again */
 484                ret = true;
 485                rq = __rq;
 486        }
 487
 488        return ret;
 489}
 490
 491void elv_merged_request(struct request_queue *q, struct request *rq, int type)
 492{
 493        struct elevator_queue *e = q->elevator;
 494
 495        if (e->type->ops.elevator_merged_fn)
 496                e->type->ops.elevator_merged_fn(q, rq, type);
 497
 498        if (type == ELEVATOR_BACK_MERGE)
 499                elv_rqhash_reposition(q, rq);
 500
 501        q->last_merge = rq;
 502}
 503
 504void elv_merge_requests(struct request_queue *q, struct request *rq,
 505                             struct request *next)
 506{
 507        struct elevator_queue *e = q->elevator;
 508        const int next_sorted = next->cmd_flags & REQ_SORTED;
 509
 510        if (next_sorted && e->type->ops.elevator_merge_req_fn)
 511                e->type->ops.elevator_merge_req_fn(q, rq, next);
 512
 513        elv_rqhash_reposition(q, rq);
 514
 515        if (next_sorted) {
 516                elv_rqhash_del(q, next);
 517                q->nr_sorted--;
 518        }
 519
 520        q->last_merge = rq;
 521}
 522
 523void elv_bio_merged(struct request_queue *q, struct request *rq,
 524                        struct bio *bio)
 525{
 526        struct elevator_queue *e = q->elevator;
 527
 528        if (e->type->ops.elevator_bio_merged_fn)
 529                e->type->ops.elevator_bio_merged_fn(q, rq, bio);
 530}
 531
 532#ifdef CONFIG_PM_RUNTIME
 533static void blk_pm_requeue_request(struct request *rq)
 534{
 535        if (rq->q->dev && !(rq->cmd_flags & REQ_PM))
 536                rq->q->nr_pending--;
 537}
 538
 539static void blk_pm_add_request(struct request_queue *q, struct request *rq)
 540{
 541        if (q->dev && !(rq->cmd_flags & REQ_PM) && q->nr_pending++ == 0 &&
 542            (q->rpm_status == RPM_SUSPENDED || q->rpm_status == RPM_SUSPENDING))
 543                pm_request_resume(q->dev);
 544}
 545#else
 546static inline void blk_pm_requeue_request(struct request *rq) {}
 547static inline void blk_pm_add_request(struct request_queue *q,
 548                                      struct request *rq)
 549{
 550}
 551#endif
 552
 553void elv_requeue_request(struct request_queue *q, struct request *rq)
 554{
 555        /*
 556         * it already went through dequeue, we need to decrement the
 557         * in_flight count again
 558         */
 559        if (blk_account_rq(rq)) {
 560                q->in_flight[rq_is_sync(rq)]--;
 561                if (rq->cmd_flags & REQ_SORTED)
 562                        elv_deactivate_rq(q, rq);
 563        }
 564
 565        rq->cmd_flags &= ~REQ_STARTED;
 566
 567        blk_pm_requeue_request(rq);
 568
 569        __elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE);
 570}
 571
 572void elv_drain_elevator(struct request_queue *q)
 573{
 574        static int printed;
 575
 576        lockdep_assert_held(q->queue_lock);
 577
 578        while (q->elevator->type->ops.elevator_dispatch_fn(q, 1))
 579                ;
 580        if (q->nr_sorted && printed++ < 10) {
 581                printk(KERN_ERR "%s: forced dispatching is broken "
 582                       "(nr_sorted=%u), please report this\n",
 583                       q->elevator->type->elevator_name, q->nr_sorted);
 584        }
 585}
 586
 587void __elv_add_request(struct request_queue *q, struct request *rq, int where)
 588{
 589        trace_block_rq_insert(q, rq);
 590
 591        blk_pm_add_request(q, rq);
 592
 593        rq->q = q;
 594
 595        if (rq->cmd_flags & REQ_SOFTBARRIER) {
 596                /* barriers are scheduling boundary, update end_sector */
 597                if (rq->cmd_type == REQ_TYPE_FS) {
 598                        q->end_sector = rq_end_sector(rq);
 599                        q->boundary_rq = rq;
 600                }
 601        } else if (!(rq->cmd_flags & REQ_ELVPRIV) &&
 602                    (where == ELEVATOR_INSERT_SORT ||
 603                     where == ELEVATOR_INSERT_SORT_MERGE))
 604                where = ELEVATOR_INSERT_BACK;
 605
 606        switch (where) {
 607        case ELEVATOR_INSERT_REQUEUE:
 608        case ELEVATOR_INSERT_FRONT:
 609                rq->cmd_flags |= REQ_SOFTBARRIER;
 610                list_add(&rq->queuelist, &q->queue_head);
 611                break;
 612
 613        case ELEVATOR_INSERT_BACK:
 614                rq->cmd_flags |= REQ_SOFTBARRIER;
 615                elv_drain_elevator(q);
 616                list_add_tail(&rq->queuelist, &q->queue_head);
 617                /*
 618                 * We kick the queue here for the following reasons.
 619                 * - The elevator might have returned NULL previously
 620                 *   to delay requests and returned them now.  As the
 621                 *   queue wasn't empty before this request, ll_rw_blk
 622                 *   won't run the queue on return, resulting in hang.
 623                 * - Usually, back inserted requests won't be merged
 624                 *   with anything.  There's no point in delaying queue
 625                 *   processing.
 626                 */
 627                __blk_run_queue(q);
 628                break;
 629
 630        case ELEVATOR_INSERT_SORT_MERGE:
 631                /*
 632                 * If we succeed in merging this request with one in the
 633                 * queue already, we are done - rq has now been freed,
 634                 * so no need to do anything further.
 635                 */
 636                if (elv_attempt_insert_merge(q, rq))
 637                        break;
 638        case ELEVATOR_INSERT_SORT:
 639                BUG_ON(rq->cmd_type != REQ_TYPE_FS);
 640                rq->cmd_flags |= REQ_SORTED;
 641                q->nr_sorted++;
 642                if (rq_mergeable(rq)) {
 643                        elv_rqhash_add(q, rq);
 644                        if (!q->last_merge)
 645                                q->last_merge = rq;
 646                }
 647
 648                /*
 649                 * Some ioscheds (cfq) run q->request_fn directly, so
 650                 * rq cannot be accessed after calling
 651                 * elevator_add_req_fn.
 652                 */
 653                q->elevator->type->ops.elevator_add_req_fn(q, rq);
 654                break;
 655
 656        case ELEVATOR_INSERT_FLUSH:
 657                rq->cmd_flags |= REQ_SOFTBARRIER;
 658                blk_insert_flush(rq);
 659                break;
 660        default:
 661                printk(KERN_ERR "%s: bad insertion point %d\n",
 662                       __func__, where);
 663                BUG();
 664        }
 665}
 666EXPORT_SYMBOL(__elv_add_request);
 667
 668void elv_add_request(struct request_queue *q, struct request *rq, int where)
 669{
 670        unsigned long flags;
 671
 672        spin_lock_irqsave(q->queue_lock, flags);
 673        __elv_add_request(q, rq, where);
 674        spin_unlock_irqrestore(q->queue_lock, flags);
 675}
 676EXPORT_SYMBOL(elv_add_request);
 677
 678struct request *elv_latter_request(struct request_queue *q, struct request *rq)
 679{
 680        struct elevator_queue *e = q->elevator;
 681
 682        if (e->type->ops.elevator_latter_req_fn)
 683                return e->type->ops.elevator_latter_req_fn(q, rq);
 684        return NULL;
 685}
 686
 687struct request *elv_former_request(struct request_queue *q, struct request *rq)
 688{
 689        struct elevator_queue *e = q->elevator;
 690
 691        if (e->type->ops.elevator_former_req_fn)
 692                return e->type->ops.elevator_former_req_fn(q, rq);
 693        return NULL;
 694}
 695
 696int elv_set_request(struct request_queue *q, struct request *rq,
 697                    struct bio *bio, gfp_t gfp_mask)
 698{
 699        struct elevator_queue *e = q->elevator;
 700
 701        if (e->type->ops.elevator_set_req_fn)
 702                return e->type->ops.elevator_set_req_fn(q, rq, bio, gfp_mask);
 703        return 0;
 704}
 705
 706void elv_put_request(struct request_queue *q, struct request *rq)
 707{
 708        struct elevator_queue *e = q->elevator;
 709
 710        if (e->type->ops.elevator_put_req_fn)
 711                e->type->ops.elevator_put_req_fn(rq);
 712}
 713
 714int elv_may_queue(struct request_queue *q, int rw)
 715{
 716        struct elevator_queue *e = q->elevator;
 717
 718        if (e->type->ops.elevator_may_queue_fn)
 719                return e->type->ops.elevator_may_queue_fn(q, rw);
 720
 721        return ELV_MQUEUE_MAY;
 722}
 723
 724void elv_abort_queue(struct request_queue *q)
 725{
 726        struct request *rq;
 727
 728        blk_abort_flushes(q);
 729
 730        while (!list_empty(&q->queue_head)) {
 731                rq = list_entry_rq(q->queue_head.next);
 732                rq->cmd_flags |= REQ_QUIET;
 733                trace_block_rq_abort(q, rq);
 734                /*
 735                 * Mark this request as started so we don't trigger
 736                 * any debug logic in the end I/O path.
 737                 */
 738                blk_start_request(rq);
 739                __blk_end_request_all(rq, -EIO);
 740        }
 741}
 742EXPORT_SYMBOL(elv_abort_queue);
 743
 744void elv_completed_request(struct request_queue *q, struct request *rq)
 745{
 746        struct elevator_queue *e = q->elevator;
 747
 748        /*
 749         * request is released from the driver, io must be done
 750         */
 751        if (blk_account_rq(rq)) {
 752                q->in_flight[rq_is_sync(rq)]--;
 753                if ((rq->cmd_flags & REQ_SORTED) &&
 754                    e->type->ops.elevator_completed_req_fn)
 755                        e->type->ops.elevator_completed_req_fn(q, rq);
 756        }
 757}
 758
 759#define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
 760
 761static ssize_t
 762elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 763{
 764        struct elv_fs_entry *entry = to_elv(attr);
 765        struct elevator_queue *e;
 766        ssize_t error;
 767
 768        if (!entry->show)
 769                return -EIO;
 770
 771        e = container_of(kobj, struct elevator_queue, kobj);
 772        mutex_lock(&e->sysfs_lock);
 773        error = e->type ? entry->show(e, page) : -ENOENT;
 774        mutex_unlock(&e->sysfs_lock);
 775        return error;
 776}
 777
 778static ssize_t
 779elv_attr_store(struct kobject *kobj, struct attribute *attr,
 780               const char *page, size_t length)
 781{
 782        struct elv_fs_entry *entry = to_elv(attr);
 783        struct elevator_queue *e;
 784        ssize_t error;
 785
 786        if (!entry->store)
 787                return -EIO;
 788
 789        e = container_of(kobj, struct elevator_queue, kobj);
 790        mutex_lock(&e->sysfs_lock);
 791        error = e->type ? entry->store(e, page, length) : -ENOENT;
 792        mutex_unlock(&e->sysfs_lock);
 793        return error;
 794}
 795
 796static const struct sysfs_ops elv_sysfs_ops = {
 797        .show   = elv_attr_show,
 798        .store  = elv_attr_store,
 799};
 800
 801static struct kobj_type elv_ktype = {
 802        .sysfs_ops      = &elv_sysfs_ops,
 803        .release        = elevator_release,
 804};
 805
 806int elv_register_queue(struct request_queue *q)
 807{
 808        struct elevator_queue *e = q->elevator;
 809        int error;
 810
 811        error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
 812        if (!error) {
 813                struct elv_fs_entry *attr = e->type->elevator_attrs;
 814                if (attr) {
 815                        while (attr->attr.name) {
 816                                if (sysfs_create_file(&e->kobj, &attr->attr))
 817                                        break;
 818                                attr++;
 819                        }
 820                }
 821                kobject_uevent(&e->kobj, KOBJ_ADD);
 822                e->registered = 1;
 823        }
 824        return error;
 825}
 826EXPORT_SYMBOL(elv_register_queue);
 827
 828void elv_unregister_queue(struct request_queue *q)
 829{
 830        if (q) {
 831                struct elevator_queue *e = q->elevator;
 832
 833                kobject_uevent(&e->kobj, KOBJ_REMOVE);
 834                kobject_del(&e->kobj);
 835                e->registered = 0;
 836        }
 837}
 838EXPORT_SYMBOL(elv_unregister_queue);
 839
 840int elv_register(struct elevator_type *e)
 841{
 842        char *def = "";
 843
 844        /* create icq_cache if requested */
 845        if (e->icq_size) {
 846                if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
 847                    WARN_ON(e->icq_align < __alignof__(struct io_cq)))
 848                        return -EINVAL;
 849
 850                snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
 851                         "%s_io_cq", e->elevator_name);
 852                e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
 853                                                 e->icq_align, 0, NULL);
 854                if (!e->icq_cache)
 855                        return -ENOMEM;
 856        }
 857
 858        /* register, don't allow duplicate names */
 859        spin_lock(&elv_list_lock);
 860        if (elevator_find(e->elevator_name)) {
 861                spin_unlock(&elv_list_lock);
 862                if (e->icq_cache)
 863                        kmem_cache_destroy(e->icq_cache);
 864                return -EBUSY;
 865        }
 866        list_add_tail(&e->list, &elv_list);
 867        spin_unlock(&elv_list_lock);
 868
 869        /* print pretty message */
 870        if (!strcmp(e->elevator_name, chosen_elevator) ||
 871                        (!*chosen_elevator &&
 872                         !strcmp(e->elevator_name, CONFIG_DEFAULT_IOSCHED)))
 873                                def = " (default)";
 874
 875        printk(KERN_INFO "io scheduler %s registered%s\n", e->elevator_name,
 876                                                                def);
 877        return 0;
 878}
 879EXPORT_SYMBOL_GPL(elv_register);
 880
 881void elv_unregister(struct elevator_type *e)
 882{
 883        /* unregister */
 884        spin_lock(&elv_list_lock);
 885        list_del_init(&e->list);
 886        spin_unlock(&elv_list_lock);
 887
 888        /*
 889         * Destroy icq_cache if it exists.  icq's are RCU managed.  Make
 890         * sure all RCU operations are complete before proceeding.
 891         */
 892        if (e->icq_cache) {
 893                rcu_barrier();
 894                kmem_cache_destroy(e->icq_cache);
 895                e->icq_cache = NULL;
 896        }
 897}
 898EXPORT_SYMBOL_GPL(elv_unregister);
 899
 900/*
 901 * switch to new_e io scheduler. be careful not to introduce deadlocks -
 902 * we don't free the old io scheduler, before we have allocated what we
 903 * need for the new one. this way we have a chance of going back to the old
 904 * one, if the new one fails init for some reason.
 905 */
 906static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
 907{
 908        struct elevator_queue *old = q->elevator;
 909        bool registered = old->registered;
 910        int err;
 911
 912        /*
 913         * Turn on BYPASS and drain all requests w/ elevator private data.
 914         * Block layer doesn't call into a quiesced elevator - all requests
 915         * are directly put on the dispatch list without elevator data
 916         * using INSERT_BACK.  All requests have SOFTBARRIER set and no
 917         * merge happens either.
 918         */
 919        blk_queue_bypass_start(q);
 920
 921        /* unregister and clear all auxiliary data of the old elevator */
 922        if (registered)
 923                elv_unregister_queue(q);
 924
 925        spin_lock_irq(q->queue_lock);
 926        ioc_clear_queue(q);
 927        spin_unlock_irq(q->queue_lock);
 928
 929        /* allocate, init and register new elevator */
 930        err = new_e->ops.elevator_init_fn(q, new_e);
 931        if (err)
 932                goto fail_init;
 933
 934        if (registered) {
 935                err = elv_register_queue(q);
 936                if (err)
 937                        goto fail_register;
 938        }
 939
 940        /* done, kill the old one and finish */
 941        elevator_exit(old);
 942        blk_queue_bypass_end(q);
 943
 944        blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
 945
 946        return 0;
 947
 948fail_register:
 949        elevator_exit(q->elevator);
 950fail_init:
 951        /* switch failed, restore and re-register old elevator */
 952        q->elevator = old;
 953        elv_register_queue(q);
 954        blk_queue_bypass_end(q);
 955
 956        return err;
 957}
 958
 959/*
 960 * Switch this queue to the given IO scheduler.
 961 */
 962int elevator_change(struct request_queue *q, const char *name)
 963{
 964        char elevator_name[ELV_NAME_MAX];
 965        struct elevator_type *e;
 966
 967        if (!q->elevator)
 968                return -ENXIO;
 969
 970        strlcpy(elevator_name, name, sizeof(elevator_name));
 971        e = elevator_get(strstrip(elevator_name), true);
 972        if (!e) {
 973                printk(KERN_ERR "elevator: type %s not found\n", elevator_name);
 974                return -EINVAL;
 975        }
 976
 977        if (!strcmp(elevator_name, q->elevator->type->elevator_name)) {
 978                elevator_put(e);
 979                return 0;
 980        }
 981
 982        return elevator_switch(q, e);
 983}
 984EXPORT_SYMBOL(elevator_change);
 985
 986ssize_t elv_iosched_store(struct request_queue *q, const char *name,
 987                          size_t count)
 988{
 989        int ret;
 990
 991        if (!q->elevator)
 992                return count;
 993
 994        ret = elevator_change(q, name);
 995        if (!ret)
 996                return count;
 997
 998        printk(KERN_ERR "elevator: switch to %s failed\n", name);
 999        return ret;
1000}
1001
1002ssize_t elv_iosched_show(struct request_queue *q, char *name)
1003{
1004        struct elevator_queue *e = q->elevator;
1005        struct elevator_type *elv;
1006        struct elevator_type *__e;
1007        int len = 0;
1008
1009        if (!q->elevator || !blk_queue_stackable(q))
1010                return sprintf(name, "none\n");
1011
1012        elv = e->type;
1013
1014        spin_lock(&elv_list_lock);
1015        list_for_each_entry(__e, &elv_list, list) {
1016                if (!strcmp(elv->elevator_name, __e->elevator_name))
1017                        len += sprintf(name+len, "[%s] ", elv->elevator_name);
1018                else
1019                        len += sprintf(name+len, "%s ", __e->elevator_name);
1020        }
1021        spin_unlock(&elv_list_lock);
1022
1023        len += sprintf(len+name, "\n");
1024        return len;
1025}
1026
1027struct request *elv_rb_former_request(struct request_queue *q,
1028                                      struct request *rq)
1029{
1030        struct rb_node *rbprev = rb_prev(&rq->rb_node);
1031
1032        if (rbprev)
1033                return rb_entry_rq(rbprev);
1034
1035        return NULL;
1036}
1037EXPORT_SYMBOL(elv_rb_former_request);
1038
1039struct request *elv_rb_latter_request(struct request_queue *q,
1040                                      struct request *rq)
1041{
1042        struct rb_node *rbnext = rb_next(&rq->rb_node);
1043
1044        if (rbnext)
1045                return rb_entry_rq(rbnext);
1046
1047        return NULL;
1048}
1049EXPORT_SYMBOL(elv_rb_latter_request);
1050
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.