linux/block/elevator.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 *  Block device elevator/IO-scheduler.
   4 *
   5 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
   6 *
   7 * 30042000 Jens Axboe <axboe@kernel.dk> :
   8 *
   9 * Split the elevator a bit so that it is possible to choose a different
  10 * one or even write a new "plug in". There are three pieces:
  11 * - elevator_fn, inserts a new request in the queue list
  12 * - elevator_merge_fn, decides whether a new buffer can be merged with
  13 *   an existing request
  14 * - elevator_dequeue_fn, called when a request is taken off the active list
  15 *
  16 * 20082000 Dave Jones <davej@suse.de> :
  17 * Removed tests for max-bomb-segments, which was breaking elvtune
  18 *  when run without -bN
  19 *
  20 * Jens:
  21 * - Rework again to work with bio instead of buffer_heads
  22 * - loose bi_dev comparisons, partition handling is right now
  23 * - completely modularize elevator setup and teardown
  24 *
  25 */
  26#include <linux/kernel.h>
  27#include <linux/fs.h>
  28#include <linux/blkdev.h>
  29#include <linux/elevator.h>
  30#include <linux/bio.h>
  31#include <linux/module.h>
  32#include <linux/slab.h>
  33#include <linux/init.h>
  34#include <linux/compiler.h>
  35#include <linux/blktrace_api.h>
  36#include <linux/hash.h>
  37#include <linux/uaccess.h>
  38#include <linux/pm_runtime.h>
  39#include <linux/blk-cgroup.h>
  40
  41#include <trace/events/block.h>
  42
  43#include "blk.h"
  44#include "blk-mq-sched.h"
  45#include "blk-pm.h"
  46#include "blk-wbt.h"
  47
  48static DEFINE_SPINLOCK(elv_list_lock);
  49static LIST_HEAD(elv_list);
  50
  51/*
  52 * Merge hash stuff.
  53 */
  54#define rq_hash_key(rq)         (blk_rq_pos(rq) + blk_rq_sectors(rq))
  55
  56/*
  57 * Query io scheduler to see if the current process issuing bio may be
  58 * merged with rq.
  59 */
  60static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio)
  61{
  62        struct request_queue *q = rq->q;
  63        struct elevator_queue *e = q->elevator;
  64
  65        if (e->type->ops.allow_merge)
  66                return e->type->ops.allow_merge(q, rq, bio);
  67
  68        return 1;
  69}
  70
  71/*
  72 * can we safely merge with this request?
  73 */
  74bool elv_bio_merge_ok(struct request *rq, struct bio *bio)
  75{
  76        if (!blk_rq_merge_ok(rq, bio))
  77                return false;
  78
  79        if (!elv_iosched_allow_bio_merge(rq, bio))
  80                return false;
  81
  82        return true;
  83}
  84EXPORT_SYMBOL(elv_bio_merge_ok);
  85
  86static inline bool elv_support_features(unsigned int elv_features,
  87                                        unsigned int required_features)
  88{
  89        return (required_features & elv_features) == required_features;
  90}
  91
  92/**
  93 * elevator_match - Test an elevator name and features
  94 * @e: Scheduler to test
  95 * @name: Elevator name to test
  96 * @required_features: Features that the elevator must provide
  97 *
  98 * Return true if the elevator @e name matches @name and if @e provides all
  99 * the features specified by @required_features.
 100 */
 101static bool elevator_match(const struct elevator_type *e, const char *name,
 102                           unsigned int required_features)
 103{
 104        if (!elv_support_features(e->elevator_features, required_features))
 105                return false;
 106        if (!strcmp(e->elevator_name, name))
 107                return true;
 108        if (e->elevator_alias && !strcmp(e->elevator_alias, name))
 109                return true;
 110
 111        return false;
 112}
 113
 114/**
 115 * elevator_find - Find an elevator
 116 * @name: Name of the elevator to find
 117 * @required_features: Features that the elevator must provide
 118 *
 119 * Return the first registered scheduler with name @name and supporting the
 120 * features @required_features and NULL otherwise.
 121 */
 122static struct elevator_type *elevator_find(const char *name,
 123                                           unsigned int required_features)
 124{
 125        struct elevator_type *e;
 126
 127        list_for_each_entry(e, &elv_list, list) {
 128                if (elevator_match(e, name, required_features))
 129                        return e;
 130        }
 131
 132        return NULL;
 133}
 134
 135static void elevator_put(struct elevator_type *e)
 136{
 137        module_put(e->elevator_owner);
 138}
 139
 140static struct elevator_type *elevator_get(struct request_queue *q,
 141                                          const char *name, bool try_loading)
 142{
 143        struct elevator_type *e;
 144
 145        spin_lock(&elv_list_lock);
 146
 147        e = elevator_find(name, q->required_elevator_features);
 148        if (!e && try_loading) {
 149                spin_unlock(&elv_list_lock);
 150                request_module("%s-iosched", name);
 151                spin_lock(&elv_list_lock);
 152                e = elevator_find(name, q->required_elevator_features);
 153        }
 154
 155        if (e && !try_module_get(e->elevator_owner))
 156                e = NULL;
 157
 158        spin_unlock(&elv_list_lock);
 159        return e;
 160}
 161
 162static struct kobj_type elv_ktype;
 163
 164struct elevator_queue *elevator_alloc(struct request_queue *q,
 165                                  struct elevator_type *e)
 166{
 167        struct elevator_queue *eq;
 168
 169        eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
 170        if (unlikely(!eq))
 171                return NULL;
 172
 173        eq->type = e;
 174        kobject_init(&eq->kobj, &elv_ktype);
 175        mutex_init(&eq->sysfs_lock);
 176        hash_init(eq->hash);
 177
 178        return eq;
 179}
 180EXPORT_SYMBOL(elevator_alloc);
 181
 182static void elevator_release(struct kobject *kobj)
 183{
 184        struct elevator_queue *e;
 185
 186        e = container_of(kobj, struct elevator_queue, kobj);
 187        elevator_put(e->type);
 188        kfree(e);
 189}
 190
 191void __elevator_exit(struct request_queue *q, struct elevator_queue *e)
 192{
 193        mutex_lock(&e->sysfs_lock);
 194        blk_mq_exit_sched(q, e);
 195        mutex_unlock(&e->sysfs_lock);
 196
 197        kobject_put(&e->kobj);
 198}
 199
 200static inline void __elv_rqhash_del(struct request *rq)
 201{
 202        hash_del(&rq->hash);
 203        rq->rq_flags &= ~RQF_HASHED;
 204}
 205
 206void elv_rqhash_del(struct request_queue *q, struct request *rq)
 207{
 208        if (ELV_ON_HASH(rq))
 209                __elv_rqhash_del(rq);
 210}
 211EXPORT_SYMBOL_GPL(elv_rqhash_del);
 212
 213void elv_rqhash_add(struct request_queue *q, struct request *rq)
 214{
 215        struct elevator_queue *e = q->elevator;
 216
 217        BUG_ON(ELV_ON_HASH(rq));
 218        hash_add(e->hash, &rq->hash, rq_hash_key(rq));
 219        rq->rq_flags |= RQF_HASHED;
 220}
 221EXPORT_SYMBOL_GPL(elv_rqhash_add);
 222
 223void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
 224{
 225        __elv_rqhash_del(rq);
 226        elv_rqhash_add(q, rq);
 227}
 228
 229struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
 230{
 231        struct elevator_queue *e = q->elevator;
 232        struct hlist_node *next;
 233        struct request *rq;
 234
 235        hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {
 236                BUG_ON(!ELV_ON_HASH(rq));
 237
 238                if (unlikely(!rq_mergeable(rq))) {
 239                        __elv_rqhash_del(rq);
 240                        continue;
 241                }
 242
 243                if (rq_hash_key(rq) == offset)
 244                        return rq;
 245        }
 246
 247        return NULL;
 248}
 249
 250/*
 251 * RB-tree support functions for inserting/lookup/removal of requests
 252 * in a sorted RB tree.
 253 */
 254void elv_rb_add(struct rb_root *root, struct request *rq)
 255{
 256        struct rb_node **p = &root->rb_node;
 257        struct rb_node *parent = NULL;
 258        struct request *__rq;
 259
 260        while (*p) {
 261                parent = *p;
 262                __rq = rb_entry(parent, struct request, rb_node);
 263
 264                if (blk_rq_pos(rq) < blk_rq_pos(__rq))
 265                        p = &(*p)->rb_left;
 266                else if (blk_rq_pos(rq) >= blk_rq_pos(__rq))
 267                        p = &(*p)->rb_right;
 268        }
 269
 270        rb_link_node(&rq->rb_node, parent, p);
 271        rb_insert_color(&rq->rb_node, root);
 272}
 273EXPORT_SYMBOL(elv_rb_add);
 274
 275void elv_rb_del(struct rb_root *root, struct request *rq)
 276{
 277        BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
 278        rb_erase(&rq->rb_node, root);
 279        RB_CLEAR_NODE(&rq->rb_node);
 280}
 281EXPORT_SYMBOL(elv_rb_del);
 282
 283struct request *elv_rb_find(struct rb_root *root, sector_t sector)
 284{
 285        struct rb_node *n = root->rb_node;
 286        struct request *rq;
 287
 288        while (n) {
 289                rq = rb_entry(n, struct request, rb_node);
 290
 291                if (sector < blk_rq_pos(rq))
 292                        n = n->rb_left;
 293                else if (sector > blk_rq_pos(rq))
 294                        n = n->rb_right;
 295                else
 296                        return rq;
 297        }
 298
 299        return NULL;
 300}
 301EXPORT_SYMBOL(elv_rb_find);
 302
 303enum elv_merge elv_merge(struct request_queue *q, struct request **req,
 304                struct bio *bio)
 305{
 306        struct elevator_queue *e = q->elevator;
 307        struct request *__rq;
 308
 309        /*
 310         * Levels of merges:
 311         *      nomerges:  No merges at all attempted
 312         *      noxmerges: Only simple one-hit cache try
 313         *      merges:    All merge tries attempted
 314         */
 315        if (blk_queue_nomerges(q) || !bio_mergeable(bio))
 316                return ELEVATOR_NO_MERGE;
 317
 318        /*
 319         * First try one-hit cache.
 320         */
 321        if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {
 322                enum elv_merge ret = blk_try_merge(q->last_merge, bio);
 323
 324                if (ret != ELEVATOR_NO_MERGE) {
 325                        *req = q->last_merge;
 326                        return ret;
 327                }
 328        }
 329
 330        if (blk_queue_noxmerges(q))
 331                return ELEVATOR_NO_MERGE;
 332
 333        /*
 334         * See if our hash lookup can find a potential backmerge.
 335         */
 336        __rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);
 337        if (__rq && elv_bio_merge_ok(__rq, bio)) {
 338                *req = __rq;
 339                return ELEVATOR_BACK_MERGE;
 340        }
 341
 342        if (e->type->ops.request_merge)
 343                return e->type->ops.request_merge(q, req, bio);
 344
 345        return ELEVATOR_NO_MERGE;
 346}
 347
 348/*
 349 * Attempt to do an insertion back merge. Only check for the case where
 350 * we can append 'rq' to an existing request, so we can throw 'rq' away
 351 * afterwards.
 352 *
 353 * Returns true if we merged, false otherwise. 'free' will contain all
 354 * requests that need to be freed.
 355 */
 356bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq,
 357                              struct list_head *free)
 358{
 359        struct request *__rq;
 360        bool ret;
 361
 362        if (blk_queue_nomerges(q))
 363                return false;
 364
 365        /*
 366         * First try one-hit cache.
 367         */
 368        if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq)) {
 369                list_add(&rq->queuelist, free);
 370                return true;
 371        }
 372
 373        if (blk_queue_noxmerges(q))
 374                return false;
 375
 376        ret = false;
 377        /*
 378         * See if our hash lookup can find a potential backmerge.
 379         */
 380        while (1) {
 381                __rq = elv_rqhash_find(q, blk_rq_pos(rq));
 382                if (!__rq || !blk_attempt_req_merge(q, __rq, rq))
 383                        break;
 384
 385                list_add(&rq->queuelist, free);
 386                /* The merged request could be merged with others, try again */
 387                ret = true;
 388                rq = __rq;
 389        }
 390
 391        return ret;
 392}
 393
 394void elv_merged_request(struct request_queue *q, struct request *rq,
 395                enum elv_merge type)
 396{
 397        struct elevator_queue *e = q->elevator;
 398
 399        if (e->type->ops.request_merged)
 400                e->type->ops.request_merged(q, rq, type);
 401
 402        if (type == ELEVATOR_BACK_MERGE)
 403                elv_rqhash_reposition(q, rq);
 404
 405        q->last_merge = rq;
 406}
 407
 408void elv_merge_requests(struct request_queue *q, struct request *rq,
 409                             struct request *next)
 410{
 411        struct elevator_queue *e = q->elevator;
 412
 413        if (e->type->ops.requests_merged)
 414                e->type->ops.requests_merged(q, rq, next);
 415
 416        elv_rqhash_reposition(q, rq);
 417        q->last_merge = rq;
 418}
 419
 420struct request *elv_latter_request(struct request_queue *q, struct request *rq)
 421{
 422        struct elevator_queue *e = q->elevator;
 423
 424        if (e->type->ops.next_request)
 425                return e->type->ops.next_request(q, rq);
 426
 427        return NULL;
 428}
 429
 430struct request *elv_former_request(struct request_queue *q, struct request *rq)
 431{
 432        struct elevator_queue *e = q->elevator;
 433
 434        if (e->type->ops.former_request)
 435                return e->type->ops.former_request(q, rq);
 436
 437        return NULL;
 438}
 439
 440#define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
 441
 442static ssize_t
 443elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 444{
 445        struct elv_fs_entry *entry = to_elv(attr);
 446        struct elevator_queue *e;
 447        ssize_t error;
 448
 449        if (!entry->show)
 450                return -EIO;
 451
 452        e = container_of(kobj, struct elevator_queue, kobj);
 453        mutex_lock(&e->sysfs_lock);
 454        error = e->type ? entry->show(e, page) : -ENOENT;
 455        mutex_unlock(&e->sysfs_lock);
 456        return error;
 457}
 458
 459static ssize_t
 460elv_attr_store(struct kobject *kobj, struct attribute *attr,
 461               const char *page, size_t length)
 462{
 463        struct elv_fs_entry *entry = to_elv(attr);
 464        struct elevator_queue *e;
 465        ssize_t error;
 466
 467        if (!entry->store)
 468                return -EIO;
 469
 470        e = container_of(kobj, struct elevator_queue, kobj);
 471        mutex_lock(&e->sysfs_lock);
 472        error = e->type ? entry->store(e, page, length) : -ENOENT;
 473        mutex_unlock(&e->sysfs_lock);
 474        return error;
 475}
 476
 477static const struct sysfs_ops elv_sysfs_ops = {
 478        .show   = elv_attr_show,
 479        .store  = elv_attr_store,
 480};
 481
 482static struct kobj_type elv_ktype = {
 483        .sysfs_ops      = &elv_sysfs_ops,
 484        .release        = elevator_release,
 485};
 486
 487int elv_register_queue(struct request_queue *q, bool uevent)
 488{
 489        struct elevator_queue *e = q->elevator;
 490        int error;
 491
 492        lockdep_assert_held(&q->sysfs_lock);
 493
 494        error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
 495        if (!error) {
 496                struct elv_fs_entry *attr = e->type->elevator_attrs;
 497                if (attr) {
 498                        while (attr->attr.name) {
 499                                if (sysfs_create_file(&e->kobj, &attr->attr))
 500                                        break;
 501                                attr++;
 502                        }
 503                }
 504                if (uevent)
 505                        kobject_uevent(&e->kobj, KOBJ_ADD);
 506
 507                e->registered = 1;
 508        }
 509        return error;
 510}
 511
 512void elv_unregister_queue(struct request_queue *q)
 513{
 514        lockdep_assert_held(&q->sysfs_lock);
 515
 516        if (q) {
 517                struct elevator_queue *e = q->elevator;
 518
 519                kobject_uevent(&e->kobj, KOBJ_REMOVE);
 520                kobject_del(&e->kobj);
 521
 522                e->registered = 0;
 523                /* Re-enable throttling in case elevator disabled it */
 524                wbt_enable_default(q);
 525        }
 526}
 527
 528int elv_register(struct elevator_type *e)
 529{
 530        /* insert_requests and dispatch_request are mandatory */
 531        if (WARN_ON_ONCE(!e->ops.insert_requests || !e->ops.dispatch_request))
 532                return -EINVAL;
 533
 534        /* create icq_cache if requested */
 535        if (e->icq_size) {
 536                if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
 537                    WARN_ON(e->icq_align < __alignof__(struct io_cq)))
 538                        return -EINVAL;
 539
 540                snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
 541                         "%s_io_cq", e->elevator_name);
 542                e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
 543                                                 e->icq_align, 0, NULL);
 544                if (!e->icq_cache)
 545                        return -ENOMEM;
 546        }
 547
 548        /* register, don't allow duplicate names */
 549        spin_lock(&elv_list_lock);
 550        if (elevator_find(e->elevator_name, 0)) {
 551                spin_unlock(&elv_list_lock);
 552                kmem_cache_destroy(e->icq_cache);
 553                return -EBUSY;
 554        }
 555        list_add_tail(&e->list, &elv_list);
 556        spin_unlock(&elv_list_lock);
 557
 558        printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);
 559
 560        return 0;
 561}
 562EXPORT_SYMBOL_GPL(elv_register);
 563
 564void elv_unregister(struct elevator_type *e)
 565{
 566        /* unregister */
 567        spin_lock(&elv_list_lock);
 568        list_del_init(&e->list);
 569        spin_unlock(&elv_list_lock);
 570
 571        /*
 572         * Destroy icq_cache if it exists.  icq's are RCU managed.  Make
 573         * sure all RCU operations are complete before proceeding.
 574         */
 575        if (e->icq_cache) {
 576                rcu_barrier();
 577                kmem_cache_destroy(e->icq_cache);
 578                e->icq_cache = NULL;
 579        }
 580}
 581EXPORT_SYMBOL_GPL(elv_unregister);
 582
 583int elevator_switch_mq(struct request_queue *q,
 584                              struct elevator_type *new_e)
 585{
 586        int ret;
 587
 588        lockdep_assert_held(&q->sysfs_lock);
 589
 590        if (q->elevator) {
 591                if (q->elevator->registered)
 592                        elv_unregister_queue(q);
 593
 594                ioc_clear_queue(q);
 595                elevator_exit(q, q->elevator);
 596        }
 597
 598        ret = blk_mq_init_sched(q, new_e);
 599        if (ret)
 600                goto out;
 601
 602        if (new_e) {
 603                ret = elv_register_queue(q, true);
 604                if (ret) {
 605                        elevator_exit(q, q->elevator);
 606                        goto out;
 607                }
 608        }
 609
 610        if (new_e)
 611                blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
 612        else
 613                blk_add_trace_msg(q, "elv switch: none");
 614
 615out:
 616        return ret;
 617}
 618
 619static inline bool elv_support_iosched(struct request_queue *q)
 620{
 621        if (!queue_is_mq(q) ||
 622            (q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))
 623                return false;
 624        return true;
 625}
 626
 627/*
 628 * For single queue devices, default to using mq-deadline. If we have multiple
 629 * queues or mq-deadline is not available, default to "none".
 630 */
 631static struct elevator_type *elevator_get_default(struct request_queue *q)
 632{
 633        if (q->nr_hw_queues != 1 &&
 634                        !blk_mq_is_sbitmap_shared(q->tag_set->flags))
 635                return NULL;
 636
 637        return elevator_get(q, "mq-deadline", false);
 638}
 639
 640/*
 641 * Get the first elevator providing the features required by the request queue.
 642 * Default to "none" if no matching elevator is found.
 643 */
 644static struct elevator_type *elevator_get_by_features(struct request_queue *q)
 645{
 646        struct elevator_type *e, *found = NULL;
 647
 648        spin_lock(&elv_list_lock);
 649
 650        list_for_each_entry(e, &elv_list, list) {
 651                if (elv_support_features(e->elevator_features,
 652                                         q->required_elevator_features)) {
 653                        found = e;
 654                        break;
 655                }
 656        }
 657
 658        if (found && !try_module_get(found->elevator_owner))
 659                found = NULL;
 660
 661        spin_unlock(&elv_list_lock);
 662        return found;
 663}
 664
 665/*
 666 * For a device queue that has no required features, use the default elevator
 667 * settings. Otherwise, use the first elevator available matching the required
 668 * features. If no suitable elevator is find or if the chosen elevator
 669 * initialization fails, fall back to the "none" elevator (no elevator).
 670 */
 671void elevator_init_mq(struct request_queue *q)
 672{
 673        struct elevator_type *e;
 674        int err;
 675
 676        if (!elv_support_iosched(q))
 677                return;
 678
 679        WARN_ON_ONCE(blk_queue_registered(q));
 680
 681        if (unlikely(q->elevator))
 682                return;
 683
 684        if (!q->required_elevator_features)
 685                e = elevator_get_default(q);
 686        else
 687                e = elevator_get_by_features(q);
 688        if (!e)
 689                return;
 690
 691        blk_mq_freeze_queue(q);
 692        blk_mq_quiesce_queue(q);
 693
 694        err = blk_mq_init_sched(q, e);
 695
 696        blk_mq_unquiesce_queue(q);
 697        blk_mq_unfreeze_queue(q);
 698
 699        if (err) {
 700                pr_warn("\"%s\" elevator initialization failed, "
 701                        "falling back to \"none\"\n", e->elevator_name);
 702                elevator_put(e);
 703        }
 704}
 705EXPORT_SYMBOL_GPL(elevator_init_mq); /* only for dm-rq */
 706
 707/*
 708 * switch to new_e io scheduler. be careful not to introduce deadlocks -
 709 * we don't free the old io scheduler, before we have allocated what we
 710 * need for the new one. this way we have a chance of going back to the old
 711 * one, if the new one fails init for some reason.
 712 */
 713static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
 714{
 715        int err;
 716
 717        lockdep_assert_held(&q->sysfs_lock);
 718
 719        blk_mq_freeze_queue(q);
 720        blk_mq_quiesce_queue(q);
 721
 722        err = elevator_switch_mq(q, new_e);
 723
 724        blk_mq_unquiesce_queue(q);
 725        blk_mq_unfreeze_queue(q);
 726
 727        return err;
 728}
 729
 730/*
 731 * Switch this queue to the given IO scheduler.
 732 */
 733static int __elevator_change(struct request_queue *q, const char *name)
 734{
 735        char elevator_name[ELV_NAME_MAX];
 736        struct elevator_type *e;
 737
 738        /* Make sure queue is not in the middle of being removed */
 739        if (!blk_queue_registered(q))
 740                return -ENOENT;
 741
 742        /*
 743         * Special case for mq, turn off scheduling
 744         */
 745        if (!strncmp(name, "none", 4)) {
 746                if (!q->elevator)
 747                        return 0;
 748                return elevator_switch(q, NULL);
 749        }
 750
 751        strlcpy(elevator_name, name, sizeof(elevator_name));
 752        e = elevator_get(q, strstrip(elevator_name), true);
 753        if (!e)
 754                return -EINVAL;
 755
 756        if (q->elevator &&
 757            elevator_match(q->elevator->type, elevator_name, 0)) {
 758                elevator_put(e);
 759                return 0;
 760        }
 761
 762        return elevator_switch(q, e);
 763}
 764
 765ssize_t elv_iosched_store(struct request_queue *q, const char *name,
 766                          size_t count)
 767{
 768        int ret;
 769
 770        if (!elv_support_iosched(q))
 771                return count;
 772
 773        ret = __elevator_change(q, name);
 774        if (!ret)
 775                return count;
 776
 777        return ret;
 778}
 779
 780ssize_t elv_iosched_show(struct request_queue *q, char *name)
 781{
 782        struct elevator_queue *e = q->elevator;
 783        struct elevator_type *elv = NULL;
 784        struct elevator_type *__e;
 785        int len = 0;
 786
 787        if (!queue_is_mq(q))
 788                return sprintf(name, "none\n");
 789
 790        if (!q->elevator)
 791                len += sprintf(name+len, "[none] ");
 792        else
 793                elv = e->type;
 794
 795        spin_lock(&elv_list_lock);
 796        list_for_each_entry(__e, &elv_list, list) {
 797                if (elv && elevator_match(elv, __e->elevator_name, 0)) {
 798                        len += sprintf(name+len, "[%s] ", elv->elevator_name);
 799                        continue;
 800                }
 801                if (elv_support_iosched(q) &&
 802                    elevator_match(__e, __e->elevator_name,
 803                                   q->required_elevator_features))
 804                        len += sprintf(name+len, "%s ", __e->elevator_name);
 805        }
 806        spin_unlock(&elv_list_lock);
 807
 808        if (q->elevator)
 809                len += sprintf(name+len, "none");
 810
 811        len += sprintf(len+name, "\n");
 812        return len;
 813}
 814
 815struct request *elv_rb_former_request(struct request_queue *q,
 816                                      struct request *rq)
 817{
 818        struct rb_node *rbprev = rb_prev(&rq->rb_node);
 819
 820        if (rbprev)
 821                return rb_entry_rq(rbprev);
 822
 823        return NULL;
 824}
 825EXPORT_SYMBOL(elv_rb_former_request);
 826
 827struct request *elv_rb_latter_request(struct request_queue *q,
 828                                      struct request *rq)
 829{
 830        struct rb_node *rbnext = rb_next(&rq->rb_node);
 831
 832        if (rbnext)
 833                return rb_entry_rq(rbnext);
 834
 835        return NULL;
 836}
 837EXPORT_SYMBOL(elv_rb_latter_request);
 838
 839static int __init elevator_setup(char *str)
 840{
 841        pr_warn("Kernel parameter elevator= does not have any effect anymore.\n"
 842                "Please use sysfs to set IO scheduler for individual devices.\n");
 843        return 1;
 844}
 845
 846__setup("elevator=", elevator_setup);
 847