linux/block/blk-iopoll.c
<<
>>
Prefs
   1/*
   2 * Functions related to interrupt-poll handling in the block layer. This
   3 * is similar to NAPI for network devices.
   4 */
   5#include <linux/kernel.h>
   6#include <linux/module.h>
   7#include <linux/init.h>
   8#include <linux/bio.h>
   9#include <linux/blkdev.h>
  10#include <linux/interrupt.h>
  11#include <linux/cpu.h>
  12#include <linux/blk-iopoll.h>
  13#include <linux/delay.h>
  14
  15#include "blk.h"
  16
  17int blk_iopoll_enabled = 1;
  18EXPORT_SYMBOL(blk_iopoll_enabled);
  19
  20static unsigned int blk_iopoll_budget __read_mostly = 256;
  21
  22static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll);
  23
  24/**
  25 * blk_iopoll_sched - Schedule a run of the iopoll handler
  26 * @iop:      The parent iopoll structure
  27 *
  28 * Description:
  29 *     Add this blk_iopoll structure to the pending poll list and trigger the
  30 *     raise of the blk iopoll softirq. The driver must already have gotten a
  31 *     successful return from blk_iopoll_sched_prep() before calling this.
  32 **/
  33void blk_iopoll_sched(struct blk_iopoll *iop)
  34{
  35        unsigned long flags;
  36
  37        local_irq_save(flags);
  38        list_add_tail(&iop->list, &__get_cpu_var(blk_cpu_iopoll));
  39        __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
  40        local_irq_restore(flags);
  41}
  42EXPORT_SYMBOL(blk_iopoll_sched);
  43
  44/**
  45 * __blk_iopoll_complete - Mark this @iop as un-polled again
  46 * @iop:      The parent iopoll structure
  47 *
  48 * Description:
  49 *     See blk_iopoll_complete(). This function must be called with interrupts
  50 *     disabled.
  51 **/
  52void __blk_iopoll_complete(struct blk_iopoll *iop)
  53{
  54        list_del(&iop->list);
  55        smp_mb__before_clear_bit();
  56        clear_bit_unlock(IOPOLL_F_SCHED, &iop->state);
  57}
  58EXPORT_SYMBOL(__blk_iopoll_complete);
  59
  60/**
  61 * blk_iopoll_complete - Mark this @iop as un-polled again
  62 * @iop:      The parent iopoll structure
  63 *
  64 * Description:
  65 *     If a driver consumes less than the assigned budget in its run of the
  66 *     iopoll handler, it'll end the polled mode by calling this function. The
  67 *     iopoll handler will not be invoked again before blk_iopoll_sched_prep()
  68 *     is called.
  69 **/
  70void blk_iopoll_complete(struct blk_iopoll *iopoll)
  71{
  72        unsigned long flags;
  73
  74        local_irq_save(flags);
  75        __blk_iopoll_complete(iopoll);
  76        local_irq_restore(flags);
  77}
  78EXPORT_SYMBOL(blk_iopoll_complete);
  79
  80static void blk_iopoll_softirq(struct softirq_action *h)
  81{
  82        struct list_head *list = &__get_cpu_var(blk_cpu_iopoll);
  83        int rearm = 0, budget = blk_iopoll_budget;
  84        unsigned long start_time = jiffies;
  85
  86        local_irq_disable();
  87
  88        while (!list_empty(list)) {
  89                struct blk_iopoll *iop;
  90                int work, weight;
  91
  92                /*
  93                 * If softirq window is exhausted then punt.
  94                 */
  95                if (budget <= 0 || time_after(jiffies, start_time)) {
  96                        rearm = 1;
  97                        break;
  98                }
  99
 100                local_irq_enable();
 101
 102                /* Even though interrupts have been re-enabled, this
 103                 * access is safe because interrupts can only add new
 104                 * entries to the tail of this list, and only ->poll()
 105                 * calls can remove this head entry from the list.
 106                 */
 107                iop = list_entry(list->next, struct blk_iopoll, list);
 108
 109                weight = iop->weight;
 110                work = 0;
 111                if (test_bit(IOPOLL_F_SCHED, &iop->state))
 112                        work = iop->poll(iop, weight);
 113
 114                budget -= work;
 115
 116                local_irq_disable();
 117
 118                /*
 119                 * Drivers must not modify the iopoll state, if they
 120                 * consume their assigned weight (or more, some drivers can't
 121                 * easily just stop processing, they have to complete an
 122                 * entire mask of commands).In such cases this code
 123                 * still "owns" the iopoll instance and therefore can
 124                 * move the instance around on the list at-will.
 125                 */
 126                if (work >= weight) {
 127                        if (blk_iopoll_disable_pending(iop))
 128                                __blk_iopoll_complete(iop);
 129                        else
 130                                list_move_tail(&iop->list, list);
 131                }
 132        }
 133
 134        if (rearm)
 135                __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
 136
 137        local_irq_enable();
 138}
 139
 140/**
 141 * blk_iopoll_disable - Disable iopoll on this @iop
 142 * @iop:      The parent iopoll structure
 143 *
 144 * Description:
 145 *     Disable io polling and wait for any pending callbacks to have completed.
 146 **/
 147void blk_iopoll_disable(struct blk_iopoll *iop)
 148{
 149        set_bit(IOPOLL_F_DISABLE, &iop->state);
 150        while (test_and_set_bit(IOPOLL_F_SCHED, &iop->state))
 151                msleep(1);
 152        clear_bit(IOPOLL_F_DISABLE, &iop->state);
 153}
 154EXPORT_SYMBOL(blk_iopoll_disable);
 155
 156/**
 157 * blk_iopoll_enable - Enable iopoll on this @iop
 158 * @iop:      The parent iopoll structure
 159 *
 160 * Description:
 161 *     Enable iopoll on this @iop. Note that the handler run will not be
 162 *     scheduled, it will only mark it as active.
 163 **/
 164void blk_iopoll_enable(struct blk_iopoll *iop)
 165{
 166        BUG_ON(!test_bit(IOPOLL_F_SCHED, &iop->state));
 167        smp_mb__before_clear_bit();
 168        clear_bit_unlock(IOPOLL_F_SCHED, &iop->state);
 169}
 170EXPORT_SYMBOL(blk_iopoll_enable);
 171
 172/**
 173 * blk_iopoll_init - Initialize this @iop
 174 * @iop:      The parent iopoll structure
 175 * @weight:   The default weight (or command completion budget)
 176 * @poll_fn:  The handler to invoke
 177 *
 178 * Description:
 179 *     Initialize this blk_iopoll structure. Before being actively used, the
 180 *     driver must call blk_iopoll_enable().
 181 **/
 182void blk_iopoll_init(struct blk_iopoll *iop, int weight, blk_iopoll_fn *poll_fn)
 183{
 184        memset(iop, 0, sizeof(*iop));
 185        INIT_LIST_HEAD(&iop->list);
 186        iop->weight = weight;
 187        iop->poll = poll_fn;
 188        set_bit(IOPOLL_F_SCHED, &iop->state);
 189}
 190EXPORT_SYMBOL(blk_iopoll_init);
 191
 192static int blk_iopoll_cpu_notify(struct notifier_block *self,
 193                                 unsigned long action, void *hcpu)
 194{
 195        /*
 196         * If a CPU goes away, splice its entries to the current CPU
 197         * and trigger a run of the softirq
 198         */
 199        if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 200                int cpu = (unsigned long) hcpu;
 201
 202                local_irq_disable();
 203                list_splice_init(&per_cpu(blk_cpu_iopoll, cpu),
 204                                 &__get_cpu_var(blk_cpu_iopoll));
 205                __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
 206                local_irq_enable();
 207        }
 208
 209        return NOTIFY_OK;
 210}
 211
 212static struct notifier_block blk_iopoll_cpu_notifier = {
 213        .notifier_call  = blk_iopoll_cpu_notify,
 214};
 215
 216static __init int blk_iopoll_setup(void)
 217{
 218        int i;
 219
 220        for_each_possible_cpu(i)
 221                INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i));
 222
 223        open_softirq(BLOCK_IOPOLL_SOFTIRQ, blk_iopoll_softirq);
 224        register_hotcpu_notifier(&blk_iopoll_cpu_notifier);
 225        return 0;
 226}
 227subsys_initcall(blk_iopoll_setup);
 228