linux/kernel/softirq.c
<<
>>
Prefs
   1/*
   2 *      linux/kernel/softirq.c
   3 *
   4 *      Copyright (C) 1992 Linus Torvalds
   5 *
   6 *      Distribute under GPLv2.
   7 *
   8 *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9 *
  10 *      Remote softirq infrastructure is by Jens Axboe.
  11 */
  12
  13#include <linux/export.h>
  14#include <linux/kernel_stat.h>
  15#include <linux/interrupt.h>
  16#include <linux/init.h>
  17#include <linux/mm.h>
  18#include <linux/notifier.h>
  19#include <linux/percpu.h>
  20#include <linux/cpu.h>
  21#include <linux/freezer.h>
  22#include <linux/kthread.h>
  23#include <linux/rcupdate.h>
  24#include <linux/ftrace.h>
  25#include <linux/smp.h>
  26#include <linux/smpboot.h>
  27#include <linux/tick.h>
  28
  29#define CREATE_TRACE_POINTS
  30#include <trace/events/irq.h>
  31
  32#include <asm/irq.h>
  33/*
  34   - No shared variables, all the data are CPU local.
  35   - If a softirq needs serialization, let it serialize itself
  36     by its own spinlocks.
  37   - Even if softirq is serialized, only local cpu is marked for
  38     execution. Hence, we get something sort of weak cpu binding.
  39     Though it is still not clear, will it result in better locality
  40     or will not.
  41
  42   Examples:
  43   - NET RX softirq. It is multithreaded and does not require
  44     any global serialization.
  45   - NET TX softirq. It kicks software netdevice queues, hence
  46     it is logically serialized per device, but this serialization
  47     is invisible to common code.
  48   - Tasklets: serialized wrt itself.
  49 */
  50
  51#ifndef __ARCH_IRQ_STAT
  52irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53EXPORT_SYMBOL(irq_stat);
  54#endif
  55
  56static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60char *softirq_to_name[NR_SOFTIRQS] = {
  61        "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62        "TASKLET", "SCHED", "HRTIMER", "RCU"
  63};
  64
  65/*
  66 * we cannot loop indefinitely here to avoid userspace starvation,
  67 * but we also don't want to introduce a worst case 1/HZ latency
  68 * to the pending events, so lets the scheduler to balance
  69 * the softirq load for us.
  70 */
  71static void wakeup_softirqd(void)
  72{
  73        /* Interrupts are disabled: no need to stop preemption */
  74        struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76        if (tsk && tsk->state != TASK_RUNNING)
  77                wake_up_process(tsk);
  78}
  79
  80/*
  81 * preempt_count and SOFTIRQ_OFFSET usage:
  82 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83 *   softirq processing.
  84 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85 *   on local_bh_disable or local_bh_enable.
  86 * This lets us distinguish between whether we are currently processing
  87 * softirq and whether we just have bh disabled.
  88 */
  89
  90/*
  91 * This one is for softirq.c-internal use,
  92 * where hardirqs are disabled legitimately:
  93 */
  94#ifdef CONFIG_TRACE_IRQFLAGS
  95static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96{
  97        unsigned long flags;
  98
  99        WARN_ON_ONCE(in_irq());
 100
 101        raw_local_irq_save(flags);
 102        /*
 103         * The preempt tracer hooks into add_preempt_count and will break
 104         * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105         * is set and before current->softirq_enabled is cleared.
 106         * We must manually increment preempt_count here and manually
 107         * call the trace_preempt_off later.
 108         */
 109        preempt_count() += cnt;
 110        /*
 111         * Were softirqs turned off above:
 112         */
 113        if (softirq_count() == cnt)
 114                trace_softirqs_off(ip);
 115        raw_local_irq_restore(flags);
 116
 117        if (preempt_count() == cnt)
 118                trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119}
 120#else /* !CONFIG_TRACE_IRQFLAGS */
 121static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122{
 123        add_preempt_count(cnt);
 124        barrier();
 125}
 126#endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128void local_bh_disable(void)
 129{
 130        __local_bh_disable((unsigned long)__builtin_return_address(0),
 131                                SOFTIRQ_DISABLE_OFFSET);
 132}
 133
 134EXPORT_SYMBOL(local_bh_disable);
 135
 136static void __local_bh_enable(unsigned int cnt)
 137{
 138        WARN_ON_ONCE(in_irq());
 139        WARN_ON_ONCE(!irqs_disabled());
 140
 141        if (softirq_count() == cnt)
 142                trace_softirqs_on((unsigned long)__builtin_return_address(0));
 143        sub_preempt_count(cnt);
 144}
 145
 146/*
 147 * Special-case - softirqs can safely be enabled in
 148 * cond_resched_softirq(), or by __do_softirq(),
 149 * without processing still-pending softirqs:
 150 */
 151void _local_bh_enable(void)
 152{
 153        __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 154}
 155
 156EXPORT_SYMBOL(_local_bh_enable);
 157
 158static inline void _local_bh_enable_ip(unsigned long ip)
 159{
 160        WARN_ON_ONCE(in_irq() || irqs_disabled());
 161#ifdef CONFIG_TRACE_IRQFLAGS
 162        local_irq_disable();
 163#endif
 164        /*
 165         * Are softirqs going to be turned on now:
 166         */
 167        if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 168                trace_softirqs_on(ip);
 169        /*
 170         * Keep preemption disabled until we are done with
 171         * softirq processing:
 172         */
 173        sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 174
 175        if (unlikely(!in_interrupt() && local_softirq_pending()))
 176                do_softirq();
 177
 178        dec_preempt_count();
 179#ifdef CONFIG_TRACE_IRQFLAGS
 180        local_irq_enable();
 181#endif
 182        preempt_check_resched();
 183}
 184
 185void local_bh_enable(void)
 186{
 187        _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 188}
 189EXPORT_SYMBOL(local_bh_enable);
 190
 191void local_bh_enable_ip(unsigned long ip)
 192{
 193        _local_bh_enable_ip(ip);
 194}
 195EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197/*
 198 * We restart softirq processing for at most 2 ms,
 199 * and if need_resched() is not set.
 200 *
 201 * These limits have been established via experimentation.
 202 * The two things to balance is latency against fairness -
 203 * we want to handle softirqs as soon as possible, but they
 204 * should not be able to lock up the box.
 205 */
 206#define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 207
 208asmlinkage void __do_softirq(void)
 209{
 210        struct softirq_action *h;
 211        __u32 pending;
 212        unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 213        int cpu;
 214        unsigned long old_flags = current->flags;
 215
 216        /*
 217         * Mask out PF_MEMALLOC s current task context is borrowed for the
 218         * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 219         * again if the socket is related to swap
 220         */
 221        current->flags &= ~PF_MEMALLOC;
 222
 223        pending = local_softirq_pending();
 224        account_irq_enter_time(current);
 225
 226        __local_bh_disable((unsigned long)__builtin_return_address(0),
 227                                SOFTIRQ_OFFSET);
 228        lockdep_softirq_enter();
 229
 230        cpu = smp_processor_id();
 231restart:
 232        /* Reset the pending bitmask before enabling irqs */
 233        set_softirq_pending(0);
 234
 235        local_irq_enable();
 236
 237        h = softirq_vec;
 238
 239        do {
 240                if (pending & 1) {
 241                        unsigned int vec_nr = h - softirq_vec;
 242                        int prev_count = preempt_count();
 243
 244                        kstat_incr_softirqs_this_cpu(vec_nr);
 245
 246                        trace_softirq_entry(vec_nr);
 247                        h->action(h);
 248                        trace_softirq_exit(vec_nr);
 249                        if (unlikely(prev_count != preempt_count())) {
 250                                printk(KERN_ERR "huh, entered softirq %u %s %p"
 251                                       "with preempt_count %08x,"
 252                                       " exited with %08x?\n", vec_nr,
 253                                       softirq_to_name[vec_nr], h->action,
 254                                       prev_count, preempt_count());
 255                                preempt_count() = prev_count;
 256                        }
 257
 258                        rcu_bh_qs(cpu);
 259                }
 260                h++;
 261                pending >>= 1;
 262        } while (pending);
 263
 264        local_irq_disable();
 265
 266        pending = local_softirq_pending();
 267        if (pending) {
 268                if (time_before(jiffies, end) && !need_resched())
 269                        goto restart;
 270
 271                wakeup_softirqd();
 272        }
 273
 274        lockdep_softirq_exit();
 275
 276        account_irq_exit_time(current);
 277        __local_bh_enable(SOFTIRQ_OFFSET);
 278        tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 279}
 280
 281#ifndef __ARCH_HAS_DO_SOFTIRQ
 282
 283asmlinkage void do_softirq(void)
 284{
 285        __u32 pending;
 286        unsigned long flags;
 287
 288        if (in_interrupt())
 289                return;
 290
 291        local_irq_save(flags);
 292
 293        pending = local_softirq_pending();
 294
 295        if (pending)
 296                __do_softirq();
 297
 298        local_irq_restore(flags);
 299}
 300
 301#endif
 302
 303/*
 304 * Enter an interrupt context.
 305 */
 306void irq_enter(void)
 307{
 308        int cpu = smp_processor_id();
 309
 310        rcu_irq_enter();
 311        if (is_idle_task(current) && !in_interrupt()) {
 312                /*
 313                 * Prevent raise_softirq from needlessly waking up ksoftirqd
 314                 * here, as softirq will be serviced on return from interrupt.
 315                 */
 316                local_bh_disable();
 317                tick_check_idle(cpu);
 318                _local_bh_enable();
 319        }
 320
 321        __irq_enter();
 322}
 323
 324static inline void invoke_softirq(void)
 325{
 326        if (!force_irqthreads)
 327                __do_softirq();
 328        else
 329                wakeup_softirqd();
 330}
 331
 332/*
 333 * Exit an interrupt context. Process softirqs if needed and possible:
 334 */
 335void irq_exit(void)
 336{
 337#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 338        local_irq_disable();
 339#else
 340        WARN_ON_ONCE(!irqs_disabled());
 341#endif
 342
 343        account_irq_exit_time(current);
 344        trace_hardirq_exit();
 345        sub_preempt_count(HARDIRQ_OFFSET);
 346        if (!in_interrupt() && local_softirq_pending())
 347                invoke_softirq();
 348
 349#ifdef CONFIG_NO_HZ
 350        /* Make sure that timer wheel updates are propagated */
 351        if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 352                tick_nohz_irq_exit();
 353#endif
 354        rcu_irq_exit();
 355}
 356
 357/*
 358 * This function must run with irqs disabled!
 359 */
 360inline void raise_softirq_irqoff(unsigned int nr)
 361{
 362        __raise_softirq_irqoff(nr);
 363
 364        /*
 365         * If we're in an interrupt or softirq, we're done
 366         * (this also catches softirq-disabled code). We will
 367         * actually run the softirq once we return from
 368         * the irq or softirq.
 369         *
 370         * Otherwise we wake up ksoftirqd to make sure we
 371         * schedule the softirq soon.
 372         */
 373        if (!in_interrupt())
 374                wakeup_softirqd();
 375}
 376
 377void raise_softirq(unsigned int nr)
 378{
 379        unsigned long flags;
 380
 381        local_irq_save(flags);
 382        raise_softirq_irqoff(nr);
 383        local_irq_restore(flags);
 384}
 385
 386void __raise_softirq_irqoff(unsigned int nr)
 387{
 388        trace_softirq_raise(nr);
 389        or_softirq_pending(1UL << nr);
 390}
 391
 392void open_softirq(int nr, void (*action)(struct softirq_action *))
 393{
 394        softirq_vec[nr].action = action;
 395}
 396
 397/*
 398 * Tasklets
 399 */
 400struct tasklet_head
 401{
 402        struct tasklet_struct *head;
 403        struct tasklet_struct **tail;
 404};
 405
 406static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 407static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 408
 409void __tasklet_schedule(struct tasklet_struct *t)
 410{
 411        unsigned long flags;
 412
 413        local_irq_save(flags);
 414        t->next = NULL;
 415        *__this_cpu_read(tasklet_vec.tail) = t;
 416        __this_cpu_write(tasklet_vec.tail, &(t->next));
 417        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 418        local_irq_restore(flags);
 419}
 420
 421EXPORT_SYMBOL(__tasklet_schedule);
 422
 423void __tasklet_hi_schedule(struct tasklet_struct *t)
 424{
 425        unsigned long flags;
 426
 427        local_irq_save(flags);
 428        t->next = NULL;
 429        *__this_cpu_read(tasklet_hi_vec.tail) = t;
 430        __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 431        raise_softirq_irqoff(HI_SOFTIRQ);
 432        local_irq_restore(flags);
 433}
 434
 435EXPORT_SYMBOL(__tasklet_hi_schedule);
 436
 437void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 438{
 439        BUG_ON(!irqs_disabled());
 440
 441        t->next = __this_cpu_read(tasklet_hi_vec.head);
 442        __this_cpu_write(tasklet_hi_vec.head, t);
 443        __raise_softirq_irqoff(HI_SOFTIRQ);
 444}
 445
 446EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 447
 448static void tasklet_action(struct softirq_action *a)
 449{
 450        struct tasklet_struct *list;
 451
 452        local_irq_disable();
 453        list = __this_cpu_read(tasklet_vec.head);
 454        __this_cpu_write(tasklet_vec.head, NULL);
 455        __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 456        local_irq_enable();
 457
 458        while (list) {
 459                struct tasklet_struct *t = list;
 460
 461                list = list->next;
 462
 463                if (tasklet_trylock(t)) {
 464                        if (!atomic_read(&t->count)) {
 465                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 466                                        BUG();
 467                                t->func(t->data);
 468                                tasklet_unlock(t);
 469                                continue;
 470                        }
 471                        tasklet_unlock(t);
 472                }
 473
 474                local_irq_disable();
 475                t->next = NULL;
 476                *__this_cpu_read(tasklet_vec.tail) = t;
 477                __this_cpu_write(tasklet_vec.tail, &(t->next));
 478                __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 479                local_irq_enable();
 480        }
 481}
 482
 483static void tasklet_hi_action(struct softirq_action *a)
 484{
 485        struct tasklet_struct *list;
 486
 487        local_irq_disable();
 488        list = __this_cpu_read(tasklet_hi_vec.head);
 489        __this_cpu_write(tasklet_hi_vec.head, NULL);
 490        __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 491        local_irq_enable();
 492
 493        while (list) {
 494                struct tasklet_struct *t = list;
 495
 496                list = list->next;
 497
 498                if (tasklet_trylock(t)) {
 499                        if (!atomic_read(&t->count)) {
 500                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 501                                        BUG();
 502                                t->func(t->data);
 503                                tasklet_unlock(t);
 504                                continue;
 505                        }
 506                        tasklet_unlock(t);
 507                }
 508
 509                local_irq_disable();
 510                t->next = NULL;
 511                *__this_cpu_read(tasklet_hi_vec.tail) = t;
 512                __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 513                __raise_softirq_irqoff(HI_SOFTIRQ);
 514                local_irq_enable();
 515        }
 516}
 517
 518
 519void tasklet_init(struct tasklet_struct *t,
 520                  void (*func)(unsigned long), unsigned long data)
 521{
 522        t->next = NULL;
 523        t->state = 0;
 524        atomic_set(&t->count, 0);
 525        t->func = func;
 526        t->data = data;
 527}
 528
 529EXPORT_SYMBOL(tasklet_init);
 530
 531void tasklet_kill(struct tasklet_struct *t)
 532{
 533        if (in_interrupt())
 534                printk("Attempt to kill tasklet from interrupt\n");
 535
 536        while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 537                do {
 538                        yield();
 539                } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 540        }
 541        tasklet_unlock_wait(t);
 542        clear_bit(TASKLET_STATE_SCHED, &t->state);
 543}
 544
 545EXPORT_SYMBOL(tasklet_kill);
 546
 547/*
 548 * tasklet_hrtimer
 549 */
 550
 551/*
 552 * The trampoline is called when the hrtimer expires. It schedules a tasklet
 553 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 554 * hrtimer callback, but from softirq context.
 555 */
 556static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 557{
 558        struct tasklet_hrtimer *ttimer =
 559                container_of(timer, struct tasklet_hrtimer, timer);
 560
 561        tasklet_hi_schedule(&ttimer->tasklet);
 562        return HRTIMER_NORESTART;
 563}
 564
 565/*
 566 * Helper function which calls the hrtimer callback from
 567 * tasklet/softirq context
 568 */
 569static void __tasklet_hrtimer_trampoline(unsigned long data)
 570{
 571        struct tasklet_hrtimer *ttimer = (void *)data;
 572        enum hrtimer_restart restart;
 573
 574        restart = ttimer->function(&ttimer->timer);
 575        if (restart != HRTIMER_NORESTART)
 576                hrtimer_restart(&ttimer->timer);
 577}
 578
 579/**
 580 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 581 * @ttimer:      tasklet_hrtimer which is initialized
 582 * @function:    hrtimer callback function which gets called from softirq context
 583 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 584 * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 585 */
 586void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 587                          enum hrtimer_restart (*function)(struct hrtimer *),
 588                          clockid_t which_clock, enum hrtimer_mode mode)
 589{
 590        hrtimer_init(&ttimer->timer, which_clock, mode);
 591        ttimer->timer.function = __hrtimer_tasklet_trampoline;
 592        tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 593                     (unsigned long)ttimer);
 594        ttimer->function = function;
 595}
 596EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 597
 598/*
 599 * Remote softirq bits
 600 */
 601
 602DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 603EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 604
 605static void __local_trigger(struct call_single_data *cp, int softirq)
 606{
 607        struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 608
 609        list_add_tail(&cp->list, head);
 610
 611        /* Trigger the softirq only if the list was previously empty.  */
 612        if (head->next == &cp->list)
 613                raise_softirq_irqoff(softirq);
 614}
 615
 616#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 617static void remote_softirq_receive(void *data)
 618{
 619        struct call_single_data *cp = data;
 620        unsigned long flags;
 621        int softirq;
 622
 623        softirq = cp->priv;
 624
 625        local_irq_save(flags);
 626        __local_trigger(cp, softirq);
 627        local_irq_restore(flags);
 628}
 629
 630static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 631{
 632        if (cpu_online(cpu)) {
 633                cp->func = remote_softirq_receive;
 634                cp->info = cp;
 635                cp->flags = 0;
 636                cp->priv = softirq;
 637
 638                __smp_call_function_single(cpu, cp, 0);
 639                return 0;
 640        }
 641        return 1;
 642}
 643#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 644static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 645{
 646        return 1;
 647}
 648#endif
 649
 650/**
 651 * __send_remote_softirq - try to schedule softirq work on a remote cpu
 652 * @cp: private SMP call function data area
 653 * @cpu: the remote cpu
 654 * @this_cpu: the currently executing cpu
 655 * @softirq: the softirq for the work
 656 *
 657 * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 658 * done, the work is instead queued up on the local cpu.
 659 *
 660 * Interrupts must be disabled.
 661 */
 662void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 663{
 664        if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 665                __local_trigger(cp, softirq);
 666}
 667EXPORT_SYMBOL(__send_remote_softirq);
 668
 669/**
 670 * send_remote_softirq - try to schedule softirq work on a remote cpu
 671 * @cp: private SMP call function data area
 672 * @cpu: the remote cpu
 673 * @softirq: the softirq for the work
 674 *
 675 * Like __send_remote_softirq except that disabling interrupts and
 676 * computing the current cpu is done for the caller.
 677 */
 678void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 679{
 680        unsigned long flags;
 681        int this_cpu;
 682
 683        local_irq_save(flags);
 684        this_cpu = smp_processor_id();
 685        __send_remote_softirq(cp, cpu, this_cpu, softirq);
 686        local_irq_restore(flags);
 687}
 688EXPORT_SYMBOL(send_remote_softirq);
 689
 690static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 691                                               unsigned long action, void *hcpu)
 692{
 693        /*
 694         * If a CPU goes away, splice its entries to the current CPU
 695         * and trigger a run of the softirq
 696         */
 697        if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 698                int cpu = (unsigned long) hcpu;
 699                int i;
 700
 701                local_irq_disable();
 702                for (i = 0; i < NR_SOFTIRQS; i++) {
 703                        struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 704                        struct list_head *local_head;
 705
 706                        if (list_empty(head))
 707                                continue;
 708
 709                        local_head = &__get_cpu_var(softirq_work_list[i]);
 710                        list_splice_init(head, local_head);
 711                        raise_softirq_irqoff(i);
 712                }
 713                local_irq_enable();
 714        }
 715
 716        return NOTIFY_OK;
 717}
 718
 719static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 720        .notifier_call  = remote_softirq_cpu_notify,
 721};
 722
 723void __init softirq_init(void)
 724{
 725        int cpu;
 726
 727        for_each_possible_cpu(cpu) {
 728                int i;
 729
 730                per_cpu(tasklet_vec, cpu).tail =
 731                        &per_cpu(tasklet_vec, cpu).head;
 732                per_cpu(tasklet_hi_vec, cpu).tail =
 733                        &per_cpu(tasklet_hi_vec, cpu).head;
 734                for (i = 0; i < NR_SOFTIRQS; i++)
 735                        INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 736        }
 737
 738        register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 739
 740        open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 741        open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 742}
 743
 744static int ksoftirqd_should_run(unsigned int cpu)
 745{
 746        return local_softirq_pending();
 747}
 748
 749static void run_ksoftirqd(unsigned int cpu)
 750{
 751        local_irq_disable();
 752        if (local_softirq_pending()) {
 753                __do_softirq();
 754                rcu_note_context_switch(cpu);
 755                local_irq_enable();
 756                cond_resched();
 757                return;
 758        }
 759        local_irq_enable();
 760}
 761
 762#ifdef CONFIG_HOTPLUG_CPU
 763/*
 764 * tasklet_kill_immediate is called to remove a tasklet which can already be
 765 * scheduled for execution on @cpu.
 766 *
 767 * Unlike tasklet_kill, this function removes the tasklet
 768 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 769 *
 770 * When this function is called, @cpu must be in the CPU_DEAD state.
 771 */
 772void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 773{
 774        struct tasklet_struct **i;
 775
 776        BUG_ON(cpu_online(cpu));
 777        BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 778
 779        if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 780                return;
 781
 782        /* CPU is dead, so no lock needed. */
 783        for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 784                if (*i == t) {
 785                        *i = t->next;
 786                        /* If this was the tail element, move the tail ptr */
 787                        if (*i == NULL)
 788                                per_cpu(tasklet_vec, cpu).tail = i;
 789                        return;
 790                }
 791        }
 792        BUG();
 793}
 794
 795static void takeover_tasklets(unsigned int cpu)
 796{
 797        /* CPU is dead, so no lock needed. */
 798        local_irq_disable();
 799
 800        /* Find end, append list for that CPU. */
 801        if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 802                *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 803                this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 804                per_cpu(tasklet_vec, cpu).head = NULL;
 805                per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 806        }
 807        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 808
 809        if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 810                *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 811                __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 812                per_cpu(tasklet_hi_vec, cpu).head = NULL;
 813                per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 814        }
 815        raise_softirq_irqoff(HI_SOFTIRQ);
 816
 817        local_irq_enable();
 818}
 819#endif /* CONFIG_HOTPLUG_CPU */
 820
 821static int __cpuinit cpu_callback(struct notifier_block *nfb,
 822                                  unsigned long action,
 823                                  void *hcpu)
 824{
 825        switch (action) {
 826#ifdef CONFIG_HOTPLUG_CPU
 827        case CPU_DEAD:
 828        case CPU_DEAD_FROZEN:
 829                takeover_tasklets((unsigned long)hcpu);
 830                break;
 831#endif /* CONFIG_HOTPLUG_CPU */
 832        }
 833        return NOTIFY_OK;
 834}
 835
 836static struct notifier_block __cpuinitdata cpu_nfb = {
 837        .notifier_call = cpu_callback
 838};
 839
 840static struct smp_hotplug_thread softirq_threads = {
 841        .store                  = &ksoftirqd,
 842        .thread_should_run      = ksoftirqd_should_run,
 843        .thread_fn              = run_ksoftirqd,
 844        .thread_comm            = "ksoftirqd/%u",
 845};
 846
 847static __init int spawn_ksoftirqd(void)
 848{
 849        register_cpu_notifier(&cpu_nfb);
 850
 851        BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 852
 853        return 0;
 854}
 855early_initcall(spawn_ksoftirqd);
 856
 857/*
 858 * [ These __weak aliases are kept in a separate compilation unit, so that
 859 *   GCC does not inline them incorrectly. ]
 860 */
 861
 862int __init __weak early_irq_init(void)
 863{
 864        return 0;
 865}
 866
 867#ifdef CONFIG_GENERIC_HARDIRQS
 868int __init __weak arch_probe_nr_irqs(void)
 869{
 870        return NR_IRQS_LEGACY;
 871}
 872
 873int __init __weak arch_early_irq_init(void)
 874{
 875        return 0;
 876}
 877#endif
 878
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.