linux/kernel/softirq.c
<<
>>
Prefs
   1/*
   2 *      linux/kernel/softirq.c
   3 *
   4 *      Copyright (C) 1992 Linus Torvalds
   5 *
   6 *      Distribute under GPLv2.
   7 *
   8 *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9 *
  10 *      Remote softirq infrastructure is by Jens Axboe.
  11 */
  12
  13#include <linux/export.h>
  14#include <linux/kernel_stat.h>
  15#include <linux/interrupt.h>
  16#include <linux/init.h>
  17#include <linux/mm.h>
  18#include <linux/notifier.h>
  19#include <linux/percpu.h>
  20#include <linux/cpu.h>
  21#include <linux/freezer.h>
  22#include <linux/kthread.h>
  23#include <linux/rcupdate.h>
  24#include <linux/ftrace.h>
  25#include <linux/smp.h>
  26#include <linux/smpboot.h>
  27#include <linux/tick.h>
  28
  29#define CREATE_TRACE_POINTS
  30#include <trace/events/irq.h>
  31
  32#include <asm/irq.h>
  33/*
  34   - No shared variables, all the data are CPU local.
  35   - If a softirq needs serialization, let it serialize itself
  36     by its own spinlocks.
  37   - Even if softirq is serialized, only local cpu is marked for
  38     execution. Hence, we get something sort of weak cpu binding.
  39     Though it is still not clear, will it result in better locality
  40     or will not.
  41
  42   Examples:
  43   - NET RX softirq. It is multithreaded and does not require
  44     any global serialization.
  45   - NET TX softirq. It kicks software netdevice queues, hence
  46     it is logically serialized per device, but this serialization
  47     is invisible to common code.
  48   - Tasklets: serialized wrt itself.
  49 */
  50
  51#ifndef __ARCH_IRQ_STAT
  52irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53EXPORT_SYMBOL(irq_stat);
  54#endif
  55
  56static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60char *softirq_to_name[NR_SOFTIRQS] = {
  61        "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62        "TASKLET", "SCHED", "HRTIMER", "RCU"
  63};
  64
  65/*
  66 * we cannot loop indefinitely here to avoid userspace starvation,
  67 * but we also don't want to introduce a worst case 1/HZ latency
  68 * to the pending events, so lets the scheduler to balance
  69 * the softirq load for us.
  70 */
  71static void wakeup_softirqd(void)
  72{
  73        /* Interrupts are disabled: no need to stop preemption */
  74        struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76        if (tsk && tsk->state != TASK_RUNNING)
  77                wake_up_process(tsk);
  78}
  79
  80/*
  81 * preempt_count and SOFTIRQ_OFFSET usage:
  82 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83 *   softirq processing.
  84 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85 *   on local_bh_disable or local_bh_enable.
  86 * This lets us distinguish between whether we are currently processing
  87 * softirq and whether we just have bh disabled.
  88 */
  89
  90/*
  91 * This one is for softirq.c-internal use,
  92 * where hardirqs are disabled legitimately:
  93 */
  94#ifdef CONFIG_TRACE_IRQFLAGS
  95static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96{
  97        unsigned long flags;
  98
  99        WARN_ON_ONCE(in_irq());
 100
 101        raw_local_irq_save(flags);
 102        /*
 103         * The preempt tracer hooks into add_preempt_count and will break
 104         * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105         * is set and before current->softirq_enabled is cleared.
 106         * We must manually increment preempt_count here and manually
 107         * call the trace_preempt_off later.
 108         */
 109        preempt_count() += cnt;
 110        /*
 111         * Were softirqs turned off above:
 112         */
 113        if (softirq_count() == cnt)
 114                trace_softirqs_off(ip);
 115        raw_local_irq_restore(flags);
 116
 117        if (preempt_count() == cnt)
 118                trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119}
 120#else /* !CONFIG_TRACE_IRQFLAGS */
 121static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122{
 123        add_preempt_count(cnt);
 124        barrier();
 125}
 126#endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128void local_bh_disable(void)
 129{
 130        __local_bh_disable((unsigned long)__builtin_return_address(0),
 131                                SOFTIRQ_DISABLE_OFFSET);
 132}
 133
 134EXPORT_SYMBOL(local_bh_disable);
 135
 136static void __local_bh_enable(unsigned int cnt)
 137{
 138        WARN_ON_ONCE(in_irq());
 139        WARN_ON_ONCE(!irqs_disabled());
 140
 141        if (softirq_count() == cnt)
 142                trace_softirqs_on((unsigned long)__builtin_return_address(0));
 143        sub_preempt_count(cnt);
 144}
 145
 146/*
 147 * Special-case - softirqs can safely be enabled in
 148 * cond_resched_softirq(), or by __do_softirq(),
 149 * without processing still-pending softirqs:
 150 */
 151void _local_bh_enable(void)
 152{
 153        __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 154}
 155
 156EXPORT_SYMBOL(_local_bh_enable);
 157
 158static inline void _local_bh_enable_ip(unsigned long ip)
 159{
 160        WARN_ON_ONCE(in_irq() || irqs_disabled());
 161#ifdef CONFIG_TRACE_IRQFLAGS
 162        local_irq_disable();
 163#endif
 164        /*
 165         * Are softirqs going to be turned on now:
 166         */
 167        if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 168                trace_softirqs_on(ip);
 169        /*
 170         * Keep preemption disabled until we are done with
 171         * softirq processing:
 172         */
 173        sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 174
 175        if (unlikely(!in_interrupt() && local_softirq_pending()))
 176                do_softirq();
 177
 178        dec_preempt_count();
 179#ifdef CONFIG_TRACE_IRQFLAGS
 180        local_irq_enable();
 181#endif
 182        preempt_check_resched();
 183}
 184
 185void local_bh_enable(void)
 186{
 187        _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 188}
 189EXPORT_SYMBOL(local_bh_enable);
 190
 191void local_bh_enable_ip(unsigned long ip)
 192{
 193        _local_bh_enable_ip(ip);
 194}
 195EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197/*
 198 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 199 * but break the loop if need_resched() is set or after 2 ms.
 200 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 201 * certain cases, such as stop_machine(), jiffies may cease to
 202 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 203 * well to make sure we eventually return from this method.
 204 *
 205 * These limits have been established via experimentation.
 206 * The two things to balance is latency against fairness -
 207 * we want to handle softirqs as soon as possible, but they
 208 * should not be able to lock up the box.
 209 */
 210#define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 211#define MAX_SOFTIRQ_RESTART 10
 212
 213asmlinkage void __do_softirq(void)
 214{
 215        struct softirq_action *h;
 216        __u32 pending;
 217        unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 218        int cpu;
 219        unsigned long old_flags = current->flags;
 220        int max_restart = MAX_SOFTIRQ_RESTART;
 221
 222        /*
 223         * Mask out PF_MEMALLOC s current task context is borrowed for the
 224         * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 225         * again if the socket is related to swap
 226         */
 227        current->flags &= ~PF_MEMALLOC;
 228
 229        pending = local_softirq_pending();
 230        account_irq_enter_time(current);
 231
 232        __local_bh_disable((unsigned long)__builtin_return_address(0),
 233                                SOFTIRQ_OFFSET);
 234        lockdep_softirq_enter();
 235
 236        cpu = smp_processor_id();
 237restart:
 238        /* Reset the pending bitmask before enabling irqs */
 239        set_softirq_pending(0);
 240
 241        local_irq_enable();
 242
 243        h = softirq_vec;
 244
 245        do {
 246                if (pending & 1) {
 247                        unsigned int vec_nr = h - softirq_vec;
 248                        int prev_count = preempt_count();
 249
 250                        kstat_incr_softirqs_this_cpu(vec_nr);
 251
 252                        trace_softirq_entry(vec_nr);
 253                        h->action(h);
 254                        trace_softirq_exit(vec_nr);
 255                        if (unlikely(prev_count != preempt_count())) {
 256                                printk(KERN_ERR "huh, entered softirq %u %s %p"
 257                                       "with preempt_count %08x,"
 258                                       " exited with %08x?\n", vec_nr,
 259                                       softirq_to_name[vec_nr], h->action,
 260                                       prev_count, preempt_count());
 261                                preempt_count() = prev_count;
 262                        }
 263
 264                        rcu_bh_qs(cpu);
 265                }
 266                h++;
 267                pending >>= 1;
 268        } while (pending);
 269
 270        local_irq_disable();
 271
 272        pending = local_softirq_pending();
 273        if (pending) {
 274                if (time_before(jiffies, end) && !need_resched() &&
 275                    --max_restart)
 276                        goto restart;
 277
 278                wakeup_softirqd();
 279        }
 280
 281        lockdep_softirq_exit();
 282
 283        account_irq_exit_time(current);
 284        __local_bh_enable(SOFTIRQ_OFFSET);
 285        tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 286}
 287
 288#ifndef __ARCH_HAS_DO_SOFTIRQ
 289
 290asmlinkage void do_softirq(void)
 291{
 292        __u32 pending;
 293        unsigned long flags;
 294
 295        if (in_interrupt())
 296                return;
 297
 298        local_irq_save(flags);
 299
 300        pending = local_softirq_pending();
 301
 302        if (pending)
 303                __do_softirq();
 304
 305        local_irq_restore(flags);
 306}
 307
 308#endif
 309
 310/*
 311 * Enter an interrupt context.
 312 */
 313void irq_enter(void)
 314{
 315        int cpu = smp_processor_id();
 316
 317        rcu_irq_enter();
 318        if (is_idle_task(current) && !in_interrupt()) {
 319                /*
 320                 * Prevent raise_softirq from needlessly waking up ksoftirqd
 321                 * here, as softirq will be serviced on return from interrupt.
 322                 */
 323                local_bh_disable();
 324                tick_check_idle(cpu);
 325                _local_bh_enable();
 326        }
 327
 328        __irq_enter();
 329}
 330
 331static inline void invoke_softirq(void)
 332{
 333        if (!force_irqthreads)
 334                __do_softirq();
 335        else
 336                wakeup_softirqd();
 337}
 338
 339/*
 340 * Exit an interrupt context. Process softirqs if needed and possible:
 341 */
 342void irq_exit(void)
 343{
 344#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 345        local_irq_disable();
 346#else
 347        WARN_ON_ONCE(!irqs_disabled());
 348#endif
 349
 350        account_irq_exit_time(current);
 351        trace_hardirq_exit();
 352        sub_preempt_count(HARDIRQ_OFFSET);
 353        if (!in_interrupt() && local_softirq_pending())
 354                invoke_softirq();
 355
 356#ifdef CONFIG_NO_HZ
 357        /* Make sure that timer wheel updates are propagated */
 358        if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 359                tick_nohz_irq_exit();
 360#endif
 361        rcu_irq_exit();
 362}
 363
 364/*
 365 * This function must run with irqs disabled!
 366 */
 367inline void raise_softirq_irqoff(unsigned int nr)
 368{
 369        __raise_softirq_irqoff(nr);
 370
 371        /*
 372         * If we're in an interrupt or softirq, we're done
 373         * (this also catches softirq-disabled code). We will
 374         * actually run the softirq once we return from
 375         * the irq or softirq.
 376         *
 377         * Otherwise we wake up ksoftirqd to make sure we
 378         * schedule the softirq soon.
 379         */
 380        if (!in_interrupt())
 381                wakeup_softirqd();
 382}
 383
 384void raise_softirq(unsigned int nr)
 385{
 386        unsigned long flags;
 387
 388        local_irq_save(flags);
 389        raise_softirq_irqoff(nr);
 390        local_irq_restore(flags);
 391}
 392
 393void __raise_softirq_irqoff(unsigned int nr)
 394{
 395        trace_softirq_raise(nr);
 396        or_softirq_pending(1UL << nr);
 397}
 398
 399void open_softirq(int nr, void (*action)(struct softirq_action *))
 400{
 401        softirq_vec[nr].action = action;
 402}
 403
 404/*
 405 * Tasklets
 406 */
 407struct tasklet_head
 408{
 409        struct tasklet_struct *head;
 410        struct tasklet_struct **tail;
 411};
 412
 413static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 414static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 415
 416void __tasklet_schedule(struct tasklet_struct *t)
 417{
 418        unsigned long flags;
 419
 420        local_irq_save(flags);
 421        t->next = NULL;
 422        *__this_cpu_read(tasklet_vec.tail) = t;
 423        __this_cpu_write(tasklet_vec.tail, &(t->next));
 424        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 425        local_irq_restore(flags);
 426}
 427
 428EXPORT_SYMBOL(__tasklet_schedule);
 429
 430void __tasklet_hi_schedule(struct tasklet_struct *t)
 431{
 432        unsigned long flags;
 433
 434        local_irq_save(flags);
 435        t->next = NULL;
 436        *__this_cpu_read(tasklet_hi_vec.tail) = t;
 437        __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 438        raise_softirq_irqoff(HI_SOFTIRQ);
 439        local_irq_restore(flags);
 440}
 441
 442EXPORT_SYMBOL(__tasklet_hi_schedule);
 443
 444void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 445{
 446        BUG_ON(!irqs_disabled());
 447
 448        t->next = __this_cpu_read(tasklet_hi_vec.head);
 449        __this_cpu_write(tasklet_hi_vec.head, t);
 450        __raise_softirq_irqoff(HI_SOFTIRQ);
 451}
 452
 453EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 454
 455static void tasklet_action(struct softirq_action *a)
 456{
 457        struct tasklet_struct *list;
 458
 459        local_irq_disable();
 460        list = __this_cpu_read(tasklet_vec.head);
 461        __this_cpu_write(tasklet_vec.head, NULL);
 462        __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 463        local_irq_enable();
 464
 465        while (list) {
 466                struct tasklet_struct *t = list;
 467
 468                list = list->next;
 469
 470                if (tasklet_trylock(t)) {
 471                        if (!atomic_read(&t->count)) {
 472                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 473                                        BUG();
 474                                t->func(t->data);
 475                                tasklet_unlock(t);
 476                                continue;
 477                        }
 478                        tasklet_unlock(t);
 479                }
 480
 481                local_irq_disable();
 482                t->next = NULL;
 483                *__this_cpu_read(tasklet_vec.tail) = t;
 484                __this_cpu_write(tasklet_vec.tail, &(t->next));
 485                __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 486                local_irq_enable();
 487        }
 488}
 489
 490static void tasklet_hi_action(struct softirq_action *a)
 491{
 492        struct tasklet_struct *list;
 493
 494        local_irq_disable();
 495        list = __this_cpu_read(tasklet_hi_vec.head);
 496        __this_cpu_write(tasklet_hi_vec.head, NULL);
 497        __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 498        local_irq_enable();
 499
 500        while (list) {
 501                struct tasklet_struct *t = list;
 502
 503                list = list->next;
 504
 505                if (tasklet_trylock(t)) {
 506                        if (!atomic_read(&t->count)) {
 507                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 508                                        BUG();
 509                                t->func(t->data);
 510                                tasklet_unlock(t);
 511                                continue;
 512                        }
 513                        tasklet_unlock(t);
 514                }
 515
 516                local_irq_disable();
 517                t->next = NULL;
 518                *__this_cpu_read(tasklet_hi_vec.tail) = t;
 519                __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 520                __raise_softirq_irqoff(HI_SOFTIRQ);
 521                local_irq_enable();
 522        }
 523}
 524
 525
 526void tasklet_init(struct tasklet_struct *t,
 527                  void (*func)(unsigned long), unsigned long data)
 528{
 529        t->next = NULL;
 530        t->state = 0;
 531        atomic_set(&t->count, 0);
 532        t->func = func;
 533        t->data = data;
 534}
 535
 536EXPORT_SYMBOL(tasklet_init);
 537
 538void tasklet_kill(struct tasklet_struct *t)
 539{
 540        if (in_interrupt())
 541                printk("Attempt to kill tasklet from interrupt\n");
 542
 543        while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 544                do {
 545                        yield();
 546                } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 547        }
 548        tasklet_unlock_wait(t);
 549        clear_bit(TASKLET_STATE_SCHED, &t->state);
 550}
 551
 552EXPORT_SYMBOL(tasklet_kill);
 553
 554/*
 555 * tasklet_hrtimer
 556 */
 557
 558/*
 559 * The trampoline is called when the hrtimer expires. It schedules a tasklet
 560 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 561 * hrtimer callback, but from softirq context.
 562 */
 563static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 564{
 565        struct tasklet_hrtimer *ttimer =
 566                container_of(timer, struct tasklet_hrtimer, timer);
 567
 568        tasklet_hi_schedule(&ttimer->tasklet);
 569        return HRTIMER_NORESTART;
 570}
 571
 572/*
 573 * Helper function which calls the hrtimer callback from
 574 * tasklet/softirq context
 575 */
 576static void __tasklet_hrtimer_trampoline(unsigned long data)
 577{
 578        struct tasklet_hrtimer *ttimer = (void *)data;
 579        enum hrtimer_restart restart;
 580
 581        restart = ttimer->function(&ttimer->timer);
 582        if (restart != HRTIMER_NORESTART)
 583                hrtimer_restart(&ttimer->timer);
 584}
 585
 586/**
 587 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 588 * @ttimer:      tasklet_hrtimer which is initialized
 589 * @function:    hrtimer callback function which gets called from softirq context
 590 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 591 * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 592 */
 593void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 594                          enum hrtimer_restart (*function)(struct hrtimer *),
 595                          clockid_t which_clock, enum hrtimer_mode mode)
 596{
 597        hrtimer_init(&ttimer->timer, which_clock, mode);
 598        ttimer->timer.function = __hrtimer_tasklet_trampoline;
 599        tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 600                     (unsigned long)ttimer);
 601        ttimer->function = function;
 602}
 603EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 604
 605/*
 606 * Remote softirq bits
 607 */
 608
 609DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 610EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 611
 612static void __local_trigger(struct call_single_data *cp, int softirq)
 613{
 614        struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 615
 616        list_add_tail(&cp->list, head);
 617
 618        /* Trigger the softirq only if the list was previously empty.  */
 619        if (head->next == &cp->list)
 620                raise_softirq_irqoff(softirq);
 621}
 622
 623#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 624static void remote_softirq_receive(void *data)
 625{
 626        struct call_single_data *cp = data;
 627        unsigned long flags;
 628        int softirq;
 629
 630        softirq = cp->priv;
 631
 632        local_irq_save(flags);
 633        __local_trigger(cp, softirq);
 634        local_irq_restore(flags);
 635}
 636
 637static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 638{
 639        if (cpu_online(cpu)) {
 640                cp->func = remote_softirq_receive;
 641                cp->info = cp;
 642                cp->flags = 0;
 643                cp->priv = softirq;
 644
 645                __smp_call_function_single(cpu, cp, 0);
 646                return 0;
 647        }
 648        return 1;
 649}
 650#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 651static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 652{
 653        return 1;
 654}
 655#endif
 656
 657/**
 658 * __send_remote_softirq - try to schedule softirq work on a remote cpu
 659 * @cp: private SMP call function data area
 660 * @cpu: the remote cpu
 661 * @this_cpu: the currently executing cpu
 662 * @softirq: the softirq for the work
 663 *
 664 * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 665 * done, the work is instead queued up on the local cpu.
 666 *
 667 * Interrupts must be disabled.
 668 */
 669void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 670{
 671        if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 672                __local_trigger(cp, softirq);
 673}
 674EXPORT_SYMBOL(__send_remote_softirq);
 675
 676/**
 677 * send_remote_softirq - try to schedule softirq work on a remote cpu
 678 * @cp: private SMP call function data area
 679 * @cpu: the remote cpu
 680 * @softirq: the softirq for the work
 681 *
 682 * Like __send_remote_softirq except that disabling interrupts and
 683 * computing the current cpu is done for the caller.
 684 */
 685void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 686{
 687        unsigned long flags;
 688        int this_cpu;
 689
 690        local_irq_save(flags);
 691        this_cpu = smp_processor_id();
 692        __send_remote_softirq(cp, cpu, this_cpu, softirq);
 693        local_irq_restore(flags);
 694}
 695EXPORT_SYMBOL(send_remote_softirq);
 696
 697static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 698                                               unsigned long action, void *hcpu)
 699{
 700        /*
 701         * If a CPU goes away, splice its entries to the current CPU
 702         * and trigger a run of the softirq
 703         */
 704        if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 705                int cpu = (unsigned long) hcpu;
 706                int i;
 707
 708                local_irq_disable();
 709                for (i = 0; i < NR_SOFTIRQS; i++) {
 710                        struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 711                        struct list_head *local_head;
 712
 713                        if (list_empty(head))
 714                                continue;
 715
 716                        local_head = &__get_cpu_var(softirq_work_list[i]);
 717                        list_splice_init(head, local_head);
 718                        raise_softirq_irqoff(i);
 719                }
 720                local_irq_enable();
 721        }
 722
 723        return NOTIFY_OK;
 724}
 725
 726static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 727        .notifier_call  = remote_softirq_cpu_notify,
 728};
 729
 730void __init softirq_init(void)
 731{
 732        int cpu;
 733
 734        for_each_possible_cpu(cpu) {
 735                int i;
 736
 737                per_cpu(tasklet_vec, cpu).tail =
 738                        &per_cpu(tasklet_vec, cpu).head;
 739                per_cpu(tasklet_hi_vec, cpu).tail =
 740                        &per_cpu(tasklet_hi_vec, cpu).head;
 741                for (i = 0; i < NR_SOFTIRQS; i++)
 742                        INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 743        }
 744
 745        register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 746
 747        open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 748        open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 749}
 750
 751static int ksoftirqd_should_run(unsigned int cpu)
 752{
 753        return local_softirq_pending();
 754}
 755
 756static void run_ksoftirqd(unsigned int cpu)
 757{
 758        local_irq_disable();
 759        if (local_softirq_pending()) {
 760                __do_softirq();
 761                rcu_note_context_switch(cpu);
 762                local_irq_enable();
 763                cond_resched();
 764                return;
 765        }
 766        local_irq_enable();
 767}
 768
 769#ifdef CONFIG_HOTPLUG_CPU
 770/*
 771 * tasklet_kill_immediate is called to remove a tasklet which can already be
 772 * scheduled for execution on @cpu.
 773 *
 774 * Unlike tasklet_kill, this function removes the tasklet
 775 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 776 *
 777 * When this function is called, @cpu must be in the CPU_DEAD state.
 778 */
 779void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 780{
 781        struct tasklet_struct **i;
 782
 783        BUG_ON(cpu_online(cpu));
 784        BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 785
 786        if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 787                return;
 788
 789        /* CPU is dead, so no lock needed. */
 790        for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 791                if (*i == t) {
 792                        *i = t->next;
 793                        /* If this was the tail element, move the tail ptr */
 794                        if (*i == NULL)
 795                                per_cpu(tasklet_vec, cpu).tail = i;
 796                        return;
 797                }
 798        }
 799        BUG();
 800}
 801
 802static void takeover_tasklets(unsigned int cpu)
 803{
 804        /* CPU is dead, so no lock needed. */
 805        local_irq_disable();
 806
 807        /* Find end, append list for that CPU. */
 808        if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 809                *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 810                this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 811                per_cpu(tasklet_vec, cpu).head = NULL;
 812                per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 813        }
 814        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 815
 816        if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 817                *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 818                __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 819                per_cpu(tasklet_hi_vec, cpu).head = NULL;
 820                per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 821        }
 822        raise_softirq_irqoff(HI_SOFTIRQ);
 823
 824        local_irq_enable();
 825}
 826#endif /* CONFIG_HOTPLUG_CPU */
 827
 828static int __cpuinit cpu_callback(struct notifier_block *nfb,
 829                                  unsigned long action,
 830                                  void *hcpu)
 831{
 832        switch (action) {
 833#ifdef CONFIG_HOTPLUG_CPU
 834        case CPU_DEAD:
 835        case CPU_DEAD_FROZEN:
 836                takeover_tasklets((unsigned long)hcpu);
 837                break;
 838#endif /* CONFIG_HOTPLUG_CPU */
 839        }
 840        return NOTIFY_OK;
 841}
 842
 843static struct notifier_block __cpuinitdata cpu_nfb = {
 844        .notifier_call = cpu_callback
 845};
 846
 847static struct smp_hotplug_thread softirq_threads = {
 848        .store                  = &ksoftirqd,
 849        .thread_should_run      = ksoftirqd_should_run,
 850        .thread_fn              = run_ksoftirqd,
 851        .thread_comm            = "ksoftirqd/%u",
 852};
 853
 854static __init int spawn_ksoftirqd(void)
 855{
 856        register_cpu_notifier(&cpu_nfb);
 857
 858        BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 859
 860        return 0;
 861}
 862early_initcall(spawn_ksoftirqd);
 863
 864/*
 865 * [ These __weak aliases are kept in a separate compilation unit, so that
 866 *   GCC does not inline them incorrectly. ]
 867 */
 868
 869int __init __weak early_irq_init(void)
 870{
 871        return 0;
 872}
 873
 874#ifdef CONFIG_GENERIC_HARDIRQS
 875int __init __weak arch_probe_nr_irqs(void)
 876{
 877        return NR_IRQS_LEGACY;
 878}
 879
 880int __init __weak arch_early_irq_init(void)
 881{
 882        return 0;
 883}
 884#endif
 885
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.