linux/kernel/softirq.c
<<
>>
Prefs
   1/*
   2 *      linux/kernel/softirq.c
   3 *
   4 *      Copyright (C) 1992 Linus Torvalds
   5 *
   6 *      Distribute under GPLv2.
   7 *
   8 *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9 *
  10 *      Remote softirq infrastructure is by Jens Axboe.
  11 */
  12
  13#include <linux/export.h>
  14#include <linux/kernel_stat.h>
  15#include <linux/interrupt.h>
  16#include <linux/init.h>
  17#include <linux/mm.h>
  18#include <linux/notifier.h>
  19#include <linux/percpu.h>
  20#include <linux/cpu.h>
  21#include <linux/freezer.h>
  22#include <linux/kthread.h>
  23#include <linux/rcupdate.h>
  24#include <linux/ftrace.h>
  25#include <linux/smp.h>
  26#include <linux/smpboot.h>
  27#include <linux/tick.h>
  28
  29#define CREATE_TRACE_POINTS
  30#include <trace/events/irq.h>
  31
  32#include <asm/irq.h>
  33/*
  34   - No shared variables, all the data are CPU local.
  35   - If a softirq needs serialization, let it serialize itself
  36     by its own spinlocks.
  37   - Even if softirq is serialized, only local cpu is marked for
  38     execution. Hence, we get something sort of weak cpu binding.
  39     Though it is still not clear, will it result in better locality
  40     or will not.
  41
  42   Examples:
  43   - NET RX softirq. It is multithreaded and does not require
  44     any global serialization.
  45   - NET TX softirq. It kicks software netdevice queues, hence
  46     it is logically serialized per device, but this serialization
  47     is invisible to common code.
  48   - Tasklets: serialized wrt itself.
  49 */
  50
  51#ifndef __ARCH_IRQ_STAT
  52irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53EXPORT_SYMBOL(irq_stat);
  54#endif
  55
  56static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60char *softirq_to_name[NR_SOFTIRQS] = {
  61        "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62        "TASKLET", "SCHED", "HRTIMER", "RCU"
  63};
  64
  65/*
  66 * we cannot loop indefinitely here to avoid userspace starvation,
  67 * but we also don't want to introduce a worst case 1/HZ latency
  68 * to the pending events, so lets the scheduler to balance
  69 * the softirq load for us.
  70 */
  71static void wakeup_softirqd(void)
  72{
  73        /* Interrupts are disabled: no need to stop preemption */
  74        struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76        if (tsk && tsk->state != TASK_RUNNING)
  77                wake_up_process(tsk);
  78}
  79
  80/*
  81 * preempt_count and SOFTIRQ_OFFSET usage:
  82 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83 *   softirq processing.
  84 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85 *   on local_bh_disable or local_bh_enable.
  86 * This lets us distinguish between whether we are currently processing
  87 * softirq and whether we just have bh disabled.
  88 */
  89
  90/*
  91 * This one is for softirq.c-internal use,
  92 * where hardirqs are disabled legitimately:
  93 */
  94#ifdef CONFIG_TRACE_IRQFLAGS
  95static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96{
  97        unsigned long flags;
  98
  99        WARN_ON_ONCE(in_irq());
 100
 101        raw_local_irq_save(flags);
 102        /*
 103         * The preempt tracer hooks into add_preempt_count and will break
 104         * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105         * is set and before current->softirq_enabled is cleared.
 106         * We must manually increment preempt_count here and manually
 107         * call the trace_preempt_off later.
 108         */
 109        preempt_count() += cnt;
 110        /*
 111         * Were softirqs turned off above:
 112         */
 113        if (softirq_count() == cnt)
 114                trace_softirqs_off(ip);
 115        raw_local_irq_restore(flags);
 116
 117        if (preempt_count() == cnt)
 118                trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119}
 120#else /* !CONFIG_TRACE_IRQFLAGS */
 121static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122{
 123        add_preempt_count(cnt);
 124        barrier();
 125}
 126#endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128void local_bh_disable(void)
 129{
 130        __local_bh_disable((unsigned long)__builtin_return_address(0),
 131                                SOFTIRQ_DISABLE_OFFSET);
 132}
 133
 134EXPORT_SYMBOL(local_bh_disable);
 135
 136static void __local_bh_enable(unsigned int cnt)
 137{
 138        WARN_ON_ONCE(in_irq());
 139        WARN_ON_ONCE(!irqs_disabled());
 140
 141        if (softirq_count() == cnt)
 142                trace_softirqs_on((unsigned long)__builtin_return_address(0));
 143        sub_preempt_count(cnt);
 144}
 145
 146/*
 147 * Special-case - softirqs can safely be enabled in
 148 * cond_resched_softirq(), or by __do_softirq(),
 149 * without processing still-pending softirqs:
 150 */
 151void _local_bh_enable(void)
 152{
 153        __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 154}
 155
 156EXPORT_SYMBOL(_local_bh_enable);
 157
 158static inline void _local_bh_enable_ip(unsigned long ip)
 159{
 160        WARN_ON_ONCE(in_irq() || irqs_disabled());
 161#ifdef CONFIG_TRACE_IRQFLAGS
 162        local_irq_disable();
 163#endif
 164        /*
 165         * Are softirqs going to be turned on now:
 166         */
 167        if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 168                trace_softirqs_on(ip);
 169        /*
 170         * Keep preemption disabled until we are done with
 171         * softirq processing:
 172         */
 173        sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 174
 175        if (unlikely(!in_interrupt() && local_softirq_pending()))
 176                do_softirq();
 177
 178        dec_preempt_count();
 179#ifdef CONFIG_TRACE_IRQFLAGS
 180        local_irq_enable();
 181#endif
 182        preempt_check_resched();
 183}
 184
 185void local_bh_enable(void)
 186{
 187        _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 188}
 189EXPORT_SYMBOL(local_bh_enable);
 190
 191void local_bh_enable_ip(unsigned long ip)
 192{
 193        _local_bh_enable_ip(ip);
 194}
 195EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197/*
 198 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 199 * but break the loop if need_resched() is set or after 2 ms.
 200 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 201 * certain cases, such as stop_machine(), jiffies may cease to
 202 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 203 * well to make sure we eventually return from this method.
 204 *
 205 * These limits have been established via experimentation.
 206 * The two things to balance is latency against fairness -
 207 * we want to handle softirqs as soon as possible, but they
 208 * should not be able to lock up the box.
 209 */
 210#define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 211#define MAX_SOFTIRQ_RESTART 10
 212
 213asmlinkage void __do_softirq(void)
 214{
 215        struct softirq_action *h;
 216        __u32 pending;
 217        unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 218        int cpu;
 219        unsigned long old_flags = current->flags;
 220        int max_restart = MAX_SOFTIRQ_RESTART;
 221
 222        /*
 223         * Mask out PF_MEMALLOC s current task context is borrowed for the
 224         * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 225         * again if the socket is related to swap
 226         */
 227        current->flags &= ~PF_MEMALLOC;
 228
 229        pending = local_softirq_pending();
 230        account_irq_enter_time(current);
 231
 232        __local_bh_disable((unsigned long)__builtin_return_address(0),
 233                                SOFTIRQ_OFFSET);
 234        lockdep_softirq_enter();
 235
 236        cpu = smp_processor_id();
 237restart:
 238        /* Reset the pending bitmask before enabling irqs */
 239        set_softirq_pending(0);
 240
 241        local_irq_enable();
 242
 243        h = softirq_vec;
 244
 245        do {
 246                if (pending & 1) {
 247                        unsigned int vec_nr = h - softirq_vec;
 248                        int prev_count = preempt_count();
 249
 250                        kstat_incr_softirqs_this_cpu(vec_nr);
 251
 252                        trace_softirq_entry(vec_nr);
 253                        h->action(h);
 254                        trace_softirq_exit(vec_nr);
 255                        if (unlikely(prev_count != preempt_count())) {
 256                                printk(KERN_ERR "huh, entered softirq %u %s %p"
 257                                       "with preempt_count %08x,"
 258                                       " exited with %08x?\n", vec_nr,
 259                                       softirq_to_name[vec_nr], h->action,
 260                                       prev_count, preempt_count());
 261                                preempt_count() = prev_count;
 262                        }
 263
 264                        rcu_bh_qs(cpu);
 265                }
 266                h++;
 267                pending >>= 1;
 268        } while (pending);
 269
 270        local_irq_disable();
 271
 272        pending = local_softirq_pending();
 273        if (pending) {
 274                if (time_before(jiffies, end) && !need_resched() &&
 275                    --max_restart)
 276                        goto restart;
 277
 278                wakeup_softirqd();
 279        }
 280
 281        lockdep_softirq_exit();
 282
 283        account_irq_exit_time(current);
 284        __local_bh_enable(SOFTIRQ_OFFSET);
 285        tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 286}
 287
 288#ifndef __ARCH_HAS_DO_SOFTIRQ
 289
 290asmlinkage void do_softirq(void)
 291{
 292        __u32 pending;
 293        unsigned long flags;
 294
 295        if (in_interrupt())
 296                return;
 297
 298        local_irq_save(flags);
 299
 300        pending = local_softirq_pending();
 301
 302        if (pending)
 303                __do_softirq();
 304
 305        local_irq_restore(flags);
 306}
 307
 308#endif
 309
 310/*
 311 * Enter an interrupt context.
 312 */
 313void irq_enter(void)
 314{
 315        int cpu = smp_processor_id();
 316
 317        rcu_irq_enter();
 318        if (is_idle_task(current) && !in_interrupt()) {
 319                /*
 320                 * Prevent raise_softirq from needlessly waking up ksoftirqd
 321                 * here, as softirq will be serviced on return from interrupt.
 322                 */
 323                local_bh_disable();
 324                tick_check_idle(cpu);
 325                _local_bh_enable();
 326        }
 327
 328        __irq_enter();
 329}
 330
 331static inline void invoke_softirq(void)
 332{
 333        if (!force_irqthreads)
 334                __do_softirq();
 335        else
 336                wakeup_softirqd();
 337}
 338
 339static inline void tick_irq_exit(void)
 340{
 341#ifdef CONFIG_NO_HZ_COMMON
 342        int cpu = smp_processor_id();
 343
 344        /* Make sure that timer wheel updates are propagated */
 345        if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 346                if (!in_interrupt())
 347                        tick_nohz_irq_exit();
 348        }
 349#endif
 350}
 351
 352/*
 353 * Exit an interrupt context. Process softirqs if needed and possible:
 354 */
 355void irq_exit(void)
 356{
 357#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 358        local_irq_disable();
 359#else
 360        WARN_ON_ONCE(!irqs_disabled());
 361#endif
 362
 363        account_irq_exit_time(current);
 364        trace_hardirq_exit();
 365        sub_preempt_count(HARDIRQ_OFFSET);
 366        if (!in_interrupt() && local_softirq_pending())
 367                invoke_softirq();
 368
 369        tick_irq_exit();
 370        rcu_irq_exit();
 371}
 372
 373/*
 374 * This function must run with irqs disabled!
 375 */
 376inline void raise_softirq_irqoff(unsigned int nr)
 377{
 378        __raise_softirq_irqoff(nr);
 379
 380        /*
 381         * If we're in an interrupt or softirq, we're done
 382         * (this also catches softirq-disabled code). We will
 383         * actually run the softirq once we return from
 384         * the irq or softirq.
 385         *
 386         * Otherwise we wake up ksoftirqd to make sure we
 387         * schedule the softirq soon.
 388         */
 389        if (!in_interrupt())
 390                wakeup_softirqd();
 391}
 392
 393void raise_softirq(unsigned int nr)
 394{
 395        unsigned long flags;
 396
 397        local_irq_save(flags);
 398        raise_softirq_irqoff(nr);
 399        local_irq_restore(flags);
 400}
 401
 402void __raise_softirq_irqoff(unsigned int nr)
 403{
 404        trace_softirq_raise(nr);
 405        or_softirq_pending(1UL << nr);
 406}
 407
 408void open_softirq(int nr, void (*action)(struct softirq_action *))
 409{
 410        softirq_vec[nr].action = action;
 411}
 412
 413/*
 414 * Tasklets
 415 */
 416struct tasklet_head
 417{
 418        struct tasklet_struct *head;
 419        struct tasklet_struct **tail;
 420};
 421
 422static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 423static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 424
 425void __tasklet_schedule(struct tasklet_struct *t)
 426{
 427        unsigned long flags;
 428
 429        local_irq_save(flags);
 430        t->next = NULL;
 431        *__this_cpu_read(tasklet_vec.tail) = t;
 432        __this_cpu_write(tasklet_vec.tail, &(t->next));
 433        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 434        local_irq_restore(flags);
 435}
 436
 437EXPORT_SYMBOL(__tasklet_schedule);
 438
 439void __tasklet_hi_schedule(struct tasklet_struct *t)
 440{
 441        unsigned long flags;
 442
 443        local_irq_save(flags);
 444        t->next = NULL;
 445        *__this_cpu_read(tasklet_hi_vec.tail) = t;
 446        __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 447        raise_softirq_irqoff(HI_SOFTIRQ);
 448        local_irq_restore(flags);
 449}
 450
 451EXPORT_SYMBOL(__tasklet_hi_schedule);
 452
 453void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 454{
 455        BUG_ON(!irqs_disabled());
 456
 457        t->next = __this_cpu_read(tasklet_hi_vec.head);
 458        __this_cpu_write(tasklet_hi_vec.head, t);
 459        __raise_softirq_irqoff(HI_SOFTIRQ);
 460}
 461
 462EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 463
 464static void tasklet_action(struct softirq_action *a)
 465{
 466        struct tasklet_struct *list;
 467
 468        local_irq_disable();
 469        list = __this_cpu_read(tasklet_vec.head);
 470        __this_cpu_write(tasklet_vec.head, NULL);
 471        __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 472        local_irq_enable();
 473
 474        while (list) {
 475                struct tasklet_struct *t = list;
 476
 477                list = list->next;
 478
 479                if (tasklet_trylock(t)) {
 480                        if (!atomic_read(&t->count)) {
 481                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 482                                        BUG();
 483                                t->func(t->data);
 484                                tasklet_unlock(t);
 485                                continue;
 486                        }
 487                        tasklet_unlock(t);
 488                }
 489
 490                local_irq_disable();
 491                t->next = NULL;
 492                *__this_cpu_read(tasklet_vec.tail) = t;
 493                __this_cpu_write(tasklet_vec.tail, &(t->next));
 494                __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 495                local_irq_enable();
 496        }
 497}
 498
 499static void tasklet_hi_action(struct softirq_action *a)
 500{
 501        struct tasklet_struct *list;
 502
 503        local_irq_disable();
 504        list = __this_cpu_read(tasklet_hi_vec.head);
 505        __this_cpu_write(tasklet_hi_vec.head, NULL);
 506        __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 507        local_irq_enable();
 508
 509        while (list) {
 510                struct tasklet_struct *t = list;
 511
 512                list = list->next;
 513
 514                if (tasklet_trylock(t)) {
 515                        if (!atomic_read(&t->count)) {
 516                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 517                                        BUG();
 518                                t->func(t->data);
 519                                tasklet_unlock(t);
 520                                continue;
 521                        }
 522                        tasklet_unlock(t);
 523                }
 524
 525                local_irq_disable();
 526                t->next = NULL;
 527                *__this_cpu_read(tasklet_hi_vec.tail) = t;
 528                __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 529                __raise_softirq_irqoff(HI_SOFTIRQ);
 530                local_irq_enable();
 531        }
 532}
 533
 534
 535void tasklet_init(struct tasklet_struct *t,
 536                  void (*func)(unsigned long), unsigned long data)
 537{
 538        t->next = NULL;
 539        t->state = 0;
 540        atomic_set(&t->count, 0);
 541        t->func = func;
 542        t->data = data;
 543}
 544
 545EXPORT_SYMBOL(tasklet_init);
 546
 547void tasklet_kill(struct tasklet_struct *t)
 548{
 549        if (in_interrupt())
 550                printk("Attempt to kill tasklet from interrupt\n");
 551
 552        while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 553                do {
 554                        yield();
 555                } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 556        }
 557        tasklet_unlock_wait(t);
 558        clear_bit(TASKLET_STATE_SCHED, &t->state);
 559}
 560
 561EXPORT_SYMBOL(tasklet_kill);
 562
 563/*
 564 * tasklet_hrtimer
 565 */
 566
 567/*
 568 * The trampoline is called when the hrtimer expires. It schedules a tasklet
 569 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 570 * hrtimer callback, but from softirq context.
 571 */
 572static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 573{
 574        struct tasklet_hrtimer *ttimer =
 575                container_of(timer, struct tasklet_hrtimer, timer);
 576
 577        tasklet_hi_schedule(&ttimer->tasklet);
 578        return HRTIMER_NORESTART;
 579}
 580
 581/*
 582 * Helper function which calls the hrtimer callback from
 583 * tasklet/softirq context
 584 */
 585static void __tasklet_hrtimer_trampoline(unsigned long data)
 586{
 587        struct tasklet_hrtimer *ttimer = (void *)data;
 588        enum hrtimer_restart restart;
 589
 590        restart = ttimer->function(&ttimer->timer);
 591        if (restart != HRTIMER_NORESTART)
 592                hrtimer_restart(&ttimer->timer);
 593}
 594
 595/**
 596 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 597 * @ttimer:      tasklet_hrtimer which is initialized
 598 * @function:    hrtimer callback function which gets called from softirq context
 599 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 600 * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 601 */
 602void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 603                          enum hrtimer_restart (*function)(struct hrtimer *),
 604                          clockid_t which_clock, enum hrtimer_mode mode)
 605{
 606        hrtimer_init(&ttimer->timer, which_clock, mode);
 607        ttimer->timer.function = __hrtimer_tasklet_trampoline;
 608        tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 609                     (unsigned long)ttimer);
 610        ttimer->function = function;
 611}
 612EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 613
 614/*
 615 * Remote softirq bits
 616 */
 617
 618DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 619EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 620
 621static void __local_trigger(struct call_single_data *cp, int softirq)
 622{
 623        struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 624
 625        list_add_tail(&cp->list, head);
 626
 627        /* Trigger the softirq only if the list was previously empty.  */
 628        if (head->next == &cp->list)
 629                raise_softirq_irqoff(softirq);
 630}
 631
 632#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 633static void remote_softirq_receive(void *data)
 634{
 635        struct call_single_data *cp = data;
 636        unsigned long flags;
 637        int softirq;
 638
 639        softirq = *(int *)cp->info;
 640        local_irq_save(flags);
 641        __local_trigger(cp, softirq);
 642        local_irq_restore(flags);
 643}
 644
 645static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 646{
 647        if (cpu_online(cpu)) {
 648                cp->func = remote_softirq_receive;
 649                cp->info = &softirq;
 650                cp->flags = 0;
 651
 652                __smp_call_function_single(cpu, cp, 0);
 653                return 0;
 654        }
 655        return 1;
 656}
 657#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 658static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 659{
 660        return 1;
 661}
 662#endif
 663
 664/**
 665 * __send_remote_softirq - try to schedule softirq work on a remote cpu
 666 * @cp: private SMP call function data area
 667 * @cpu: the remote cpu
 668 * @this_cpu: the currently executing cpu
 669 * @softirq: the softirq for the work
 670 *
 671 * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 672 * done, the work is instead queued up on the local cpu.
 673 *
 674 * Interrupts must be disabled.
 675 */
 676void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 677{
 678        if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 679                __local_trigger(cp, softirq);
 680}
 681EXPORT_SYMBOL(__send_remote_softirq);
 682
 683/**
 684 * send_remote_softirq - try to schedule softirq work on a remote cpu
 685 * @cp: private SMP call function data area
 686 * @cpu: the remote cpu
 687 * @softirq: the softirq for the work
 688 *
 689 * Like __send_remote_softirq except that disabling interrupts and
 690 * computing the current cpu is done for the caller.
 691 */
 692void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 693{
 694        unsigned long flags;
 695        int this_cpu;
 696
 697        local_irq_save(flags);
 698        this_cpu = smp_processor_id();
 699        __send_remote_softirq(cp, cpu, this_cpu, softirq);
 700        local_irq_restore(flags);
 701}
 702EXPORT_SYMBOL(send_remote_softirq);
 703
 704static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 705                                               unsigned long action, void *hcpu)
 706{
 707        /*
 708         * If a CPU goes away, splice its entries to the current CPU
 709         * and trigger a run of the softirq
 710         */
 711        if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 712                int cpu = (unsigned long) hcpu;
 713                int i;
 714
 715                local_irq_disable();
 716                for (i = 0; i < NR_SOFTIRQS; i++) {
 717                        struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 718                        struct list_head *local_head;
 719
 720                        if (list_empty(head))
 721                                continue;
 722
 723                        local_head = &__get_cpu_var(softirq_work_list[i]);
 724                        list_splice_init(head, local_head);
 725                        raise_softirq_irqoff(i);
 726                }
 727                local_irq_enable();
 728        }
 729
 730        return NOTIFY_OK;
 731}
 732
 733static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 734        .notifier_call  = remote_softirq_cpu_notify,
 735};
 736
 737void __init softirq_init(void)
 738{
 739        int cpu;
 740
 741        for_each_possible_cpu(cpu) {
 742                int i;
 743
 744                per_cpu(tasklet_vec, cpu).tail =
 745                        &per_cpu(tasklet_vec, cpu).head;
 746                per_cpu(tasklet_hi_vec, cpu).tail =
 747                        &per_cpu(tasklet_hi_vec, cpu).head;
 748                for (i = 0; i < NR_SOFTIRQS; i++)
 749                        INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 750        }
 751
 752        register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 753
 754        open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 755        open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 756}
 757
 758static int ksoftirqd_should_run(unsigned int cpu)
 759{
 760        return local_softirq_pending();
 761}
 762
 763static void run_ksoftirqd(unsigned int cpu)
 764{
 765        local_irq_disable();
 766        if (local_softirq_pending()) {
 767                __do_softirq();
 768                rcu_note_context_switch(cpu);
 769                local_irq_enable();
 770                cond_resched();
 771                return;
 772        }
 773        local_irq_enable();
 774}
 775
 776#ifdef CONFIG_HOTPLUG_CPU
 777/*
 778 * tasklet_kill_immediate is called to remove a tasklet which can already be
 779 * scheduled for execution on @cpu.
 780 *
 781 * Unlike tasklet_kill, this function removes the tasklet
 782 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 783 *
 784 * When this function is called, @cpu must be in the CPU_DEAD state.
 785 */
 786void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 787{
 788        struct tasklet_struct **i;
 789
 790        BUG_ON(cpu_online(cpu));
 791        BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 792
 793        if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 794                return;
 795
 796        /* CPU is dead, so no lock needed. */
 797        for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 798                if (*i == t) {
 799                        *i = t->next;
 800                        /* If this was the tail element, move the tail ptr */
 801                        if (*i == NULL)
 802                                per_cpu(tasklet_vec, cpu).tail = i;
 803                        return;
 804                }
 805        }
 806        BUG();
 807}
 808
 809static void takeover_tasklets(unsigned int cpu)
 810{
 811        /* CPU is dead, so no lock needed. */
 812        local_irq_disable();
 813
 814        /* Find end, append list for that CPU. */
 815        if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 816                *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 817                this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 818                per_cpu(tasklet_vec, cpu).head = NULL;
 819                per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 820        }
 821        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 822
 823        if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 824                *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 825                __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 826                per_cpu(tasklet_hi_vec, cpu).head = NULL;
 827                per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 828        }
 829        raise_softirq_irqoff(HI_SOFTIRQ);
 830
 831        local_irq_enable();
 832}
 833#endif /* CONFIG_HOTPLUG_CPU */
 834
 835static int __cpuinit cpu_callback(struct notifier_block *nfb,
 836                                  unsigned long action,
 837                                  void *hcpu)
 838{
 839        switch (action) {
 840#ifdef CONFIG_HOTPLUG_CPU
 841        case CPU_DEAD:
 842        case CPU_DEAD_FROZEN:
 843                takeover_tasklets((unsigned long)hcpu);
 844                break;
 845#endif /* CONFIG_HOTPLUG_CPU */
 846        }
 847        return NOTIFY_OK;
 848}
 849
 850static struct notifier_block __cpuinitdata cpu_nfb = {
 851        .notifier_call = cpu_callback
 852};
 853
 854static struct smp_hotplug_thread softirq_threads = {
 855        .store                  = &ksoftirqd,
 856        .thread_should_run      = ksoftirqd_should_run,
 857        .thread_fn              = run_ksoftirqd,
 858        .thread_comm            = "ksoftirqd/%u",
 859};
 860
 861static __init int spawn_ksoftirqd(void)
 862{
 863        register_cpu_notifier(&cpu_nfb);
 864
 865        BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 866
 867        return 0;
 868}
 869early_initcall(spawn_ksoftirqd);
 870
 871/*
 872 * [ These __weak aliases are kept in a separate compilation unit, so that
 873 *   GCC does not inline them incorrectly. ]
 874 */
 875
 876int __init __weak early_irq_init(void)
 877{
 878        return 0;
 879}
 880
 881#ifdef CONFIG_GENERIC_HARDIRQS
 882int __init __weak arch_probe_nr_irqs(void)
 883{
 884        return NR_IRQS_LEGACY;
 885}
 886
 887int __init __weak arch_early_irq_init(void)
 888{
 889        return 0;
 890}
 891#endif
 892
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.