linux/kernel/softirq.c
<<
>>
Prefs
   1/*
   2 *      linux/kernel/softirq.c
   3 *
   4 *      Copyright (C) 1992 Linus Torvalds
   5 *
   6 *      Distribute under GPLv2.
   7 *
   8 *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9 *
  10 *      Remote softirq infrastructure is by Jens Axboe.
  11 */
  12
  13#include <linux/module.h>
  14#include <linux/kernel_stat.h>
  15#include <linux/interrupt.h>
  16#include <linux/init.h>
  17#include <linux/mm.h>
  18#include <linux/notifier.h>
  19#include <linux/percpu.h>
  20#include <linux/cpu.h>
  21#include <linux/freezer.h>
  22#include <linux/kthread.h>
  23#include <linux/rcupdate.h>
  24#include <linux/ftrace.h>
  25#include <linux/smp.h>
  26#include <linux/tick.h>
  27#include <trace/irq.h>
  28
  29#include <asm/irq.h>
  30/*
  31   - No shared variables, all the data are CPU local.
  32   - If a softirq needs serialization, let it serialize itself
  33     by its own spinlocks.
  34   - Even if softirq is serialized, only local cpu is marked for
  35     execution. Hence, we get something sort of weak cpu binding.
  36     Though it is still not clear, will it result in better locality
  37     or will not.
  38
  39   Examples:
  40   - NET RX softirq. It is multithreaded and does not require
  41     any global serialization.
  42   - NET TX softirq. It kicks software netdevice queues, hence
  43     it is logically serialized per device, but this serialization
  44     is invisible to common code.
  45   - Tasklets: serialized wrt itself.
  46 */
  47
  48#ifndef __ARCH_IRQ_STAT
  49irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  50EXPORT_SYMBOL(irq_stat);
  51#endif
  52
  53static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  54
  55static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  56
  57char *softirq_to_name[NR_SOFTIRQS] = {
  58        "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
  59        "TASKLET", "SCHED", "HRTIMER",  "RCU"
  60};
  61
  62/*
  63 * we cannot loop indefinitely here to avoid userspace starvation,
  64 * but we also don't want to introduce a worst case 1/HZ latency
  65 * to the pending events, so lets the scheduler to balance
  66 * the softirq load for us.
  67 */
  68void wakeup_softirqd(void)
  69{
  70        /* Interrupts are disabled: no need to stop preemption */
  71        struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  72
  73        if (tsk && tsk->state != TASK_RUNNING)
  74                wake_up_process(tsk);
  75}
  76
  77/*
  78 * This one is for softirq.c-internal use,
  79 * where hardirqs are disabled legitimately:
  80 */
  81#ifdef CONFIG_TRACE_IRQFLAGS
  82static void __local_bh_disable(unsigned long ip)
  83{
  84        unsigned long flags;
  85
  86        WARN_ON_ONCE(in_irq());
  87
  88        raw_local_irq_save(flags);
  89        /*
  90         * The preempt tracer hooks into add_preempt_count and will break
  91         * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
  92         * is set and before current->softirq_enabled is cleared.
  93         * We must manually increment preempt_count here and manually
  94         * call the trace_preempt_off later.
  95         */
  96        preempt_count() += SOFTIRQ_OFFSET;
  97        /*
  98         * Were softirqs turned off above:
  99         */
 100        if (softirq_count() == SOFTIRQ_OFFSET)
 101                trace_softirqs_off(ip);
 102        raw_local_irq_restore(flags);
 103
 104        if (preempt_count() == SOFTIRQ_OFFSET)
 105                trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 106}
 107#else /* !CONFIG_TRACE_IRQFLAGS */
 108static inline void __local_bh_disable(unsigned long ip)
 109{
 110        add_preempt_count(SOFTIRQ_OFFSET);
 111        barrier();
 112}
 113#endif /* CONFIG_TRACE_IRQFLAGS */
 114
 115void local_bh_disable(void)
 116{
 117        __local_bh_disable((unsigned long)__builtin_return_address(0));
 118}
 119
 120EXPORT_SYMBOL(local_bh_disable);
 121
 122/*
 123 * Special-case - softirqs can safely be enabled in
 124 * cond_resched_softirq(), or by __do_softirq(),
 125 * without processing still-pending softirqs:
 126 */
 127void _local_bh_enable(void)
 128{
 129        WARN_ON_ONCE(in_irq());
 130        WARN_ON_ONCE(!irqs_disabled());
 131
 132        if (softirq_count() == SOFTIRQ_OFFSET)
 133                trace_softirqs_on((unsigned long)__builtin_return_address(0));
 134        sub_preempt_count(SOFTIRQ_OFFSET);
 135}
 136
 137EXPORT_SYMBOL(_local_bh_enable);
 138
 139static inline void _local_bh_enable_ip(unsigned long ip)
 140{
 141        WARN_ON_ONCE(in_irq() || irqs_disabled());
 142#ifdef CONFIG_TRACE_IRQFLAGS
 143        local_irq_disable();
 144#endif
 145        /*
 146         * Are softirqs going to be turned on now:
 147         */
 148        if (softirq_count() == SOFTIRQ_OFFSET)
 149                trace_softirqs_on(ip);
 150        /*
 151         * Keep preemption disabled until we are done with
 152         * softirq processing:
 153         */
 154        sub_preempt_count(SOFTIRQ_OFFSET - 1);
 155
 156        if (unlikely(!in_interrupt() && local_softirq_pending()))
 157                do_softirq();
 158
 159        dec_preempt_count();
 160#ifdef CONFIG_TRACE_IRQFLAGS
 161        local_irq_enable();
 162#endif
 163        preempt_check_resched();
 164}
 165
 166void local_bh_enable(void)
 167{
 168        _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 169}
 170EXPORT_SYMBOL(local_bh_enable);
 171
 172void local_bh_enable_ip(unsigned long ip)
 173{
 174        _local_bh_enable_ip(ip);
 175}
 176EXPORT_SYMBOL(local_bh_enable_ip);
 177
 178/*
 179 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 180 * and we fall back to softirqd after that.
 181 *
 182 * This number has been established via experimentation.
 183 * The two things to balance is latency against fairness -
 184 * we want to handle softirqs as soon as possible, but they
 185 * should not be able to lock up the box.
 186 */
 187#define MAX_SOFTIRQ_RESTART 10
 188
 189DEFINE_TRACE(softirq_entry);
 190DEFINE_TRACE(softirq_exit);
 191
 192asmlinkage void __do_softirq(void)
 193{
 194        struct softirq_action *h;
 195        __u32 pending;
 196        int max_restart = MAX_SOFTIRQ_RESTART;
 197        int cpu;
 198
 199        pending = local_softirq_pending();
 200        account_system_vtime(current);
 201
 202        __local_bh_disable((unsigned long)__builtin_return_address(0));
 203        lockdep_softirq_enter();
 204
 205        cpu = smp_processor_id();
 206restart:
 207        /* Reset the pending bitmask before enabling irqs */
 208        set_softirq_pending(0);
 209
 210        local_irq_enable();
 211
 212        h = softirq_vec;
 213
 214        do {
 215                if (pending & 1) {
 216                        int prev_count = preempt_count();
 217
 218                        trace_softirq_entry(h, softirq_vec);
 219                        h->action(h);
 220                        trace_softirq_exit(h, softirq_vec);
 221                        if (unlikely(prev_count != preempt_count())) {
 222                                printk(KERN_ERR "huh, entered softirq %td %s %p"
 223                                       "with preempt_count %08x,"
 224                                       " exited with %08x?\n", h - softirq_vec,
 225                                       softirq_to_name[h - softirq_vec],
 226                                       h->action, prev_count, preempt_count());
 227                                preempt_count() = prev_count;
 228                        }
 229
 230                        rcu_bh_qsctr_inc(cpu);
 231                }
 232                h++;
 233                pending >>= 1;
 234        } while (pending);
 235
 236        local_irq_disable();
 237
 238        pending = local_softirq_pending();
 239        if (pending && --max_restart)
 240                goto restart;
 241
 242        if (pending)
 243                wakeup_softirqd();
 244
 245        lockdep_softirq_exit();
 246
 247        account_system_vtime(current);
 248        _local_bh_enable();
 249}
 250
 251#ifndef __ARCH_HAS_DO_SOFTIRQ
 252
 253asmlinkage void do_softirq(void)
 254{
 255        __u32 pending;
 256        unsigned long flags;
 257
 258        if (in_interrupt())
 259                return;
 260
 261        local_irq_save(flags);
 262
 263        pending = local_softirq_pending();
 264
 265        if (pending)
 266                __do_softirq();
 267
 268        local_irq_restore(flags);
 269}
 270
 271#endif
 272
 273/*
 274 * Enter an interrupt context.
 275 */
 276void irq_enter(void)
 277{
 278        int cpu = smp_processor_id();
 279
 280        rcu_irq_enter();
 281        if (idle_cpu(cpu) && !in_interrupt()) {
 282                __irq_enter();
 283                tick_check_idle(cpu);
 284        } else
 285                __irq_enter();
 286}
 287
 288#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 289# define invoke_softirq()       __do_softirq()
 290#else
 291# define invoke_softirq()       do_softirq()
 292#endif
 293
 294/*
 295 * Exit an interrupt context. Process softirqs if needed and possible:
 296 */
 297void irq_exit(void)
 298{
 299        account_system_vtime(current);
 300        trace_hardirq_exit();
 301        sub_preempt_count(IRQ_EXIT_OFFSET);
 302        if (!in_interrupt() && local_softirq_pending())
 303                invoke_softirq();
 304
 305#ifdef CONFIG_NO_HZ
 306        /* Make sure that timer wheel updates are propagated */
 307        rcu_irq_exit();
 308        if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 309                tick_nohz_stop_sched_tick(0);
 310#endif
 311        preempt_enable_no_resched();
 312}
 313
 314/*
 315 * This function must run with irqs disabled!
 316 */
 317inline void raise_softirq_irqoff(unsigned int nr)
 318{
 319        __raise_softirq_irqoff(nr);
 320
 321        /*
 322         * If we're in an interrupt or softirq, we're done
 323         * (this also catches softirq-disabled code). We will
 324         * actually run the softirq once we return from
 325         * the irq or softirq.
 326         *
 327         * Otherwise we wake up ksoftirqd to make sure we
 328         * schedule the softirq soon.
 329         */
 330        if (!in_interrupt())
 331                wakeup_softirqd();
 332}
 333
 334void raise_softirq(unsigned int nr)
 335{
 336        unsigned long flags;
 337
 338        local_irq_save(flags);
 339        raise_softirq_irqoff(nr);
 340        local_irq_restore(flags);
 341}
 342
 343void open_softirq(int nr, void (*action)(struct softirq_action *))
 344{
 345        softirq_vec[nr].action = action;
 346}
 347
 348/* Tasklets */
 349struct tasklet_head
 350{
 351        struct tasklet_struct *head;
 352        struct tasklet_struct **tail;
 353};
 354
 355static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 356static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 357
 358void __tasklet_schedule(struct tasklet_struct *t)
 359{
 360        unsigned long flags;
 361
 362        local_irq_save(flags);
 363        t->next = NULL;
 364        *__get_cpu_var(tasklet_vec).tail = t;
 365        __get_cpu_var(tasklet_vec).tail = &(t->next);
 366        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 367        local_irq_restore(flags);
 368}
 369
 370EXPORT_SYMBOL(__tasklet_schedule);
 371
 372void __tasklet_hi_schedule(struct tasklet_struct *t)
 373{
 374        unsigned long flags;
 375
 376        local_irq_save(flags);
 377        t->next = NULL;
 378        *__get_cpu_var(tasklet_hi_vec).tail = t;
 379        __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 380        raise_softirq_irqoff(HI_SOFTIRQ);
 381        local_irq_restore(flags);
 382}
 383
 384EXPORT_SYMBOL(__tasklet_hi_schedule);
 385
 386static void tasklet_action(struct softirq_action *a)
 387{
 388        struct tasklet_struct *list;
 389
 390        local_irq_disable();
 391        list = __get_cpu_var(tasklet_vec).head;
 392        __get_cpu_var(tasklet_vec).head = NULL;
 393        __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 394        local_irq_enable();
 395
 396        while (list) {
 397                struct tasklet_struct *t = list;
 398
 399                list = list->next;
 400
 401                if (tasklet_trylock(t)) {
 402                        if (!atomic_read(&t->count)) {
 403                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 404                                        BUG();
 405                                t->func(t->data);
 406                                tasklet_unlock(t);
 407                                continue;
 408                        }
 409                        tasklet_unlock(t);
 410                }
 411
 412                local_irq_disable();
 413                t->next = NULL;
 414                *__get_cpu_var(tasklet_vec).tail = t;
 415                __get_cpu_var(tasklet_vec).tail = &(t->next);
 416                __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 417                local_irq_enable();
 418        }
 419}
 420
 421static void tasklet_hi_action(struct softirq_action *a)
 422{
 423        struct tasklet_struct *list;
 424
 425        local_irq_disable();
 426        list = __get_cpu_var(tasklet_hi_vec).head;
 427        __get_cpu_var(tasklet_hi_vec).head = NULL;
 428        __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 429        local_irq_enable();
 430
 431        while (list) {
 432                struct tasklet_struct *t = list;
 433
 434                list = list->next;
 435
 436                if (tasklet_trylock(t)) {
 437                        if (!atomic_read(&t->count)) {
 438                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 439                                        BUG();
 440                                t->func(t->data);
 441                                tasklet_unlock(t);
 442                                continue;
 443                        }
 444                        tasklet_unlock(t);
 445                }
 446
 447                local_irq_disable();
 448                t->next = NULL;
 449                *__get_cpu_var(tasklet_hi_vec).tail = t;
 450                __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 451                __raise_softirq_irqoff(HI_SOFTIRQ);
 452                local_irq_enable();
 453        }
 454}
 455
 456
 457void tasklet_init(struct tasklet_struct *t,
 458                  void (*func)(unsigned long), unsigned long data)
 459{
 460        t->next = NULL;
 461        t->state = 0;
 462        atomic_set(&t->count, 0);
 463        t->func = func;
 464        t->data = data;
 465}
 466
 467EXPORT_SYMBOL(tasklet_init);
 468
 469void tasklet_kill(struct tasklet_struct *t)
 470{
 471        if (in_interrupt())
 472                printk("Attempt to kill tasklet from interrupt\n");
 473
 474        while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 475                do {
 476                        yield();
 477                } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 478        }
 479        tasklet_unlock_wait(t);
 480        clear_bit(TASKLET_STATE_SCHED, &t->state);
 481}
 482
 483EXPORT_SYMBOL(tasklet_kill);
 484
 485DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 486EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 487
 488static void __local_trigger(struct call_single_data *cp, int softirq)
 489{
 490        struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 491
 492        list_add_tail(&cp->list, head);
 493
 494        /* Trigger the softirq only if the list was previously empty.  */
 495        if (head->next == &cp->list)
 496                raise_softirq_irqoff(softirq);
 497}
 498
 499#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 500static void remote_softirq_receive(void *data)
 501{
 502        struct call_single_data *cp = data;
 503        unsigned long flags;
 504        int softirq;
 505
 506        softirq = cp->priv;
 507
 508        local_irq_save(flags);
 509        __local_trigger(cp, softirq);
 510        local_irq_restore(flags);
 511}
 512
 513static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 514{
 515        if (cpu_online(cpu)) {
 516                cp->func = remote_softirq_receive;
 517                cp->info = cp;
 518                cp->flags = 0;
 519                cp->priv = softirq;
 520
 521                __smp_call_function_single(cpu, cp, 0);
 522                return 0;
 523        }
 524        return 1;
 525}
 526#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 527static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 528{
 529        return 1;
 530}
 531#endif
 532
 533/**
 534 * __send_remote_softirq - try to schedule softirq work on a remote cpu
 535 * @cp: private SMP call function data area
 536 * @cpu: the remote cpu
 537 * @this_cpu: the currently executing cpu
 538 * @softirq: the softirq for the work
 539 *
 540 * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 541 * done, the work is instead queued up on the local cpu.
 542 *
 543 * Interrupts must be disabled.
 544 */
 545void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 546{
 547        if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 548                __local_trigger(cp, softirq);
 549}
 550EXPORT_SYMBOL(__send_remote_softirq);
 551
 552/**
 553 * send_remote_softirq - try to schedule softirq work on a remote cpu
 554 * @cp: private SMP call function data area
 555 * @cpu: the remote cpu
 556 * @softirq: the softirq for the work
 557 *
 558 * Like __send_remote_softirq except that disabling interrupts and
 559 * computing the current cpu is done for the caller.
 560 */
 561void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 562{
 563        unsigned long flags;
 564        int this_cpu;
 565
 566        local_irq_save(flags);
 567        this_cpu = smp_processor_id();
 568        __send_remote_softirq(cp, cpu, this_cpu, softirq);
 569        local_irq_restore(flags);
 570}
 571EXPORT_SYMBOL(send_remote_softirq);
 572
 573static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 574                                               unsigned long action, void *hcpu)
 575{
 576        /*
 577         * If a CPU goes away, splice its entries to the current CPU
 578         * and trigger a run of the softirq
 579         */
 580        if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 581                int cpu = (unsigned long) hcpu;
 582                int i;
 583
 584                local_irq_disable();
 585                for (i = 0; i < NR_SOFTIRQS; i++) {
 586                        struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 587                        struct list_head *local_head;
 588
 589                        if (list_empty(head))
 590                                continue;
 591
 592                        local_head = &__get_cpu_var(softirq_work_list[i]);
 593                        list_splice_init(head, local_head);
 594                        raise_softirq_irqoff(i);
 595                }
 596                local_irq_enable();
 597        }
 598
 599        return NOTIFY_OK;
 600}
 601
 602static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 603        .notifier_call  = remote_softirq_cpu_notify,
 604};
 605
 606void __init softirq_init(void)
 607{
 608        int cpu;
 609
 610        for_each_possible_cpu(cpu) {
 611                int i;
 612
 613                per_cpu(tasklet_vec, cpu).tail =
 614                        &per_cpu(tasklet_vec, cpu).head;
 615                per_cpu(tasklet_hi_vec, cpu).tail =
 616                        &per_cpu(tasklet_hi_vec, cpu).head;
 617                for (i = 0; i < NR_SOFTIRQS; i++)
 618                        INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 619        }
 620
 621        register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 622
 623        open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 624        open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 625}
 626
 627static int ksoftirqd(void * __bind_cpu)
 628{
 629        set_current_state(TASK_INTERRUPTIBLE);
 630
 631        while (!kthread_should_stop()) {
 632                preempt_disable();
 633                if (!local_softirq_pending()) {
 634                        preempt_enable_no_resched();
 635                        schedule();
 636                        preempt_disable();
 637                }
 638
 639                __set_current_state(TASK_RUNNING);
 640
 641                while (local_softirq_pending()) {
 642                        /* Preempt disable stops cpu going offline.
 643                           If already offline, we'll be on wrong CPU:
 644                           don't process */
 645                        if (cpu_is_offline((long)__bind_cpu))
 646                                goto wait_to_die;
 647                        do_softirq();
 648                        preempt_enable_no_resched();
 649                        cond_resched();
 650                        preempt_disable();
 651                        rcu_qsctr_inc((long)__bind_cpu);
 652                }
 653                preempt_enable();
 654                set_current_state(TASK_INTERRUPTIBLE);
 655        }
 656        __set_current_state(TASK_RUNNING);
 657        return 0;
 658
 659wait_to_die:
 660        preempt_enable();
 661        /* Wait for kthread_stop */
 662        set_current_state(TASK_INTERRUPTIBLE);
 663        while (!kthread_should_stop()) {
 664                schedule();
 665                set_current_state(TASK_INTERRUPTIBLE);
 666        }
 667        __set_current_state(TASK_RUNNING);
 668        return 0;
 669}
 670
 671#ifdef CONFIG_HOTPLUG_CPU
 672/*
 673 * tasklet_kill_immediate is called to remove a tasklet which can already be
 674 * scheduled for execution on @cpu.
 675 *
 676 * Unlike tasklet_kill, this function removes the tasklet
 677 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 678 *
 679 * When this function is called, @cpu must be in the CPU_DEAD state.
 680 */
 681void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 682{
 683        struct tasklet_struct **i;
 684
 685        BUG_ON(cpu_online(cpu));
 686        BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 687
 688        if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 689                return;
 690
 691        /* CPU is dead, so no lock needed. */
 692        for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 693                if (*i == t) {
 694                        *i = t->next;
 695                        /* If this was the tail element, move the tail ptr */
 696                        if (*i == NULL)
 697                                per_cpu(tasklet_vec, cpu).tail = i;
 698                        return;
 699                }
 700        }
 701        BUG();
 702}
 703
 704static void takeover_tasklets(unsigned int cpu)
 705{
 706        /* CPU is dead, so no lock needed. */
 707        local_irq_disable();
 708
 709        /* Find end, append list for that CPU. */
 710        if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 711                *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 712                __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 713                per_cpu(tasklet_vec, cpu).head = NULL;
 714                per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 715        }
 716        raise_softirq_irqoff(TASKLET_SOFTIRQ);
 717
 718        if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 719                *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 720                __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 721                per_cpu(tasklet_hi_vec, cpu).head = NULL;
 722                per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 723        }
 724        raise_softirq_irqoff(HI_SOFTIRQ);
 725
 726        local_irq_enable();
 727}
 728#endif /* CONFIG_HOTPLUG_CPU */
 729
 730static int __cpuinit cpu_callback(struct notifier_block *nfb,
 731                                  unsigned long action,
 732                                  void *hcpu)
 733{
 734        int hotcpu = (unsigned long)hcpu;
 735        struct task_struct *p;
 736
 737        switch (action) {
 738        case CPU_UP_PREPARE:
 739        case CPU_UP_PREPARE_FROZEN:
 740                p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 741                if (IS_ERR(p)) {
 742                        printk("ksoftirqd for %i failed\n", hotcpu);
 743                        return NOTIFY_BAD;
 744                }
 745                kthread_bind(p, hotcpu);
 746                per_cpu(ksoftirqd, hotcpu) = p;
 747                break;
 748        case CPU_ONLINE:
 749        case CPU_ONLINE_FROZEN:
 750                wake_up_process(per_cpu(ksoftirqd, hotcpu));
 751                break;
 752#ifdef CONFIG_HOTPLUG_CPU
 753        case CPU_UP_CANCELED:
 754        case CPU_UP_CANCELED_FROZEN:
 755                if (!per_cpu(ksoftirqd, hotcpu))
 756                        break;
 757                /* Unbind so it can run.  Fall thru. */
 758                kthread_bind(per_cpu(ksoftirqd, hotcpu),
 759                             cpumask_any(cpu_online_mask));
 760        case CPU_DEAD:
 761        case CPU_DEAD_FROZEN: {
 762                struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 763
 764                p = per_cpu(ksoftirqd, hotcpu);
 765                per_cpu(ksoftirqd, hotcpu) = NULL;
 766                sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 767                kthread_stop(p);
 768                takeover_tasklets(hotcpu);
 769                break;
 770        }
 771#endif /* CONFIG_HOTPLUG_CPU */
 772        }
 773        return NOTIFY_OK;
 774}
 775
 776static struct notifier_block __cpuinitdata cpu_nfb = {
 777        .notifier_call = cpu_callback
 778};
 779
 780static __init int spawn_ksoftirqd(void)
 781{
 782        void *cpu = (void *)(long)smp_processor_id();
 783        int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 784
 785        BUG_ON(err == NOTIFY_BAD);
 786        cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 787        register_cpu_notifier(&cpu_nfb);
 788        return 0;
 789}
 790early_initcall(spawn_ksoftirqd);
 791
 792#ifdef CONFIG_SMP
 793/*
 794 * Call a function on all processors
 795 */
 796int on_each_cpu(void (*func) (void *info), void *info, int wait)
 797{
 798        int ret = 0;
 799
 800        preempt_disable();
 801        ret = smp_call_function(func, info, wait);
 802        local_irq_disable();
 803        func(info);
 804        local_irq_enable();
 805        preempt_enable();
 806        return ret;
 807}
 808EXPORT_SYMBOL(on_each_cpu);
 809#endif
 810
 811/*
 812 * [ These __weak aliases are kept in a separate compilation unit, so that
 813 *   GCC does not inline them incorrectly. ]
 814 */
 815
 816int __init __weak early_irq_init(void)
 817{
 818        return 0;
 819}
 820
 821int __init __weak arch_probe_nr_irqs(void)
 822{
 823        return 0;
 824}
 825
 826int __init __weak arch_early_irq_init(void)
 827{
 828        return 0;
 829}
 830
 831int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
 832{
 833        return 0;
 834}
 835
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.