1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/module.h>
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
21#include <linux/freezer.h>
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
25#include <linux/smp.h>
26#include <linux/tick.h>
27#include <trace/irq.h>
28
29#include <asm/irq.h>
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48#ifndef __ARCH_IRQ_STAT
49irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
50EXPORT_SYMBOL(irq_stat);
51#endif
52
53static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
54
55static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
56
57char *softirq_to_name[NR_SOFTIRQS] = {
58 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
59 "TASKLET", "SCHED", "HRTIMER", "RCU"
60};
61
62
63
64
65
66
67
68void wakeup_softirqd(void)
69{
70
71 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
72
73 if (tsk && tsk->state != TASK_RUNNING)
74 wake_up_process(tsk);
75}
76
77
78
79
80
81#ifdef CONFIG_TRACE_IRQFLAGS
82static void __local_bh_disable(unsigned long ip)
83{
84 unsigned long flags;
85
86 WARN_ON_ONCE(in_irq());
87
88 raw_local_irq_save(flags);
89
90
91
92
93
94
95
96 preempt_count() += SOFTIRQ_OFFSET;
97
98
99
100 if (softirq_count() == SOFTIRQ_OFFSET)
101 trace_softirqs_off(ip);
102 raw_local_irq_restore(flags);
103
104 if (preempt_count() == SOFTIRQ_OFFSET)
105 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
106}
107#else
108static inline void __local_bh_disable(unsigned long ip)
109{
110 add_preempt_count(SOFTIRQ_OFFSET);
111 barrier();
112}
113#endif
114
115void local_bh_disable(void)
116{
117 __local_bh_disable((unsigned long)__builtin_return_address(0));
118}
119
120EXPORT_SYMBOL(local_bh_disable);
121
122
123
124
125
126
127void _local_bh_enable(void)
128{
129 WARN_ON_ONCE(in_irq());
130 WARN_ON_ONCE(!irqs_disabled());
131
132 if (softirq_count() == SOFTIRQ_OFFSET)
133 trace_softirqs_on((unsigned long)__builtin_return_address(0));
134 sub_preempt_count(SOFTIRQ_OFFSET);
135}
136
137EXPORT_SYMBOL(_local_bh_enable);
138
139static inline void _local_bh_enable_ip(unsigned long ip)
140{
141 WARN_ON_ONCE(in_irq() || irqs_disabled());
142#ifdef CONFIG_TRACE_IRQFLAGS
143 local_irq_disable();
144#endif
145
146
147
148 if (softirq_count() == SOFTIRQ_OFFSET)
149 trace_softirqs_on(ip);
150
151
152
153
154 sub_preempt_count(SOFTIRQ_OFFSET - 1);
155
156 if (unlikely(!in_interrupt() && local_softirq_pending()))
157 do_softirq();
158
159 dec_preempt_count();
160#ifdef CONFIG_TRACE_IRQFLAGS
161 local_irq_enable();
162#endif
163 preempt_check_resched();
164}
165
166void local_bh_enable(void)
167{
168 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
169}
170EXPORT_SYMBOL(local_bh_enable);
171
172void local_bh_enable_ip(unsigned long ip)
173{
174 _local_bh_enable_ip(ip);
175}
176EXPORT_SYMBOL(local_bh_enable_ip);
177
178
179
180
181
182
183
184
185
186
187#define MAX_SOFTIRQ_RESTART 10
188
189DEFINE_TRACE(softirq_entry);
190DEFINE_TRACE(softirq_exit);
191
192asmlinkage void __do_softirq(void)
193{
194 struct softirq_action *h;
195 __u32 pending;
196 int max_restart = MAX_SOFTIRQ_RESTART;
197 int cpu;
198
199 pending = local_softirq_pending();
200 account_system_vtime(current);
201
202 __local_bh_disable((unsigned long)__builtin_return_address(0));
203 lockdep_softirq_enter();
204
205 cpu = smp_processor_id();
206restart:
207
208 set_softirq_pending(0);
209
210 local_irq_enable();
211
212 h = softirq_vec;
213
214 do {
215 if (pending & 1) {
216 int prev_count = preempt_count();
217
218 trace_softirq_entry(h, softirq_vec);
219 h->action(h);
220 trace_softirq_exit(h, softirq_vec);
221 if (unlikely(prev_count != preempt_count())) {
222 printk(KERN_ERR "huh, entered softirq %td %s %p"
223 "with preempt_count %08x,"
224 " exited with %08x?\n", h - softirq_vec,
225 softirq_to_name[h - softirq_vec],
226 h->action, prev_count, preempt_count());
227 preempt_count() = prev_count;
228 }
229
230 rcu_bh_qsctr_inc(cpu);
231 }
232 h++;
233 pending >>= 1;
234 } while (pending);
235
236 local_irq_disable();
237
238 pending = local_softirq_pending();
239 if (pending && --max_restart)
240 goto restart;
241
242 if (pending)
243 wakeup_softirqd();
244
245 lockdep_softirq_exit();
246
247 account_system_vtime(current);
248 _local_bh_enable();
249}
250
251#ifndef __ARCH_HAS_DO_SOFTIRQ
252
253asmlinkage void do_softirq(void)
254{
255 __u32 pending;
256 unsigned long flags;
257
258 if (in_interrupt())
259 return;
260
261 local_irq_save(flags);
262
263 pending = local_softirq_pending();
264
265 if (pending)
266 __do_softirq();
267
268 local_irq_restore(flags);
269}
270
271#endif
272
273
274
275
276void irq_enter(void)
277{
278 int cpu = smp_processor_id();
279
280 rcu_irq_enter();
281 if (idle_cpu(cpu) && !in_interrupt()) {
282 __irq_enter();
283 tick_check_idle(cpu);
284 } else
285 __irq_enter();
286}
287
288#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
289# define invoke_softirq() __do_softirq()
290#else
291# define invoke_softirq() do_softirq()
292#endif
293
294
295
296
297void irq_exit(void)
298{
299 account_system_vtime(current);
300 trace_hardirq_exit();
301 sub_preempt_count(IRQ_EXIT_OFFSET);
302 if (!in_interrupt() && local_softirq_pending())
303 invoke_softirq();
304
305#ifdef CONFIG_NO_HZ
306
307 rcu_irq_exit();
308 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
309 tick_nohz_stop_sched_tick(0);
310#endif
311 preempt_enable_no_resched();
312}
313
314
315
316
317inline void raise_softirq_irqoff(unsigned int nr)
318{
319 __raise_softirq_irqoff(nr);
320
321
322
323
324
325
326
327
328
329
330 if (!in_interrupt())
331 wakeup_softirqd();
332}
333
334void raise_softirq(unsigned int nr)
335{
336 unsigned long flags;
337
338 local_irq_save(flags);
339 raise_softirq_irqoff(nr);
340 local_irq_restore(flags);
341}
342
343void open_softirq(int nr, void (*action)(struct softirq_action *))
344{
345 softirq_vec[nr].action = action;
346}
347
348
349struct tasklet_head
350{
351 struct tasklet_struct *head;
352 struct tasklet_struct **tail;
353};
354
355static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
356static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
357
358void __tasklet_schedule(struct tasklet_struct *t)
359{
360 unsigned long flags;
361
362 local_irq_save(flags);
363 t->next = NULL;
364 *__get_cpu_var(tasklet_vec).tail = t;
365 __get_cpu_var(tasklet_vec).tail = &(t->next);
366 raise_softirq_irqoff(TASKLET_SOFTIRQ);
367 local_irq_restore(flags);
368}
369
370EXPORT_SYMBOL(__tasklet_schedule);
371
372void __tasklet_hi_schedule(struct tasklet_struct *t)
373{
374 unsigned long flags;
375
376 local_irq_save(flags);
377 t->next = NULL;
378 *__get_cpu_var(tasklet_hi_vec).tail = t;
379 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
380 raise_softirq_irqoff(HI_SOFTIRQ);
381 local_irq_restore(flags);
382}
383
384EXPORT_SYMBOL(__tasklet_hi_schedule);
385
386static void tasklet_action(struct softirq_action *a)
387{
388 struct tasklet_struct *list;
389
390 local_irq_disable();
391 list = __get_cpu_var(tasklet_vec).head;
392 __get_cpu_var(tasklet_vec).head = NULL;
393 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
394 local_irq_enable();
395
396 while (list) {
397 struct tasklet_struct *t = list;
398
399 list = list->next;
400
401 if (tasklet_trylock(t)) {
402 if (!atomic_read(&t->count)) {
403 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
404 BUG();
405 t->func(t->data);
406 tasklet_unlock(t);
407 continue;
408 }
409 tasklet_unlock(t);
410 }
411
412 local_irq_disable();
413 t->next = NULL;
414 *__get_cpu_var(tasklet_vec).tail = t;
415 __get_cpu_var(tasklet_vec).tail = &(t->next);
416 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
417 local_irq_enable();
418 }
419}
420
421static void tasklet_hi_action(struct softirq_action *a)
422{
423 struct tasklet_struct *list;
424
425 local_irq_disable();
426 list = __get_cpu_var(tasklet_hi_vec).head;
427 __get_cpu_var(tasklet_hi_vec).head = NULL;
428 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
429 local_irq_enable();
430
431 while (list) {
432 struct tasklet_struct *t = list;
433
434 list = list->next;
435
436 if (tasklet_trylock(t)) {
437 if (!atomic_read(&t->count)) {
438 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
439 BUG();
440 t->func(t->data);
441 tasklet_unlock(t);
442 continue;
443 }
444 tasklet_unlock(t);
445 }
446
447 local_irq_disable();
448 t->next = NULL;
449 *__get_cpu_var(tasklet_hi_vec).tail = t;
450 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
451 __raise_softirq_irqoff(HI_SOFTIRQ);
452 local_irq_enable();
453 }
454}
455
456
457void tasklet_init(struct tasklet_struct *t,
458 void (*func)(unsigned long), unsigned long data)
459{
460 t->next = NULL;
461 t->state = 0;
462 atomic_set(&t->count, 0);
463 t->func = func;
464 t->data = data;
465}
466
467EXPORT_SYMBOL(tasklet_init);
468
469void tasklet_kill(struct tasklet_struct *t)
470{
471 if (in_interrupt())
472 printk("Attempt to kill tasklet from interrupt\n");
473
474 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
475 do {
476 yield();
477 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
478 }
479 tasklet_unlock_wait(t);
480 clear_bit(TASKLET_STATE_SCHED, &t->state);
481}
482
483EXPORT_SYMBOL(tasklet_kill);
484
485DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
486EXPORT_PER_CPU_SYMBOL(softirq_work_list);
487
488static void __local_trigger(struct call_single_data *cp, int softirq)
489{
490 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
491
492 list_add_tail(&cp->list, head);
493
494
495 if (head->next == &cp->list)
496 raise_softirq_irqoff(softirq);
497}
498
499#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
500static void remote_softirq_receive(void *data)
501{
502 struct call_single_data *cp = data;
503 unsigned long flags;
504 int softirq;
505
506 softirq = cp->priv;
507
508 local_irq_save(flags);
509 __local_trigger(cp, softirq);
510 local_irq_restore(flags);
511}
512
513static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
514{
515 if (cpu_online(cpu)) {
516 cp->func = remote_softirq_receive;
517 cp->info = cp;
518 cp->flags = 0;
519 cp->priv = softirq;
520
521 __smp_call_function_single(cpu, cp, 0);
522 return 0;
523 }
524 return 1;
525}
526#else
527static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
528{
529 return 1;
530}
531#endif
532
533
534
535
536
537
538
539
540
541
542
543
544
545void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
546{
547 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
548 __local_trigger(cp, softirq);
549}
550EXPORT_SYMBOL(__send_remote_softirq);
551
552
553
554
555
556
557
558
559
560
561void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
562{
563 unsigned long flags;
564 int this_cpu;
565
566 local_irq_save(flags);
567 this_cpu = smp_processor_id();
568 __send_remote_softirq(cp, cpu, this_cpu, softirq);
569 local_irq_restore(flags);
570}
571EXPORT_SYMBOL(send_remote_softirq);
572
573static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
574 unsigned long action, void *hcpu)
575{
576
577
578
579
580 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
581 int cpu = (unsigned long) hcpu;
582 int i;
583
584 local_irq_disable();
585 for (i = 0; i < NR_SOFTIRQS; i++) {
586 struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
587 struct list_head *local_head;
588
589 if (list_empty(head))
590 continue;
591
592 local_head = &__get_cpu_var(softirq_work_list[i]);
593 list_splice_init(head, local_head);
594 raise_softirq_irqoff(i);
595 }
596 local_irq_enable();
597 }
598
599 return NOTIFY_OK;
600}
601
602static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
603 .notifier_call = remote_softirq_cpu_notify,
604};
605
606void __init softirq_init(void)
607{
608 int cpu;
609
610 for_each_possible_cpu(cpu) {
611 int i;
612
613 per_cpu(tasklet_vec, cpu).tail =
614 &per_cpu(tasklet_vec, cpu).head;
615 per_cpu(tasklet_hi_vec, cpu).tail =
616 &per_cpu(tasklet_hi_vec, cpu).head;
617 for (i = 0; i < NR_SOFTIRQS; i++)
618 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
619 }
620
621 register_hotcpu_notifier(&remote_softirq_cpu_notifier);
622
623 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
624 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
625}
626
627static int ksoftirqd(void * __bind_cpu)
628{
629 set_current_state(TASK_INTERRUPTIBLE);
630
631 while (!kthread_should_stop()) {
632 preempt_disable();
633 if (!local_softirq_pending()) {
634 preempt_enable_no_resched();
635 schedule();
636 preempt_disable();
637 }
638
639 __set_current_state(TASK_RUNNING);
640
641 while (local_softirq_pending()) {
642
643
644
645 if (cpu_is_offline((long)__bind_cpu))
646 goto wait_to_die;
647 do_softirq();
648 preempt_enable_no_resched();
649 cond_resched();
650 preempt_disable();
651 rcu_qsctr_inc((long)__bind_cpu);
652 }
653 preempt_enable();
654 set_current_state(TASK_INTERRUPTIBLE);
655 }
656 __set_current_state(TASK_RUNNING);
657 return 0;
658
659wait_to_die:
660 preempt_enable();
661
662 set_current_state(TASK_INTERRUPTIBLE);
663 while (!kthread_should_stop()) {
664 schedule();
665 set_current_state(TASK_INTERRUPTIBLE);
666 }
667 __set_current_state(TASK_RUNNING);
668 return 0;
669}
670
671#ifdef CONFIG_HOTPLUG_CPU
672
673
674
675
676
677
678
679
680
681void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
682{
683 struct tasklet_struct **i;
684
685 BUG_ON(cpu_online(cpu));
686 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
687
688 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
689 return;
690
691
692 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
693 if (*i == t) {
694 *i = t->next;
695
696 if (*i == NULL)
697 per_cpu(tasklet_vec, cpu).tail = i;
698 return;
699 }
700 }
701 BUG();
702}
703
704static void takeover_tasklets(unsigned int cpu)
705{
706
707 local_irq_disable();
708
709
710 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
711 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
712 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
713 per_cpu(tasklet_vec, cpu).head = NULL;
714 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
715 }
716 raise_softirq_irqoff(TASKLET_SOFTIRQ);
717
718 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
719 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
720 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
721 per_cpu(tasklet_hi_vec, cpu).head = NULL;
722 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
723 }
724 raise_softirq_irqoff(HI_SOFTIRQ);
725
726 local_irq_enable();
727}
728#endif
729
730static int __cpuinit cpu_callback(struct notifier_block *nfb,
731 unsigned long action,
732 void *hcpu)
733{
734 int hotcpu = (unsigned long)hcpu;
735 struct task_struct *p;
736
737 switch (action) {
738 case CPU_UP_PREPARE:
739 case CPU_UP_PREPARE_FROZEN:
740 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
741 if (IS_ERR(p)) {
742 printk("ksoftirqd for %i failed\n", hotcpu);
743 return NOTIFY_BAD;
744 }
745 kthread_bind(p, hotcpu);
746 per_cpu(ksoftirqd, hotcpu) = p;
747 break;
748 case CPU_ONLINE:
749 case CPU_ONLINE_FROZEN:
750 wake_up_process(per_cpu(ksoftirqd, hotcpu));
751 break;
752#ifdef CONFIG_HOTPLUG_CPU
753 case CPU_UP_CANCELED:
754 case CPU_UP_CANCELED_FROZEN:
755 if (!per_cpu(ksoftirqd, hotcpu))
756 break;
757
758 kthread_bind(per_cpu(ksoftirqd, hotcpu),
759 cpumask_any(cpu_online_mask));
760 case CPU_DEAD:
761 case CPU_DEAD_FROZEN: {
762 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
763
764 p = per_cpu(ksoftirqd, hotcpu);
765 per_cpu(ksoftirqd, hotcpu) = NULL;
766 sched_setscheduler_nocheck(p, SCHED_FIFO, ¶m);
767 kthread_stop(p);
768 takeover_tasklets(hotcpu);
769 break;
770 }
771#endif
772 }
773 return NOTIFY_OK;
774}
775
776static struct notifier_block __cpuinitdata cpu_nfb = {
777 .notifier_call = cpu_callback
778};
779
780static __init int spawn_ksoftirqd(void)
781{
782 void *cpu = (void *)(long)smp_processor_id();
783 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
784
785 BUG_ON(err == NOTIFY_BAD);
786 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
787 register_cpu_notifier(&cpu_nfb);
788 return 0;
789}
790early_initcall(spawn_ksoftirqd);
791
792#ifdef CONFIG_SMP
793
794
795
796int on_each_cpu(void (*func) (void *info), void *info, int wait)
797{
798 int ret = 0;
799
800 preempt_disable();
801 ret = smp_call_function(func, info, wait);
802 local_irq_disable();
803 func(info);
804 local_irq_enable();
805 preempt_enable();
806 return ret;
807}
808EXPORT_SYMBOL(on_each_cpu);
809#endif
810
811
812
813
814
815
816int __init __weak early_irq_init(void)
817{
818 return 0;
819}
820
821int __init __weak arch_probe_nr_irqs(void)
822{
823 return 0;
824}
825
826int __init __weak arch_early_irq_init(void)
827{
828 return 0;
829}
830
831int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
832{
833 return 0;
834}
835