1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/kernel/softirq.c
4 *
5 * Copyright (C) 1992 Linus Torvalds
6 *
7 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
8 */
9
10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12#include <linux/export.h>
13#include <linux/kernel_stat.h>
14#include <linux/interrupt.h>
15#include <linux/init.h>
16#include <linux/local_lock.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
21#include <linux/freezer.h>
22#include <linux/kthread.h>
23#include <linux/rcupdate.h>
24#include <linux/ftrace.h>
25#include <linux/smp.h>
26#include <linux/smpboot.h>
27#include <linux/tick.h>
28#include <linux/irq.h>
29#include <linux/wait_bit.h>
30#include <linux/workqueue.h>
31
32#include <asm/softirq_stack.h>
33
34#define CREATE_TRACE_POINTS
35#include <trace/events/irq.h>
36
37/*
38 - No shared variables, all the data are CPU local.
39 - If a softirq needs serialization, let it serialize itself
40 by its own spinlocks.
41 - Even if softirq is serialized, only local cpu is marked for
42 execution. Hence, we get something sort of weak cpu binding.
43 Though it is still not clear, will it result in better locality
44 or will not.
45
46 Examples:
47 - NET RX softirq. It is multithreaded and does not require
48 any global serialization.
49 - NET TX softirq. It kicks software netdevice queues, hence
50 it is logically serialized per device, but this serialization
51 is invisible to common code.
52 - Tasklets: serialized wrt itself.
53 */
54
55#ifndef __ARCH_IRQ_STAT
56DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
57EXPORT_PER_CPU_SYMBOL(irq_stat);
58#endif
59
60static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
61
62DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
63
64const char * const softirq_to_name[NR_SOFTIRQS] = {
65 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
66 "TASKLET", "SCHED", "HRTIMER", "RCU"
67};
68
69/*
70 * we cannot loop indefinitely here to avoid userspace starvation,
71 * but we also don't want to introduce a worst case 1/HZ latency
72 * to the pending events, so lets the scheduler to balance
73 * the softirq load for us.
74 */
75static void wakeup_softirqd(void)
76{
77 /* Interrupts are disabled: no need to stop preemption */
78 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
79
80 if (tsk)
81 wake_up_process(tsk);
82}
83
84#ifdef CONFIG_TRACE_IRQFLAGS
85DEFINE_PER_CPU(int, hardirqs_enabled);
86DEFINE_PER_CPU(int, hardirq_context);
87EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
88EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
89#endif
90
91/*
92 * SOFTIRQ_OFFSET usage:
93 *
94 * On !RT kernels 'count' is the preempt counter, on RT kernels this applies
95 * to a per CPU counter and to task::softirqs_disabled_cnt.
96 *
97 * - count is changed by SOFTIRQ_OFFSET on entering or leaving softirq
98 * processing.
99 *
100 * - count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
101 * on local_bh_disable or local_bh_enable.
102 *
103 * This lets us distinguish between whether we are currently processing
104 * softirq and whether we just have bh disabled.
105 */
106#ifdef CONFIG_PREEMPT_RT
107
108/*
109 * RT accounts for BH disabled sections in task::softirqs_disabled_cnt and
110 * also in per CPU softirq_ctrl::cnt. This is necessary to allow tasks in a
111 * softirq disabled section to be preempted.
112 *
113 * The per task counter is used for softirq_count(), in_softirq() and
114 * in_serving_softirqs() because these counts are only valid when the task
115 * holding softirq_ctrl::lock is running.
116 *
117 * The per CPU counter prevents pointless wakeups of ksoftirqd in case that
118 * the task which is in a softirq disabled section is preempted or blocks.
119 */
120struct softirq_ctrl {
121 local_lock_t lock;
122 int cnt;
123};
124
125static DEFINE_PER_CPU(struct softirq_ctrl, softirq_ctrl) = {
126 .lock = INIT_LOCAL_LOCK(softirq_ctrl.lock),
127};
128
129#ifdef CONFIG_DEBUG_LOCK_ALLOC
130static struct lock_class_key bh_lock_key;
131struct lockdep_map bh_lock_map = {
132 .name = "local_bh",
133 .key = &bh_lock_key,
134 .wait_type_outer = LD_WAIT_FREE,
135 .wait_type_inner = LD_WAIT_CONFIG, /* PREEMPT_RT makes BH preemptible. */
136 .lock_type = LD_LOCK_PERCPU,
137};
138EXPORT_SYMBOL_GPL(bh_lock_map);
139#endif
140
141/**
142 * local_bh_blocked() - Check for idle whether BH processing is blocked
143 *
144 * Returns false if the per CPU softirq::cnt is 0 otherwise true.
145 *
146 * This is invoked from the idle task to guard against false positive
147 * softirq pending warnings, which would happen when the task which holds
148 * softirq_ctrl::lock was the only running task on the CPU and blocks on
149 * some other lock.
150 */
151bool local_bh_blocked(void)
152{
153 return __this_cpu_read(softirq_ctrl.cnt) != 0;
154}
155
156void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
157{
158 unsigned long flags;
159 int newcnt;
160
161 WARN_ON_ONCE(in_hardirq());
162
163 lock_map_acquire_read(&bh_lock_map);
164
165 /* First entry of a task into a BH disabled section? */
166 if (!current->softirq_disable_cnt) {
167 if (preemptible()) {
168 if (IS_ENABLED(CONFIG_PREEMPT_RT_NEEDS_BH_LOCK))
169 local_lock(&softirq_ctrl.lock);
170 else
171 migrate_disable();
172
173 /* Required to meet the RCU bottomhalf requirements. */
174 rcu_read_lock();
175 } else {
176 DEBUG_LOCKS_WARN_ON(this_cpu_read(softirq_ctrl.cnt));
177 }
178 }
179
180 /*
181 * Track the per CPU softirq disabled state. On RT this is per CPU
182 * state to allow preemption of bottom half disabled sections.
183 */
184 if (IS_ENABLED(CONFIG_PREEMPT_RT_NEEDS_BH_LOCK)) {
185 newcnt = this_cpu_add_return(softirq_ctrl.cnt, cnt);
186 /*
187 * Reflect the result in the task state to prevent recursion on the
188 * local lock and to make softirq_count() & al work.
189 */
190 current->softirq_disable_cnt = newcnt;
191
192 if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && newcnt == cnt) {
193 raw_local_irq_save(flags);
194 lockdep_softirqs_off(ip);
195 raw_local_irq_restore(flags);
196 }
197 } else {
198 bool sirq_dis = false;
199
200 if (!current->softirq_disable_cnt)
201 sirq_dis = true;
202
203 this_cpu_add(softirq_ctrl.cnt, cnt);
204 current->softirq_disable_cnt += cnt;
205 WARN_ON_ONCE(current->softirq_disable_cnt < 0);
206
207 if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && sirq_dis) {
208 raw_local_irq_save(flags);
209 lockdep_softirqs_off(ip);
210 raw_local_irq_restore(flags);
211 }
212 }
213}
214EXPORT_SYMBOL(__local_bh_disable_ip);
215
216static void __local_bh_enable(unsigned int cnt, bool unlock)
217{
218 unsigned long flags;
219 bool sirq_en = false;
220 int newcnt;
221
222 if (IS_ENABLED(CONFIG_PREEMPT_RT_NEEDS_BH_LOCK)) {
223 DEBUG_LOCKS_WARN_ON(current->softirq_disable_cnt !=
224 this_cpu_read(softirq_ctrl.cnt));
225 if (softirq_count() == cnt)
226 sirq_en = true;
227 } else {
228 if (current->softirq_disable_cnt == cnt)
229 sirq_en = true;
230 }
231
232 if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && sirq_en) {
233 raw_local_irq_save(flags);
234 lockdep_softirqs_on(_RET_IP_);
235 raw_local_irq_restore(flags);
236 }
237
238 if (IS_ENABLED(CONFIG_PREEMPT_RT_NEEDS_BH_LOCK)) {
239 newcnt = this_cpu_sub_return(softirq_ctrl.cnt, cnt);
240 current->softirq_disable_cnt = newcnt;
241
242 if (!newcnt && unlock) {
243 rcu_read_unlock();
244 local_unlock(&softirq_ctrl.lock);
245 }
246 } else {
247 current->softirq_disable_cnt -= cnt;
248 this_cpu_sub(softirq_ctrl.cnt, cnt);
249 if (unlock && !current->softirq_disable_cnt) {
250 migrate_enable();
251 rcu_read_unlock();
252 } else {
253 WARN_ON_ONCE(current->softirq_disable_cnt < 0);
254 }
255 }
256}
257
258void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
259{
260 bool preempt_on = preemptible();
261 unsigned long flags;
262 u32 pending;
263 int curcnt;
264
265 WARN_ON_ONCE(in_hardirq());
266 lockdep_assert_irqs_enabled();
267
268 lock_map_release(&bh_lock_map);
269
270 local_irq_save(flags);
271 if (IS_ENABLED(CONFIG_PREEMPT_RT_NEEDS_BH_LOCK))
272 curcnt = this_cpu_read(softirq_ctrl.cnt);
273 else
274 curcnt = current->softirq_disable_cnt;
275
276 /*
277 * If this is not reenabling soft interrupts, no point in trying to
278 * run pending ones.
279 */
280 if (curcnt != cnt)
281 goto out;
282
283 pending = local_softirq_pending();
284 if (!pending)
285 goto out;
286
287 /*
288 * If this was called from non preemptible context, wake up the
289 * softirq daemon.
290 */
291 if (!preempt_on) {
292 wakeup_softirqd();
293 goto out;
294 }
295
296 /*
297 * Adjust softirq count to SOFTIRQ_OFFSET which makes
298 * in_serving_softirq() become true.
299 */
300 cnt = SOFTIRQ_OFFSET;
301 __local_bh_enable(cnt, false);
302 __do_softirq();
303
304out:
305 __local_bh_enable(cnt, preempt_on);
306 local_irq_restore(flags);
307}
308EXPORT_SYMBOL(__local_bh_enable_ip);
309
310/*
311 * Invoked from ksoftirqd_run() outside of the interrupt disabled section
312 * to acquire the per CPU local lock for reentrancy protection.
313 */
314static inline void ksoftirqd_run_begin(void)
315{
316 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
317 local_irq_disable();
318}
319
320/* Counterpart to ksoftirqd_run_begin() */
321static inline void ksoftirqd_run_end(void)
322{
323 /* pairs with the lock_map_acquire_read() in ksoftirqd_run_begin() */
324 lock_map_release(&bh_lock_map);
325 __local_bh_enable(SOFTIRQ_OFFSET, true);
326 WARN_ON_ONCE(in_interrupt());
327 local_irq_enable();
328}
329
330static inline void softirq_handle_begin(void) { }
331static inline void softirq_handle_end(void) { }
332
333static inline bool should_wake_ksoftirqd(void)
334{
335 return !this_cpu_read(softirq_ctrl.cnt);
336}
337
338static inline void invoke_softirq(void)
339{
340 if (should_wake_ksoftirqd())
341 wakeup_softirqd();
342}
343
344#define SCHED_SOFTIRQ_MASK BIT(SCHED_SOFTIRQ)
345
346/*
347 * flush_smp_call_function_queue() can raise a soft interrupt in a function
348 * call. On RT kernels this is undesired and the only known functionalities
349 * are in the block layer which is disabled on RT, and in the scheduler for
350 * idle load balancing. If soft interrupts get raised which haven't been
351 * raised before the flush, warn if it is not a SCHED_SOFTIRQ so it can be
352 * investigated.
353 */
354void do_softirq_post_smp_call_flush(unsigned int was_pending)
355{
356 unsigned int is_pending = local_softirq_pending();
357
358 if (unlikely(was_pending != is_pending)) {
359 WARN_ON_ONCE(was_pending != (is_pending & ~SCHED_SOFTIRQ_MASK));
360 invoke_softirq();
361 }
362}
363
364#else /* CONFIG_PREEMPT_RT */
365
366/*
367 * This one is for softirq.c-internal use, where hardirqs are disabled
368 * legitimately:
369 */
370#ifdef CONFIG_TRACE_IRQFLAGS
371void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
372{
373 unsigned long flags;
374
375 WARN_ON_ONCE(in_hardirq());
376
377 raw_local_irq_save(flags);
378 /*
379 * The preempt tracer hooks into preempt_count_add and will break
380 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
381 * is set and before current->softirq_enabled is cleared.
382 * We must manually increment preempt_count here and manually
383 * call the trace_preempt_off later.
384 */
385 __preempt_count_add(cnt);
386 /*
387 * Were softirqs turned off above:
388 */
389 if (softirq_count() == (cnt & SOFTIRQ_MASK))
390 lockdep_softirqs_off(ip);
391 raw_local_irq_restore(flags);
392
393 if (preempt_count() == cnt) {
394#ifdef CONFIG_DEBUG_PREEMPT
395 current->preempt_disable_ip = get_lock_parent_ip();
396#endif
397 trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
398 }
399}
400EXPORT_SYMBOL(__local_bh_disable_ip);
401#endif /* CONFIG_TRACE_IRQFLAGS */
402
403static void __local_bh_enable(unsigned int cnt)
404{
405 lockdep_assert_irqs_disabled();
406
407 if (preempt_count() == cnt)
408 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
409
410 if (softirq_count() == (cnt & SOFTIRQ_MASK))
411 lockdep_softirqs_on(_RET_IP_);
412
413 __preempt_count_sub(val: cnt);
414}
415
416/*
417 * Special-case - softirqs can safely be enabled by __do_softirq(),
418 * without processing still-pending softirqs:
419 */
420void _local_bh_enable(void)
421{
422 WARN_ON_ONCE(in_hardirq());
423 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
424}
425EXPORT_SYMBOL(_local_bh_enable);
426
427void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
428{
429 WARN_ON_ONCE(in_hardirq());
430 lockdep_assert_irqs_enabled();
431#ifdef CONFIG_TRACE_IRQFLAGS
432 local_irq_disable();
433#endif
434 /*
435 * Are softirqs going to be turned on now:
436 */
437 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
438 lockdep_softirqs_on(ip);
439 /*
440 * Keep preemption disabled until we are done with
441 * softirq processing:
442 */
443 __preempt_count_sub(val: cnt - 1);
444
445 if (unlikely(!in_interrupt() && local_softirq_pending())) {
446 /*
447 * Run softirq if any pending. And do it in its own stack
448 * as we may be calling this deep in a task call stack already.
449 */
450 do_softirq();
451 }
452
453 preempt_count_dec();
454#ifdef CONFIG_TRACE_IRQFLAGS
455 local_irq_enable();
456#endif
457 preempt_check_resched();
458}
459EXPORT_SYMBOL(__local_bh_enable_ip);
460
461static inline void softirq_handle_begin(void)
462{
463 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
464}
465
466static inline void softirq_handle_end(void)
467{
468 __local_bh_enable(SOFTIRQ_OFFSET);
469 WARN_ON_ONCE(in_interrupt());
470}
471
472static inline void ksoftirqd_run_begin(void)
473{
474 local_irq_disable();
475}
476
477static inline void ksoftirqd_run_end(void)
478{
479 local_irq_enable();
480}
481
482static inline bool should_wake_ksoftirqd(void)
483{
484 return true;
485}
486
487static inline void invoke_softirq(void)
488{
489 if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) {
490#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
491 /*
492 * We can safely execute softirq on the current stack if
493 * it is the irq stack, because it should be near empty
494 * at this stage.
495 */
496 __do_softirq();
497#else
498 /*
499 * Otherwise, irq_exit() is called on the task stack that can
500 * be potentially deep already. So call softirq in its own stack
501 * to prevent from any overrun.
502 */
503 do_softirq_own_stack();
504#endif
505 } else {
506 wakeup_softirqd();
507 }
508}
509
510asmlinkage __visible void do_softirq(void)
511{
512 __u32 pending;
513 unsigned long flags;
514
515 if (in_interrupt())
516 return;
517
518 local_irq_save(flags);
519
520 pending = local_softirq_pending();
521
522 if (pending)
523 do_softirq_own_stack();
524
525 local_irq_restore(flags);
526}
527
528#endif /* !CONFIG_PREEMPT_RT */
529
530/*
531 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
532 * but break the loop if need_resched() is set or after 2 ms.
533 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
534 * certain cases, such as stop_machine(), jiffies may cease to
535 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
536 * well to make sure we eventually return from this method.
537 *
538 * These limits have been established via experimentation.
539 * The two things to balance is latency against fairness -
540 * we want to handle softirqs as soon as possible, but they
541 * should not be able to lock up the box.
542 */
543#define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
544#define MAX_SOFTIRQ_RESTART 10
545
546#ifdef CONFIG_TRACE_IRQFLAGS
547/*
548 * When we run softirqs from irq_exit() and thus on the hardirq stack we need
549 * to keep the lockdep irq context tracking as tight as possible in order to
550 * not miss-qualify lock contexts and miss possible deadlocks.
551 */
552
553static inline bool lockdep_softirq_start(void)
554{
555 bool in_hardirq = false;
556
557 if (lockdep_hardirq_context()) {
558 in_hardirq = true;
559 lockdep_hardirq_exit();
560 }
561
562 lockdep_softirq_enter();
563
564 return in_hardirq;
565}
566
567static inline void lockdep_softirq_end(bool in_hardirq)
568{
569 lockdep_softirq_exit();
570
571 if (in_hardirq)
572 lockdep_hardirq_enter();
573}
574#else
575static inline bool lockdep_softirq_start(void) { return false; }
576static inline void lockdep_softirq_end(bool in_hardirq) { }
577#endif
578
579static void handle_softirqs(bool ksirqd)
580{
581 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
582 unsigned long old_flags = current->flags;
583 int max_restart = MAX_SOFTIRQ_RESTART;
584 struct softirq_action *h;
585 bool in_hardirq;
586 __u32 pending;
587 int softirq_bit;
588
589 /*
590 * Mask out PF_MEMALLOC as the current task context is borrowed for the
591 * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
592 * again if the socket is related to swapping.
593 */
594 current->flags &= ~PF_MEMALLOC;
595
596 pending = local_softirq_pending();
597
598 softirq_handle_begin();
599 in_hardirq = lockdep_softirq_start();
600 account_softirq_enter(current);
601
602restart:
603 /* Reset the pending bitmask before enabling irqs */
604 set_softirq_pending(0);
605
606 local_irq_enable();
607
608 h = softirq_vec;
609
610 while ((softirq_bit = ffs(pending))) {
611 unsigned int vec_nr;
612 int prev_count;
613
614 h += softirq_bit - 1;
615
616 vec_nr = h - softirq_vec;
617 prev_count = preempt_count();
618
619 kstat_incr_softirqs_this_cpu(irq: vec_nr);
620
621 trace_softirq_entry(vec_nr);
622 h->action();
623 trace_softirq_exit(vec_nr);
624 if (unlikely(prev_count != preempt_count())) {
625 pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
626 vec_nr, softirq_to_name[vec_nr], h->action,
627 prev_count, preempt_count());
628 preempt_count_set(pc: prev_count);
629 }
630 h++;
631 pending >>= softirq_bit;
632 }
633
634 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && ksirqd)
635 rcu_softirq_qs();
636
637 local_irq_disable();
638
639 pending = local_softirq_pending();
640 if (pending) {
641 if (time_before(jiffies, end) && !need_resched() &&
642 --max_restart)
643 goto restart;
644
645 wakeup_softirqd();
646 }
647
648 account_softirq_exit(current);
649 lockdep_softirq_end(in_hardirq);
650 softirq_handle_end();
651 current_restore_flags(orig_flags: old_flags, PF_MEMALLOC);
652}
653
654asmlinkage __visible void __softirq_entry __do_softirq(void)
655{
656 handle_softirqs(ksirqd: false);
657}
658
659/**
660 * irq_enter_rcu - Enter an interrupt context with RCU watching
661 */
662void irq_enter_rcu(void)
663{
664 __irq_enter_raw();
665
666 if (tick_nohz_full_cpu(smp_processor_id()) ||
667 (is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET)))
668 tick_irq_enter();
669
670 account_hardirq_enter(current);
671}
672
673/**
674 * irq_enter - Enter an interrupt context including RCU update
675 */
676void irq_enter(void)
677{
678 ct_irq_enter();
679 irq_enter_rcu();
680}
681
682static inline void tick_irq_exit(void)
683{
684#ifdef CONFIG_NO_HZ_COMMON
685 int cpu = smp_processor_id();
686
687 /* Make sure that timer wheel updates are propagated */
688 if ((sched_core_idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
689 if (!in_hardirq())
690 tick_nohz_irq_exit();
691 }
692#endif
693}
694
695#ifdef CONFIG_IRQ_FORCED_THREADING
696DEFINE_PER_CPU(struct task_struct *, ktimerd);
697DEFINE_PER_CPU(unsigned long, pending_timer_softirq);
698
699static void wake_timersd(void)
700{
701 struct task_struct *tsk = __this_cpu_read(ktimerd);
702
703 if (tsk)
704 wake_up_process(tsk);
705}
706
707#else
708
709static inline void wake_timersd(void) { }
710
711#endif
712
713static inline void __irq_exit_rcu(void)
714{
715#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
716 local_irq_disable();
717#else
718 lockdep_assert_irqs_disabled();
719#endif
720 account_hardirq_exit(current);
721 preempt_count_sub(HARDIRQ_OFFSET);
722 if (!in_interrupt() && local_softirq_pending())
723 invoke_softirq();
724
725 if (IS_ENABLED(CONFIG_IRQ_FORCED_THREADING) && force_irqthreads() &&
726 local_timers_pending_force_th() && !(in_nmi() | in_hardirq()))
727 wake_timersd();
728
729 tick_irq_exit();
730}
731
732/**
733 * irq_exit_rcu() - Exit an interrupt context without updating RCU
734 *
735 * Also processes softirqs if needed and possible.
736 */
737void irq_exit_rcu(void)
738{
739 __irq_exit_rcu();
740 /* must be last! */
741 lockdep_hardirq_exit();
742}
743
744/**
745 * irq_exit - Exit an interrupt context, update RCU and lockdep
746 *
747 * Also processes softirqs if needed and possible.
748 */
749void irq_exit(void)
750{
751 __irq_exit_rcu();
752 ct_irq_exit();
753 /* must be last! */
754 lockdep_hardirq_exit();
755}
756
757/*
758 * This function must run with irqs disabled!
759 */
760inline void raise_softirq_irqoff(unsigned int nr)
761{
762 __raise_softirq_irqoff(nr);
763
764 /*
765 * If we're in an interrupt or softirq, we're done
766 * (this also catches softirq-disabled code). We will
767 * actually run the softirq once we return from
768 * the irq or softirq.
769 *
770 * Otherwise we wake up ksoftirqd to make sure we
771 * schedule the softirq soon.
772 */
773 if (!in_interrupt() && should_wake_ksoftirqd())
774 wakeup_softirqd();
775}
776
777void raise_softirq(unsigned int nr)
778{
779 unsigned long flags;
780
781 local_irq_save(flags);
782 raise_softirq_irqoff(nr);
783 local_irq_restore(flags);
784}
785
786void __raise_softirq_irqoff(unsigned int nr)
787{
788 lockdep_assert_irqs_disabled();
789 trace_softirq_raise(vec_nr: nr);
790 or_softirq_pending(1UL << nr);
791}
792
793void open_softirq(int nr, void (*action)(void))
794{
795 softirq_vec[nr].action = action;
796}
797
798/*
799 * Tasklets
800 */
801struct tasklet_head {
802 struct tasklet_struct *head;
803 struct tasklet_struct **tail;
804};
805
806static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
807static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
808
809static void __tasklet_schedule_common(struct tasklet_struct *t,
810 struct tasklet_head __percpu *headp,
811 unsigned int softirq_nr)
812{
813 struct tasklet_head *head;
814 unsigned long flags;
815
816 local_irq_save(flags);
817 head = this_cpu_ptr(headp);
818 t->next = NULL;
819 *head->tail = t;
820 head->tail = &(t->next);
821 raise_softirq_irqoff(nr: softirq_nr);
822 local_irq_restore(flags);
823}
824
825void __tasklet_schedule(struct tasklet_struct *t)
826{
827 __tasklet_schedule_common(t, headp: &tasklet_vec,
828 softirq_nr: TASKLET_SOFTIRQ);
829}
830EXPORT_SYMBOL(__tasklet_schedule);
831
832void __tasklet_hi_schedule(struct tasklet_struct *t)
833{
834 __tasklet_schedule_common(t, headp: &tasklet_hi_vec,
835 softirq_nr: HI_SOFTIRQ);
836}
837EXPORT_SYMBOL(__tasklet_hi_schedule);
838
839static bool tasklet_clear_sched(struct tasklet_struct *t)
840{
841 if (test_and_clear_wake_up_bit(bit: TASKLET_STATE_SCHED, word: &t->state))
842 return true;
843
844 WARN_ONCE(1, "tasklet SCHED state not set: %s %pS\n",
845 t->use_callback ? "callback" : "func",
846 t->use_callback ? (void *)t->callback : (void *)t->func);
847
848 return false;
849}
850
851#ifdef CONFIG_PREEMPT_RT
852struct tasklet_sync_callback {
853 spinlock_t cb_lock;
854 atomic_t cb_waiters;
855};
856
857static DEFINE_PER_CPU(struct tasklet_sync_callback, tasklet_sync_callback) = {
858 .cb_lock = __SPIN_LOCK_UNLOCKED(tasklet_sync_callback.cb_lock),
859 .cb_waiters = ATOMIC_INIT(0),
860};
861
862static void tasklet_lock_callback(void)
863{
864 spin_lock(this_cpu_ptr(&tasklet_sync_callback.cb_lock));
865}
866
867static void tasklet_unlock_callback(void)
868{
869 spin_unlock(this_cpu_ptr(&tasklet_sync_callback.cb_lock));
870}
871
872static void tasklet_callback_cancel_wait_running(void)
873{
874 struct tasklet_sync_callback *sync_cb = this_cpu_ptr(&tasklet_sync_callback);
875
876 atomic_inc(&sync_cb->cb_waiters);
877 spin_lock(&sync_cb->cb_lock);
878 atomic_dec(&sync_cb->cb_waiters);
879 spin_unlock(&sync_cb->cb_lock);
880}
881
882static void tasklet_callback_sync_wait_running(void)
883{
884 struct tasklet_sync_callback *sync_cb = this_cpu_ptr(&tasklet_sync_callback);
885
886 if (atomic_read(&sync_cb->cb_waiters)) {
887 spin_unlock(&sync_cb->cb_lock);
888 spin_lock(&sync_cb->cb_lock);
889 }
890}
891
892#else /* !CONFIG_PREEMPT_RT: */
893
894static void tasklet_lock_callback(void) { }
895static void tasklet_unlock_callback(void) { }
896static void tasklet_callback_sync_wait_running(void) { }
897
898#ifdef CONFIG_SMP
899static void tasklet_callback_cancel_wait_running(void) { }
900#endif
901#endif /* !CONFIG_PREEMPT_RT */
902
903static void tasklet_action_common(struct tasklet_head *tl_head,
904 unsigned int softirq_nr)
905{
906 struct tasklet_struct *list;
907
908 local_irq_disable();
909 list = tl_head->head;
910 tl_head->head = NULL;
911 tl_head->tail = &tl_head->head;
912 local_irq_enable();
913
914 tasklet_lock_callback();
915 while (list) {
916 struct tasklet_struct *t = list;
917
918 list = list->next;
919
920 if (tasklet_trylock(t)) {
921 if (!atomic_read(v: &t->count)) {
922 if (tasklet_clear_sched(t)) {
923 if (t->use_callback) {
924 trace_tasklet_entry(t, func: t->callback);
925 t->callback(t);
926 trace_tasklet_exit(t, func: t->callback);
927 } else {
928 trace_tasklet_entry(t, func: t->func);
929 t->func(t->data);
930 trace_tasklet_exit(t, func: t->func);
931 }
932 }
933 tasklet_unlock(t);
934 tasklet_callback_sync_wait_running();
935 continue;
936 }
937 tasklet_unlock(t);
938 }
939
940 local_irq_disable();
941 t->next = NULL;
942 *tl_head->tail = t;
943 tl_head->tail = &t->next;
944 __raise_softirq_irqoff(nr: softirq_nr);
945 local_irq_enable();
946 }
947 tasklet_unlock_callback();
948}
949
950static __latent_entropy void tasklet_action(void)
951{
952 workqueue_softirq_action(highpri: false);
953 tasklet_action_common(this_cpu_ptr(&tasklet_vec), softirq_nr: TASKLET_SOFTIRQ);
954}
955
956static __latent_entropy void tasklet_hi_action(void)
957{
958 workqueue_softirq_action(highpri: true);
959 tasklet_action_common(this_cpu_ptr(&tasklet_hi_vec), softirq_nr: HI_SOFTIRQ);
960}
961
962void tasklet_setup(struct tasklet_struct *t,
963 void (*callback)(struct tasklet_struct *))
964{
965 t->next = NULL;
966 t->state = 0;
967 atomic_set(v: &t->count, i: 0);
968 t->callback = callback;
969 t->use_callback = true;
970 t->data = 0;
971}
972EXPORT_SYMBOL(tasklet_setup);
973
974void tasklet_init(struct tasklet_struct *t,
975 void (*func)(unsigned long), unsigned long data)
976{
977 t->next = NULL;
978 t->state = 0;
979 atomic_set(v: &t->count, i: 0);
980 t->func = func;
981 t->use_callback = false;
982 t->data = data;
983}
984EXPORT_SYMBOL(tasklet_init);
985
986#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
987/*
988 * Do not use in new code. Waiting for tasklets from atomic contexts is
989 * error prone and should be avoided.
990 */
991void tasklet_unlock_spin_wait(struct tasklet_struct *t)
992{
993 while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
994 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
995 /*
996 * Prevent a live lock when current preempted soft
997 * interrupt processing or prevents ksoftirqd from
998 * running.
999 */
1000 tasklet_callback_cancel_wait_running();
1001 } else {
1002 cpu_relax();
1003 }
1004 }
1005}
1006EXPORT_SYMBOL(tasklet_unlock_spin_wait);
1007#endif
1008
1009void tasklet_kill(struct tasklet_struct *t)
1010{
1011 if (in_interrupt())
1012 pr_notice("Attempt to kill tasklet from interrupt\n");
1013
1014 wait_on_bit_lock(word: &t->state, bit: TASKLET_STATE_SCHED, TASK_UNINTERRUPTIBLE);
1015
1016 tasklet_unlock_wait(t);
1017 tasklet_clear_sched(t);
1018}
1019EXPORT_SYMBOL(tasklet_kill);
1020
1021#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
1022void tasklet_unlock(struct tasklet_struct *t)
1023{
1024 clear_and_wake_up_bit(bit: TASKLET_STATE_RUN, word: &t->state);
1025}
1026EXPORT_SYMBOL_GPL(tasklet_unlock);
1027
1028void tasklet_unlock_wait(struct tasklet_struct *t)
1029{
1030 wait_on_bit(word: &t->state, bit: TASKLET_STATE_RUN, TASK_UNINTERRUPTIBLE);
1031}
1032EXPORT_SYMBOL_GPL(tasklet_unlock_wait);
1033#endif
1034
1035void __init softirq_init(void)
1036{
1037 int cpu;
1038
1039 for_each_possible_cpu(cpu) {
1040 per_cpu(tasklet_vec, cpu).tail =
1041 &per_cpu(tasklet_vec, cpu).head;
1042 per_cpu(tasklet_hi_vec, cpu).tail =
1043 &per_cpu(tasklet_hi_vec, cpu).head;
1044 }
1045
1046 open_softirq(nr: TASKLET_SOFTIRQ, action: tasklet_action);
1047 open_softirq(nr: HI_SOFTIRQ, action: tasklet_hi_action);
1048}
1049
1050static int ksoftirqd_should_run(unsigned int cpu)
1051{
1052 return local_softirq_pending();
1053}
1054
1055static void run_ksoftirqd(unsigned int cpu)
1056{
1057 ksoftirqd_run_begin();
1058 if (local_softirq_pending()) {
1059 /*
1060 * We can safely run softirq on inline stack, as we are not deep
1061 * in the task stack here.
1062 */
1063 handle_softirqs(ksirqd: true);
1064 ksoftirqd_run_end();
1065 cond_resched();
1066 return;
1067 }
1068 ksoftirqd_run_end();
1069}
1070
1071#ifdef CONFIG_HOTPLUG_CPU
1072static int takeover_tasklets(unsigned int cpu)
1073{
1074 workqueue_softirq_dead(cpu);
1075
1076 /* CPU is dead, so no lock needed. */
1077 local_irq_disable();
1078
1079 /* Find end, append list for that CPU. */
1080 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
1081 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
1082 __this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
1083 per_cpu(tasklet_vec, cpu).head = NULL;
1084 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
1085 }
1086 raise_softirq_irqoff(nr: TASKLET_SOFTIRQ);
1087
1088 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
1089 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
1090 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
1091 per_cpu(tasklet_hi_vec, cpu).head = NULL;
1092 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
1093 }
1094 raise_softirq_irqoff(nr: HI_SOFTIRQ);
1095
1096 local_irq_enable();
1097 return 0;
1098}
1099#else
1100#define takeover_tasklets NULL
1101#endif /* CONFIG_HOTPLUG_CPU */
1102
1103static struct smp_hotplug_thread softirq_threads = {
1104 .store = &ksoftirqd,
1105 .thread_should_run = ksoftirqd_should_run,
1106 .thread_fn = run_ksoftirqd,
1107 .thread_comm = "ksoftirqd/%u",
1108};
1109
1110#ifdef CONFIG_IRQ_FORCED_THREADING
1111static void ktimerd_setup(unsigned int cpu)
1112{
1113 /* Above SCHED_NORMAL to handle timers before regular tasks. */
1114 sched_set_fifo_low(current);
1115}
1116
1117static int ktimerd_should_run(unsigned int cpu)
1118{
1119 return local_timers_pending_force_th();
1120}
1121
1122void raise_ktimers_thread(unsigned int nr)
1123{
1124 trace_softirq_raise(vec_nr: nr);
1125 __this_cpu_or(pending_timer_softirq, BIT(nr));
1126}
1127
1128static void run_ktimerd(unsigned int cpu)
1129{
1130 unsigned int timer_si;
1131
1132 ksoftirqd_run_begin();
1133
1134 timer_si = local_timers_pending_force_th();
1135 __this_cpu_write(pending_timer_softirq, 0);
1136 or_softirq_pending(timer_si);
1137
1138 __do_softirq();
1139
1140 ksoftirqd_run_end();
1141}
1142
1143static struct smp_hotplug_thread timer_thread = {
1144 .store = &ktimerd,
1145 .setup = ktimerd_setup,
1146 .thread_should_run = ktimerd_should_run,
1147 .thread_fn = run_ktimerd,
1148 .thread_comm = "ktimers/%u",
1149};
1150#endif
1151
1152static __init int spawn_ksoftirqd(void)
1153{
1154 cpuhp_setup_state_nocalls(state: CPUHP_SOFTIRQ_DEAD, name: "softirq:dead", NULL,
1155 teardown: takeover_tasklets);
1156 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
1157#ifdef CONFIG_IRQ_FORCED_THREADING
1158 if (force_irqthreads())
1159 BUG_ON(smpboot_register_percpu_thread(&timer_thread));
1160#endif
1161 return 0;
1162}
1163early_initcall(spawn_ksoftirqd);
1164
1165/*
1166 * [ These __weak aliases are kept in a separate compilation unit, so that
1167 * GCC does not inline them incorrectly. ]
1168 */
1169
1170int __init __weak early_irq_init(void)
1171{
1172 return 0;
1173}
1174
1175int __init __weak arch_probe_nr_irqs(void)
1176{
1177 return NR_IRQS_LEGACY;
1178}
1179
1180int __init __weak arch_early_irq_init(void)
1181{
1182 return 0;
1183}
1184
1185unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
1186{
1187 return from;
1188}
1189