1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4 * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
5 *
6 * This file contains the core interrupt handling code, for irq-chip based
7 * architectures. Detailed information is available in
8 * Documentation/core-api/genericirq.rst
9 */
10
11#include <linux/irq.h>
12#include <linux/msi.h>
13#include <linux/module.h>
14#include <linux/interrupt.h>
15#include <linux/kernel_stat.h>
16#include <linux/irqdomain.h>
17
18#include <trace/events/irq.h>
19
20#include "internals.h"
21
22static irqreturn_t bad_chained_irq(int irq, void *dev_id)
23{
24 WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
25 return IRQ_NONE;
26}
27
28/*
29 * Chained handlers should never call action on their IRQ. This default
30 * action will emit warning if such thing happens.
31 */
32struct irqaction chained_action = {
33 .handler = bad_chained_irq,
34};
35
36/**
37 * irq_set_chip - set the irq chip for an irq
38 * @irq: irq number
39 * @chip: pointer to irq chip description structure
40 */
41int irq_set_chip(unsigned int irq, const struct irq_chip *chip)
42{
43 int ret = -EINVAL;
44
45 scoped_irqdesc_get_and_lock(irq, 0) {
46 scoped_irqdesc->irq_data.chip = (struct irq_chip *)(chip ?: &no_irq_chip);
47 ret = 0;
48 }
49 /* For !CONFIG_SPARSE_IRQ make the irq show up in allocated_irqs. */
50 if (!ret)
51 irq_mark_irq(irq);
52 return ret;
53}
54EXPORT_SYMBOL(irq_set_chip);
55
56/**
57 * irq_set_irq_type - set the irq trigger type for an irq
58 * @irq: irq number
59 * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
60 */
61int irq_set_irq_type(unsigned int irq, unsigned int type)
62{
63 scoped_irqdesc_get_and_buslock(irq, IRQ_GET_DESC_CHECK_GLOBAL)
64 return __irq_set_trigger(scoped_irqdesc, flags: type);
65 return -EINVAL;
66}
67EXPORT_SYMBOL(irq_set_irq_type);
68
69/**
70 * irq_set_handler_data - set irq handler data for an irq
71 * @irq: Interrupt number
72 * @data: Pointer to interrupt specific data
73 *
74 * Set the hardware irq controller data for an irq
75 */
76int irq_set_handler_data(unsigned int irq, void *data)
77{
78 scoped_irqdesc_get_and_lock(irq, 0) {
79 scoped_irqdesc->irq_common_data.handler_data = data;
80 return 0;
81 }
82 return -EINVAL;
83}
84EXPORT_SYMBOL(irq_set_handler_data);
85
86/**
87 * irq_set_msi_desc_off - set MSI descriptor data for an irq at offset
88 * @irq_base: Interrupt number base
89 * @irq_offset: Interrupt number offset
90 * @entry: Pointer to MSI descriptor data
91 *
92 * Set the MSI descriptor entry for an irq at offset
93 */
94int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset, struct msi_desc *entry)
95{
96 scoped_irqdesc_get_and_lock(irq_base + irq_offset, IRQ_GET_DESC_CHECK_GLOBAL) {
97 scoped_irqdesc->irq_common_data.msi_desc = entry;
98 if (entry && !irq_offset)
99 entry->irq = irq_base;
100 return 0;
101 }
102 return -EINVAL;
103}
104
105/**
106 * irq_set_msi_desc - set MSI descriptor data for an irq
107 * @irq: Interrupt number
108 * @entry: Pointer to MSI descriptor data
109 *
110 * Set the MSI descriptor entry for an irq
111 */
112int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
113{
114 return irq_set_msi_desc_off(irq_base: irq, irq_offset: 0, entry);
115}
116
117/**
118 * irq_set_chip_data - set irq chip data for an irq
119 * @irq: Interrupt number
120 * @data: Pointer to chip specific data
121 *
122 * Set the hardware irq chip data for an irq
123 */
124int irq_set_chip_data(unsigned int irq, void *data)
125{
126 scoped_irqdesc_get_and_lock(irq, 0) {
127 scoped_irqdesc->irq_data.chip_data = data;
128 return 0;
129 }
130 return -EINVAL;
131}
132EXPORT_SYMBOL(irq_set_chip_data);
133
134struct irq_data *irq_get_irq_data(unsigned int irq)
135{
136 struct irq_desc *desc = irq_to_desc(irq);
137
138 return desc ? &desc->irq_data : NULL;
139}
140EXPORT_SYMBOL_GPL(irq_get_irq_data);
141
142static void irq_state_clr_disabled(struct irq_desc *desc)
143{
144 irqd_clear(d: &desc->irq_data, mask: IRQD_IRQ_DISABLED);
145}
146
147static void irq_state_clr_masked(struct irq_desc *desc)
148{
149 irqd_clear(d: &desc->irq_data, mask: IRQD_IRQ_MASKED);
150}
151
152static void irq_state_clr_started(struct irq_desc *desc)
153{
154 irqd_clear(d: &desc->irq_data, mask: IRQD_IRQ_STARTED);
155}
156
157static void irq_state_set_started(struct irq_desc *desc)
158{
159 irqd_set(d: &desc->irq_data, mask: IRQD_IRQ_STARTED);
160}
161
162enum {
163 IRQ_STARTUP_NORMAL,
164 IRQ_STARTUP_MANAGED,
165 IRQ_STARTUP_ABORT,
166};
167
168#ifdef CONFIG_SMP
169static int
170__irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff,
171 bool force)
172{
173 struct irq_data *d = irq_desc_get_irq_data(desc);
174
175 if (!irqd_affinity_is_managed(d))
176 return IRQ_STARTUP_NORMAL;
177
178 irqd_clr_managed_shutdown(d);
179
180 if (!cpumask_intersects(src1p: aff, cpu_online_mask)) {
181 /*
182 * Catch code which fiddles with enable_irq() on a managed
183 * and potentially shutdown IRQ. Chained interrupt
184 * installment or irq auto probing should not happen on
185 * managed irqs either.
186 */
187 if (WARN_ON_ONCE(force))
188 return IRQ_STARTUP_ABORT;
189 /*
190 * The interrupt was requested, but there is no online CPU
191 * in it's affinity mask. Put it into managed shutdown
192 * state and let the cpu hotplug mechanism start it up once
193 * a CPU in the mask becomes available.
194 */
195 return IRQ_STARTUP_ABORT;
196 }
197 /*
198 * Managed interrupts have reserved resources, so this should not
199 * happen.
200 */
201 if (WARN_ON(irq_domain_activate_irq(d, false)))
202 return IRQ_STARTUP_ABORT;
203 return IRQ_STARTUP_MANAGED;
204}
205
206void irq_startup_managed(struct irq_desc *desc)
207{
208 struct irq_data *d = irq_desc_get_irq_data(desc);
209
210 /*
211 * Clear managed-shutdown flag, so we don't repeat managed-startup for
212 * multiple hotplugs, and cause imbalanced disable depth.
213 */
214 irqd_clr_managed_shutdown(d);
215
216 /*
217 * Only start it up when the disable depth is 1, so that a disable,
218 * hotunplug, hotplug sequence does not end up enabling it during
219 * hotplug unconditionally.
220 */
221 desc->depth--;
222 if (!desc->depth)
223 irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
224}
225
226#else
227static __always_inline int
228__irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff,
229 bool force)
230{
231 return IRQ_STARTUP_NORMAL;
232}
233#endif
234
235static void irq_enable(struct irq_desc *desc)
236{
237 if (!irqd_irq_disabled(d: &desc->irq_data)) {
238 unmask_irq(desc);
239 } else {
240 irq_state_clr_disabled(desc);
241 if (desc->irq_data.chip->irq_enable) {
242 desc->irq_data.chip->irq_enable(&desc->irq_data);
243 irq_state_clr_masked(desc);
244 } else {
245 unmask_irq(desc);
246 }
247 }
248}
249
250static int __irq_startup(struct irq_desc *desc)
251{
252 struct irq_data *d = irq_desc_get_irq_data(desc);
253 int ret = 0;
254
255 /* Warn if this interrupt is not activated but try nevertheless */
256 WARN_ON_ONCE(!irqd_is_activated(d));
257
258 if (d->chip->irq_startup) {
259 ret = d->chip->irq_startup(d);
260 irq_state_clr_disabled(desc);
261 irq_state_clr_masked(desc);
262 } else {
263 irq_enable(desc);
264 }
265 irq_state_set_started(desc);
266 return ret;
267}
268
269int irq_startup(struct irq_desc *desc, bool resend, bool force)
270{
271 struct irq_data *d = irq_desc_get_irq_data(desc);
272 const struct cpumask *aff = irq_data_get_affinity_mask(d);
273 int ret = 0;
274
275 desc->depth = 0;
276
277 if (irqd_is_started(d)) {
278 irq_enable(desc);
279 } else {
280 switch (__irq_startup_managed(desc, aff, force)) {
281 case IRQ_STARTUP_NORMAL:
282 if (d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP)
283 irq_setup_affinity(desc);
284 ret = __irq_startup(desc);
285 if (!(d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP))
286 irq_setup_affinity(desc);
287 break;
288 case IRQ_STARTUP_MANAGED:
289 irq_do_set_affinity(data: d, dest: aff, force: false);
290 ret = __irq_startup(desc);
291 break;
292 case IRQ_STARTUP_ABORT:
293 desc->depth = 1;
294 irqd_set_managed_shutdown(d);
295 return 0;
296 }
297 }
298 if (resend)
299 check_irq_resend(desc, inject: false);
300
301 return ret;
302}
303
304int irq_activate(struct irq_desc *desc)
305{
306 struct irq_data *d = irq_desc_get_irq_data(desc);
307
308 if (!irqd_affinity_is_managed(d))
309 return irq_domain_activate_irq(irq_data: d, early: false);
310 return 0;
311}
312
313int irq_activate_and_startup(struct irq_desc *desc, bool resend)
314{
315 if (WARN_ON(irq_activate(desc)))
316 return 0;
317 return irq_startup(desc, resend, IRQ_START_FORCE);
318}
319
320static void __irq_disable(struct irq_desc *desc, bool mask);
321
322void irq_shutdown(struct irq_desc *desc)
323{
324 if (irqd_is_started(d: &desc->irq_data)) {
325 clear_irq_resend(desc);
326 /*
327 * Increment disable depth, so that a managed shutdown on
328 * CPU hotunplug preserves the actual disabled state when the
329 * CPU comes back online. See irq_startup_managed().
330 */
331 desc->depth++;
332
333 if (desc->irq_data.chip->irq_shutdown) {
334 desc->irq_data.chip->irq_shutdown(&desc->irq_data);
335 irq_state_set_disabled(desc);
336 irq_state_set_masked(desc);
337 } else {
338 __irq_disable(desc, mask: true);
339 }
340 irq_state_clr_started(desc);
341 }
342}
343
344
345void irq_shutdown_and_deactivate(struct irq_desc *desc)
346{
347 irq_shutdown(desc);
348 /*
349 * This must be called even if the interrupt was never started up,
350 * because the activation can happen before the interrupt is
351 * available for request/startup. It has it's own state tracking so
352 * it's safe to call it unconditionally.
353 */
354 irq_domain_deactivate_irq(irq_data: &desc->irq_data);
355}
356
357static void __irq_disable(struct irq_desc *desc, bool mask)
358{
359 if (irqd_irq_disabled(d: &desc->irq_data)) {
360 if (mask)
361 mask_irq(desc);
362 } else {
363 irq_state_set_disabled(desc);
364 if (desc->irq_data.chip->irq_disable) {
365 desc->irq_data.chip->irq_disable(&desc->irq_data);
366 irq_state_set_masked(desc);
367 } else if (mask) {
368 mask_irq(desc);
369 }
370 }
371}
372
373/**
374 * irq_disable - Mark interrupt disabled
375 * @desc: irq descriptor which should be disabled
376 *
377 * If the chip does not implement the irq_disable callback, we
378 * use a lazy disable approach. That means we mark the interrupt
379 * disabled, but leave the hardware unmasked. That's an
380 * optimization because we avoid the hardware access for the
381 * common case where no interrupt happens after we marked it
382 * disabled. If an interrupt happens, then the interrupt flow
383 * handler masks the line at the hardware level and marks it
384 * pending.
385 *
386 * If the interrupt chip does not implement the irq_disable callback,
387 * a driver can disable the lazy approach for a particular irq line by
388 * calling 'irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY)'. This can
389 * be used for devices which cannot disable the interrupt at the
390 * device level under certain circumstances and have to use
391 * disable_irq[_nosync] instead.
392 */
393void irq_disable(struct irq_desc *desc)
394{
395 __irq_disable(desc, mask: irq_settings_disable_unlazy(desc));
396}
397
398void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
399{
400 if (desc->irq_data.chip->irq_enable)
401 desc->irq_data.chip->irq_enable(&desc->irq_data);
402 else
403 desc->irq_data.chip->irq_unmask(&desc->irq_data);
404 cpumask_set_cpu(cpu, dstp: desc->percpu_enabled);
405}
406
407void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
408{
409 if (desc->irq_data.chip->irq_disable)
410 desc->irq_data.chip->irq_disable(&desc->irq_data);
411 else
412 desc->irq_data.chip->irq_mask(&desc->irq_data);
413 cpumask_clear_cpu(cpu, dstp: desc->percpu_enabled);
414}
415
416static inline void mask_ack_irq(struct irq_desc *desc)
417{
418 if (desc->irq_data.chip->irq_mask_ack) {
419 desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
420 irq_state_set_masked(desc);
421 } else {
422 mask_irq(desc);
423 if (desc->irq_data.chip->irq_ack)
424 desc->irq_data.chip->irq_ack(&desc->irq_data);
425 }
426}
427
428void mask_irq(struct irq_desc *desc)
429{
430 if (irqd_irq_masked(d: &desc->irq_data))
431 return;
432
433 if (desc->irq_data.chip->irq_mask) {
434 desc->irq_data.chip->irq_mask(&desc->irq_data);
435 irq_state_set_masked(desc);
436 }
437}
438
439void unmask_irq(struct irq_desc *desc)
440{
441 if (!irqd_irq_masked(d: &desc->irq_data))
442 return;
443
444 if (desc->irq_data.chip->irq_unmask) {
445 desc->irq_data.chip->irq_unmask(&desc->irq_data);
446 irq_state_clr_masked(desc);
447 }
448}
449
450void unmask_threaded_irq(struct irq_desc *desc)
451{
452 struct irq_chip *chip = desc->irq_data.chip;
453
454 if (chip->flags & IRQCHIP_EOI_THREADED)
455 chip->irq_eoi(&desc->irq_data);
456
457 unmask_irq(desc);
458}
459
460/* Busy wait until INPROGRESS is cleared */
461static bool irq_wait_on_inprogress(struct irq_desc *desc)
462{
463 if (IS_ENABLED(CONFIG_SMP)) {
464 do {
465 raw_spin_unlock(&desc->lock);
466 while (irqd_irq_inprogress(d: &desc->irq_data))
467 cpu_relax();
468 raw_spin_lock(&desc->lock);
469 } while (irqd_irq_inprogress(d: &desc->irq_data));
470
471 /* Might have been disabled in meantime */
472 return !irqd_irq_disabled(d: &desc->irq_data) && desc->action;
473 }
474 return false;
475}
476
477static bool irq_can_handle_pm(struct irq_desc *desc)
478{
479 struct irq_data *irqd = &desc->irq_data;
480 const struct cpumask *aff;
481
482 /*
483 * If the interrupt is not in progress and is not an armed
484 * wakeup interrupt, proceed.
485 */
486 if (!irqd_has_set(d: irqd, mask: IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED))
487 return true;
488
489 /*
490 * If the interrupt is an armed wakeup source, mark it pending
491 * and suspended, disable it and notify the pm core about the
492 * event.
493 */
494 if (unlikely(irqd_has_set(irqd, IRQD_WAKEUP_ARMED))) {
495 irq_pm_handle_wakeup(desc);
496 return false;
497 }
498
499 /* Check whether the interrupt is polled on another CPU */
500 if (unlikely(desc->istate & IRQS_POLL_INPROGRESS)) {
501 if (WARN_ONCE(irq_poll_cpu == smp_processor_id(),
502 "irq poll in progress on cpu %d for irq %d\n",
503 smp_processor_id(), desc->irq_data.irq))
504 return false;
505 return irq_wait_on_inprogress(desc);
506 }
507
508 /* The below works only for single target interrupts */
509 if (!IS_ENABLED(CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK) ||
510 !irqd_is_single_target(d: irqd) || desc->handle_irq != handle_edge_irq)
511 return false;
512
513 /*
514 * If the interrupt affinity was moved to this CPU and the
515 * interrupt is currently handled on the previous target CPU, then
516 * busy wait for INPROGRESS to be cleared. Otherwise for edge type
517 * interrupts the handler might get stuck on the previous target:
518 *
519 * CPU 0 CPU 1 (new target)
520 * handle_edge_irq()
521 * repeat:
522 * handle_event() handle_edge_irq()
523 * if (INPROGESS) {
524 * set(PENDING);
525 * mask();
526 * return;
527 * }
528 * if (PENDING) {
529 * clear(PENDING);
530 * unmask();
531 * goto repeat;
532 * }
533 *
534 * This happens when the device raises interrupts with a high rate
535 * and always before handle_event() completes and the CPU0 handler
536 * can clear INPROGRESS. This has been observed in virtual machines.
537 */
538 aff = irq_data_get_effective_affinity_mask(d: irqd);
539 if (cpumask_first(srcp: aff) != smp_processor_id())
540 return false;
541 return irq_wait_on_inprogress(desc);
542}
543
544static inline bool irq_can_handle_actions(struct irq_desc *desc)
545{
546 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
547
548 if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
549 desc->istate |= IRQS_PENDING;
550 return false;
551 }
552 return true;
553}
554
555static inline bool irq_can_handle(struct irq_desc *desc)
556{
557 if (!irq_can_handle_pm(desc))
558 return false;
559
560 return irq_can_handle_actions(desc);
561}
562
563/**
564 * handle_nested_irq - Handle a nested irq from a irq thread
565 * @irq: the interrupt number
566 *
567 * Handle interrupts which are nested into a threaded interrupt
568 * handler. The handler function is called inside the calling threads
569 * context.
570 */
571void handle_nested_irq(unsigned int irq)
572{
573 struct irq_desc *desc = irq_to_desc(irq);
574 struct irqaction *action;
575 irqreturn_t action_ret;
576
577 might_sleep();
578
579 scoped_guard(raw_spinlock_irq, &desc->lock) {
580 if (!irq_can_handle_actions(desc))
581 return;
582
583 action = desc->action;
584 kstat_incr_irqs_this_cpu(desc);
585 atomic_inc(v: &desc->threads_active);
586 }
587
588 action_ret = IRQ_NONE;
589 for_each_action_of_desc(desc, action)
590 action_ret |= action->thread_fn(action->irq, action->dev_id);
591
592 if (!irq_settings_no_debug(desc))
593 note_interrupt(desc, action_ret);
594
595 wake_threads_waitq(desc);
596}
597EXPORT_SYMBOL_GPL(handle_nested_irq);
598
599/**
600 * handle_simple_irq - Simple and software-decoded IRQs.
601 * @desc: the interrupt description structure for this irq
602 *
603 * Simple interrupts are either sent from a demultiplexing interrupt
604 * handler or come from hardware, where no interrupt hardware control is
605 * necessary.
606 *
607 * Note: The caller is expected to handle the ack, clear, mask and unmask
608 * issues if necessary.
609 */
610void handle_simple_irq(struct irq_desc *desc)
611{
612 guard(raw_spinlock)(l: &desc->lock);
613
614 if (!irq_can_handle_pm(desc)) {
615 if (irqd_needs_resend_when_in_progress(d: &desc->irq_data))
616 desc->istate |= IRQS_PENDING;
617 return;
618 }
619
620 if (!irq_can_handle_actions(desc))
621 return;
622
623 kstat_incr_irqs_this_cpu(desc);
624 handle_irq_event(desc);
625}
626EXPORT_SYMBOL_GPL(handle_simple_irq);
627
628/**
629 * handle_untracked_irq - Simple and software-decoded IRQs.
630 * @desc: the interrupt description structure for this irq
631 *
632 * Untracked interrupts are sent from a demultiplexing interrupt handler
633 * when the demultiplexer does not know which device it its multiplexed irq
634 * domain generated the interrupt. IRQ's handled through here are not
635 * subjected to stats tracking, randomness, or spurious interrupt
636 * detection.
637 *
638 * Note: Like handle_simple_irq, the caller is expected to handle the ack,
639 * clear, mask and unmask issues if necessary.
640 */
641void handle_untracked_irq(struct irq_desc *desc)
642{
643 scoped_guard(raw_spinlock, &desc->lock) {
644 if (!irq_can_handle(desc))
645 return;
646
647 desc->istate &= ~IRQS_PENDING;
648 irqd_set(d: &desc->irq_data, mask: IRQD_IRQ_INPROGRESS);
649 }
650
651 __handle_irq_event_percpu(desc);
652
653 scoped_guard(raw_spinlock, &desc->lock)
654 irqd_clear(d: &desc->irq_data, mask: IRQD_IRQ_INPROGRESS);
655}
656EXPORT_SYMBOL_GPL(handle_untracked_irq);
657
658/*
659 * Called unconditionally from handle_level_irq() and only for oneshot
660 * interrupts from handle_fasteoi_irq()
661 */
662static void cond_unmask_irq(struct irq_desc *desc)
663{
664 /*
665 * We need to unmask in the following cases:
666 * - Standard level irq (IRQF_ONESHOT is not set)
667 * - Oneshot irq which did not wake the thread (caused by a
668 * spurious interrupt or a primary handler handling it
669 * completely).
670 */
671 if (!irqd_irq_disabled(d: &desc->irq_data) &&
672 irqd_irq_masked(d: &desc->irq_data) && !desc->threads_oneshot)
673 unmask_irq(desc);
674}
675
676/**
677 * handle_level_irq - Level type irq handler
678 * @desc: the interrupt description structure for this irq
679 *
680 * Level type interrupts are active as long as the hardware line has the
681 * active level. This may require to mask the interrupt and unmask it after
682 * the associated handler has acknowledged the device, so the interrupt
683 * line is back to inactive.
684 */
685void handle_level_irq(struct irq_desc *desc)
686{
687 guard(raw_spinlock)(l: &desc->lock);
688 mask_ack_irq(desc);
689
690 if (!irq_can_handle(desc))
691 return;
692
693 kstat_incr_irqs_this_cpu(desc);
694 handle_irq_event(desc);
695
696 cond_unmask_irq(desc);
697}
698EXPORT_SYMBOL_GPL(handle_level_irq);
699
700static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
701{
702 if (!(desc->istate & IRQS_ONESHOT)) {
703 chip->irq_eoi(&desc->irq_data);
704 return;
705 }
706 /*
707 * We need to unmask in the following cases:
708 * - Oneshot irq which did not wake the thread (caused by a
709 * spurious interrupt or a primary handler handling it
710 * completely).
711 */
712 if (!irqd_irq_disabled(d: &desc->irq_data) &&
713 irqd_irq_masked(d: &desc->irq_data) && !desc->threads_oneshot) {
714 chip->irq_eoi(&desc->irq_data);
715 unmask_irq(desc);
716 } else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
717 chip->irq_eoi(&desc->irq_data);
718 }
719}
720
721static inline void cond_eoi_irq(struct irq_chip *chip, struct irq_data *data)
722{
723 if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
724 chip->irq_eoi(data);
725}
726
727/**
728 * handle_fasteoi_irq - irq handler for transparent controllers
729 * @desc: the interrupt description structure for this irq
730 *
731 * Only a single callback will be issued to the chip: an ->eoi() call when
732 * the interrupt has been serviced. This enables support for modern forms
733 * of interrupt handlers, which handle the flow details in hardware,
734 * transparently.
735 */
736void handle_fasteoi_irq(struct irq_desc *desc)
737{
738 struct irq_chip *chip = desc->irq_data.chip;
739
740 guard(raw_spinlock)(l: &desc->lock);
741
742 /*
743 * When an affinity change races with IRQ handling, the next interrupt
744 * can arrive on the new CPU before the original CPU has completed
745 * handling the previous one - it may need to be resent.
746 */
747 if (!irq_can_handle_pm(desc)) {
748 if (irqd_needs_resend_when_in_progress(d: &desc->irq_data))
749 desc->istate |= IRQS_PENDING;
750 cond_eoi_irq(chip, data: &desc->irq_data);
751 return;
752 }
753
754 if (!irq_can_handle_actions(desc)) {
755 mask_irq(desc);
756 cond_eoi_irq(chip, data: &desc->irq_data);
757 return;
758 }
759
760 kstat_incr_irqs_this_cpu(desc);
761 if (desc->istate & IRQS_ONESHOT)
762 mask_irq(desc);
763
764 handle_irq_event(desc);
765
766 cond_unmask_eoi_irq(desc, chip);
767
768 /*
769 * When the race described above happens this will resend the interrupt.
770 */
771 if (unlikely(desc->istate & IRQS_PENDING))
772 check_irq_resend(desc, inject: false);
773}
774EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
775
776/**
777 * handle_fasteoi_nmi - irq handler for NMI interrupt lines
778 * @desc: the interrupt description structure for this irq
779 *
780 * A simple NMI-safe handler, considering the restrictions
781 * from request_nmi.
782 *
783 * Only a single callback will be issued to the chip: an ->eoi()
784 * call when the interrupt has been serviced. This enables support
785 * for modern forms of interrupt handlers, which handle the flow
786 * details in hardware, transparently.
787 */
788void handle_fasteoi_nmi(struct irq_desc *desc)
789{
790 struct irq_chip *chip = irq_desc_get_chip(desc);
791 struct irqaction *action = desc->action;
792 unsigned int irq = irq_desc_get_irq(desc);
793 irqreturn_t res;
794
795 __kstat_incr_irqs_this_cpu(desc);
796
797 trace_irq_handler_entry(irq, action);
798 /*
799 * NMIs cannot be shared, there is only one action.
800 */
801 res = action->handler(irq, action->dev_id);
802 trace_irq_handler_exit(irq, action, ret: res);
803
804 if (chip->irq_eoi)
805 chip->irq_eoi(&desc->irq_data);
806}
807EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
808
809/**
810 * handle_edge_irq - edge type IRQ handler
811 * @desc: the interrupt description structure for this irq
812 *
813 * Interrupt occurs on the falling and/or rising edge of a hardware
814 * signal. The occurrence is latched into the irq controller hardware and
815 * must be acked in order to be reenabled. After the ack another interrupt
816 * can happen on the same source even before the first one is handled by
817 * the associated event handler. If this happens it might be necessary to
818 * disable (mask) the interrupt depending on the controller hardware. This
819 * requires to reenable the interrupt inside of the loop which handles the
820 * interrupts which have arrived while the handler was running. If all
821 * pending interrupts are handled, the loop is left.
822 */
823void handle_edge_irq(struct irq_desc *desc)
824{
825 guard(raw_spinlock)(l: &desc->lock);
826
827 if (!irq_can_handle(desc)) {
828 desc->istate |= IRQS_PENDING;
829 mask_ack_irq(desc);
830 return;
831 }
832
833 kstat_incr_irqs_this_cpu(desc);
834
835 /* Start handling the irq */
836 desc->irq_data.chip->irq_ack(&desc->irq_data);
837
838 do {
839 if (unlikely(!desc->action)) {
840 mask_irq(desc);
841 return;
842 }
843
844 /*
845 * When another irq arrived while we were handling
846 * one, we could have masked the irq.
847 * Reenable it, if it was not disabled in meantime.
848 */
849 if (unlikely(desc->istate & IRQS_PENDING)) {
850 if (!irqd_irq_disabled(d: &desc->irq_data) &&
851 irqd_irq_masked(d: &desc->irq_data))
852 unmask_irq(desc);
853 }
854
855 handle_irq_event(desc);
856
857 } while ((desc->istate & IRQS_PENDING) && !irqd_irq_disabled(d: &desc->irq_data));
858}
859EXPORT_SYMBOL(handle_edge_irq);
860
861/**
862 * handle_percpu_irq - Per CPU local irq handler
863 * @desc: the interrupt description structure for this irq
864 *
865 * Per CPU interrupts on SMP machines without locking requirements
866 */
867void handle_percpu_irq(struct irq_desc *desc)
868{
869 struct irq_chip *chip = irq_desc_get_chip(desc);
870
871 /*
872 * PER CPU interrupts are not serialized. Do not touch
873 * desc->tot_count.
874 */
875 __kstat_incr_irqs_this_cpu(desc);
876
877 if (chip->irq_ack)
878 chip->irq_ack(&desc->irq_data);
879
880 handle_irq_event_percpu(desc);
881
882 if (chip->irq_eoi)
883 chip->irq_eoi(&desc->irq_data);
884}
885
886/**
887 * handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
888 * @desc: the interrupt description structure for this irq
889 *
890 * Per CPU interrupts on SMP machines without locking requirements. Same as
891 * handle_percpu_irq() above but with the following extras:
892 *
893 * action->percpu_dev_id is a pointer to percpu variables which
894 * contain the real device id for the cpu on which this handler is
895 * called
896 */
897void handle_percpu_devid_irq(struct irq_desc *desc)
898{
899 struct irq_chip *chip = irq_desc_get_chip(desc);
900 struct irqaction *action = desc->action;
901 unsigned int irq = irq_desc_get_irq(desc);
902 irqreturn_t res;
903
904 /*
905 * PER CPU interrupts are not serialized. Do not touch
906 * desc->tot_count.
907 */
908 __kstat_incr_irqs_this_cpu(desc);
909
910 if (chip->irq_ack)
911 chip->irq_ack(&desc->irq_data);
912
913 if (likely(action)) {
914 trace_irq_handler_entry(irq, action);
915 res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
916 trace_irq_handler_exit(irq, action, ret: res);
917 } else {
918 unsigned int cpu = smp_processor_id();
919 bool enabled = cpumask_test_cpu(cpu, cpumask: desc->percpu_enabled);
920
921 if (enabled)
922 irq_percpu_disable(desc, cpu);
923
924 pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n",
925 enabled ? " and unmasked" : "", irq, cpu);
926 }
927
928 if (chip->irq_eoi)
929 chip->irq_eoi(&desc->irq_data);
930}
931
932/**
933 * handle_percpu_devid_fasteoi_nmi - Per CPU local NMI handler with per cpu
934 * dev ids
935 * @desc: the interrupt description structure for this irq
936 *
937 * Similar to handle_fasteoi_nmi, but handling the dev_id cookie
938 * as a percpu pointer.
939 */
940void handle_percpu_devid_fasteoi_nmi(struct irq_desc *desc)
941{
942 struct irq_chip *chip = irq_desc_get_chip(desc);
943 struct irqaction *action = desc->action;
944 unsigned int irq = irq_desc_get_irq(desc);
945 irqreturn_t res;
946
947 __kstat_incr_irqs_this_cpu(desc);
948
949 trace_irq_handler_entry(irq, action);
950 res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
951 trace_irq_handler_exit(irq, action, ret: res);
952
953 if (chip->irq_eoi)
954 chip->irq_eoi(&desc->irq_data);
955}
956
957static void
958__irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
959 int is_chained, const char *name)
960{
961 if (!handle) {
962 handle = handle_bad_irq;
963 } else {
964 struct irq_data *irq_data = &desc->irq_data;
965#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
966 /*
967 * With hierarchical domains we might run into a
968 * situation where the outermost chip is not yet set
969 * up, but the inner chips are there. Instead of
970 * bailing we install the handler, but obviously we
971 * cannot enable/startup the interrupt at this point.
972 */
973 while (irq_data) {
974 if (irq_data->chip != &no_irq_chip)
975 break;
976 /*
977 * Bail out if the outer chip is not set up
978 * and the interrupt supposed to be started
979 * right away.
980 */
981 if (WARN_ON(is_chained))
982 return;
983 /* Try the parent */
984 irq_data = irq_data->parent_data;
985 }
986#endif
987 if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
988 return;
989 }
990
991 /* Uninstall? */
992 if (handle == handle_bad_irq) {
993 if (desc->irq_data.chip != &no_irq_chip)
994 mask_ack_irq(desc);
995 irq_state_set_disabled(desc);
996 if (is_chained) {
997 desc->action = NULL;
998 WARN_ON(irq_chip_pm_put(irq_desc_get_irq_data(desc)));
999 }
1000 desc->depth = 1;
1001 }
1002 desc->handle_irq = handle;
1003 desc->name = name;
1004
1005 if (handle != handle_bad_irq && is_chained) {
1006 unsigned int type = irqd_get_trigger_type(d: &desc->irq_data);
1007
1008 /*
1009 * We're about to start this interrupt immediately,
1010 * hence the need to set the trigger configuration.
1011 * But the .set_type callback may have overridden the
1012 * flow handler, ignoring that we're dealing with a
1013 * chained interrupt. Reset it immediately because we
1014 * do know better.
1015 */
1016 if (type != IRQ_TYPE_NONE) {
1017 __irq_set_trigger(desc, flags: type);
1018 desc->handle_irq = handle;
1019 }
1020
1021 irq_settings_set_noprobe(desc);
1022 irq_settings_set_norequest(desc);
1023 irq_settings_set_nothread(desc);
1024 desc->action = &chained_action;
1025 WARN_ON(irq_chip_pm_get(irq_desc_get_irq_data(desc)));
1026 irq_activate_and_startup(desc, IRQ_RESEND);
1027 }
1028}
1029
1030void __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
1031 const char *name)
1032{
1033 scoped_irqdesc_get_and_lock(irq, 0)
1034 __irq_do_set_handler(scoped_irqdesc, handle, is_chained, name);
1035}
1036EXPORT_SYMBOL_GPL(__irq_set_handler);
1037
1038void irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
1039 void *data)
1040{
1041 scoped_irqdesc_get_and_buslock(irq, 0) {
1042 struct irq_desc *desc = scoped_irqdesc;
1043
1044 desc->irq_common_data.handler_data = data;
1045 __irq_do_set_handler(desc, handle, is_chained: 1, NULL);
1046 }
1047}
1048EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
1049
1050void
1051irq_set_chip_and_handler_name(unsigned int irq, const struct irq_chip *chip,
1052 irq_flow_handler_t handle, const char *name)
1053{
1054 irq_set_chip(irq, chip);
1055 __irq_set_handler(irq, handle, 0, name);
1056}
1057EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
1058
1059void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
1060{
1061 scoped_irqdesc_get_and_lock(irq, 0) {
1062 struct irq_desc *desc = scoped_irqdesc;
1063 unsigned long trigger, tmp;
1064 /*
1065 * Warn when a driver sets the no autoenable flag on an already
1066 * active interrupt.
1067 */
1068 WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
1069
1070 irq_settings_clr_and_set(desc, clr, set);
1071
1072 trigger = irqd_get_trigger_type(d: &desc->irq_data);
1073
1074 irqd_clear(d: &desc->irq_data, mask: IRQD_NO_BALANCING | IRQD_PER_CPU |
1075 IRQD_TRIGGER_MASK | IRQD_LEVEL);
1076 if (irq_settings_has_no_balance_set(desc))
1077 irqd_set(d: &desc->irq_data, mask: IRQD_NO_BALANCING);
1078 if (irq_settings_is_per_cpu(desc))
1079 irqd_set(d: &desc->irq_data, mask: IRQD_PER_CPU);
1080 if (irq_settings_is_level(desc))
1081 irqd_set(d: &desc->irq_data, mask: IRQD_LEVEL);
1082
1083 tmp = irq_settings_get_trigger_mask(desc);
1084 if (tmp != IRQ_TYPE_NONE)
1085 trigger = tmp;
1086
1087 irqd_set(d: &desc->irq_data, mask: trigger);
1088 }
1089}
1090EXPORT_SYMBOL_GPL(irq_modify_status);
1091
1092#ifdef CONFIG_DEPRECATED_IRQ_CPU_ONOFFLINE
1093/**
1094 * irq_cpu_online - Invoke all irq_cpu_online functions.
1095 *
1096 * Iterate through all irqs and invoke the chip.irq_cpu_online()
1097 * for each.
1098 */
1099void irq_cpu_online(void)
1100{
1101 unsigned int irq;
1102
1103 for_each_active_irq(irq) {
1104 struct irq_desc *desc = irq_to_desc(irq);
1105 struct irq_chip *chip;
1106
1107 if (!desc)
1108 continue;
1109
1110 guard(raw_spinlock_irqsave)(&desc->lock);
1111 chip = irq_data_get_irq_chip(&desc->irq_data);
1112 if (chip && chip->irq_cpu_online &&
1113 (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1114 !irqd_irq_disabled(&desc->irq_data)))
1115 chip->irq_cpu_online(&desc->irq_data);
1116 }
1117}
1118
1119/**
1120 * irq_cpu_offline - Invoke all irq_cpu_offline functions.
1121 *
1122 * Iterate through all irqs and invoke the chip.irq_cpu_offline()
1123 * for each.
1124 */
1125void irq_cpu_offline(void)
1126{
1127 unsigned int irq;
1128
1129 for_each_active_irq(irq) {
1130 struct irq_desc *desc = irq_to_desc(irq);
1131 struct irq_chip *chip;
1132
1133 if (!desc)
1134 continue;
1135
1136 guard(raw_spinlock_irqsave)(&desc->lock);
1137 chip = irq_data_get_irq_chip(&desc->irq_data);
1138 if (chip && chip->irq_cpu_offline &&
1139 (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1140 !irqd_irq_disabled(&desc->irq_data)))
1141 chip->irq_cpu_offline(&desc->irq_data);
1142 }
1143}
1144#endif
1145
1146#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1147
1148#ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
1149/**
1150 * handle_fasteoi_ack_irq - irq handler for edge hierarchy stacked on
1151 * transparent controllers
1152 *
1153 * @desc: the interrupt description structure for this irq
1154 *
1155 * Like handle_fasteoi_irq(), but for use with hierarchy where the irq_chip
1156 * also needs to have its ->irq_ack() function called.
1157 */
1158void handle_fasteoi_ack_irq(struct irq_desc *desc)
1159{
1160 struct irq_chip *chip = desc->irq_data.chip;
1161
1162 guard(raw_spinlock)(&desc->lock);
1163
1164 if (!irq_can_handle_pm(desc)) {
1165 cond_eoi_irq(chip, &desc->irq_data);
1166 return;
1167 }
1168
1169 if (unlikely(!irq_can_handle_actions(desc))) {
1170 mask_irq(desc);
1171 cond_eoi_irq(chip, &desc->irq_data);
1172 return;
1173 }
1174
1175 kstat_incr_irqs_this_cpu(desc);
1176 if (desc->istate & IRQS_ONESHOT)
1177 mask_irq(desc);
1178
1179 desc->irq_data.chip->irq_ack(&desc->irq_data);
1180
1181 handle_irq_event(desc);
1182
1183 cond_unmask_eoi_irq(desc, chip);
1184}
1185EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
1186
1187/**
1188 * handle_fasteoi_mask_irq - irq handler for level hierarchy stacked on
1189 * transparent controllers
1190 *
1191 * @desc: the interrupt description structure for this irq
1192 *
1193 * Like handle_fasteoi_irq(), but for use with hierarchy where the irq_chip
1194 * also needs to have its ->irq_mask_ack() function called.
1195 */
1196void handle_fasteoi_mask_irq(struct irq_desc *desc)
1197{
1198 struct irq_chip *chip = desc->irq_data.chip;
1199
1200 guard(raw_spinlock)(&desc->lock);
1201 mask_ack_irq(desc);
1202
1203 if (!irq_can_handle(desc)) {
1204 cond_eoi_irq(chip, &desc->irq_data);
1205 return;
1206 }
1207
1208 kstat_incr_irqs_this_cpu(desc);
1209
1210 handle_irq_event(desc);
1211
1212 cond_unmask_eoi_irq(desc, chip);
1213}
1214EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
1215
1216#endif /* CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS */
1217
1218/**
1219 * irq_chip_set_parent_state - set the state of a parent interrupt.
1220 *
1221 * @data: Pointer to interrupt specific data
1222 * @which: State to be restored (one of IRQCHIP_STATE_*)
1223 * @val: Value corresponding to @which
1224 *
1225 * Conditional success, if the underlying irqchip does not implement it.
1226 */
1227int irq_chip_set_parent_state(struct irq_data *data,
1228 enum irqchip_irq_state which,
1229 bool val)
1230{
1231 data = data->parent_data;
1232
1233 if (!data || !data->chip->irq_set_irqchip_state)
1234 return 0;
1235
1236 return data->chip->irq_set_irqchip_state(data, which, val);
1237}
1238EXPORT_SYMBOL_GPL(irq_chip_set_parent_state);
1239
1240/**
1241 * irq_chip_get_parent_state - get the state of a parent interrupt.
1242 *
1243 * @data: Pointer to interrupt specific data
1244 * @which: one of IRQCHIP_STATE_* the caller wants to know
1245 * @state: a pointer to a boolean where the state is to be stored
1246 *
1247 * Conditional success, if the underlying irqchip does not implement it.
1248 */
1249int irq_chip_get_parent_state(struct irq_data *data,
1250 enum irqchip_irq_state which,
1251 bool *state)
1252{
1253 data = data->parent_data;
1254
1255 if (!data || !data->chip->irq_get_irqchip_state)
1256 return 0;
1257
1258 return data->chip->irq_get_irqchip_state(data, which, state);
1259}
1260EXPORT_SYMBOL_GPL(irq_chip_get_parent_state);
1261
1262/**
1263 * irq_chip_shutdown_parent - Shutdown the parent interrupt
1264 * @data: Pointer to interrupt specific data
1265 *
1266 * Invokes the irq_shutdown() callback of the parent if available or falls
1267 * back to irq_chip_disable_parent().
1268 */
1269void irq_chip_shutdown_parent(struct irq_data *data)
1270{
1271 struct irq_data *parent = data->parent_data;
1272
1273 if (parent->chip->irq_shutdown)
1274 parent->chip->irq_shutdown(parent);
1275 else
1276 irq_chip_disable_parent(data);
1277}
1278EXPORT_SYMBOL_GPL(irq_chip_shutdown_parent);
1279
1280/**
1281 * irq_chip_startup_parent - Startup the parent interrupt
1282 * @data: Pointer to interrupt specific data
1283 *
1284 * Invokes the irq_startup() callback of the parent if available or falls
1285 * back to irq_chip_enable_parent().
1286 */
1287unsigned int irq_chip_startup_parent(struct irq_data *data)
1288{
1289 struct irq_data *parent = data->parent_data;
1290
1291 if (parent->chip->irq_startup)
1292 return parent->chip->irq_startup(parent);
1293
1294 irq_chip_enable_parent(data);
1295 return 0;
1296}
1297EXPORT_SYMBOL_GPL(irq_chip_startup_parent);
1298
1299/**
1300 * irq_chip_enable_parent - Enable the parent interrupt (defaults to unmask if
1301 * NULL)
1302 * @data: Pointer to interrupt specific data
1303 */
1304void irq_chip_enable_parent(struct irq_data *data)
1305{
1306 data = data->parent_data;
1307 if (data->chip->irq_enable)
1308 data->chip->irq_enable(data);
1309 else
1310 data->chip->irq_unmask(data);
1311}
1312EXPORT_SYMBOL_GPL(irq_chip_enable_parent);
1313
1314/**
1315 * irq_chip_disable_parent - Disable the parent interrupt (defaults to mask if
1316 * NULL)
1317 * @data: Pointer to interrupt specific data
1318 */
1319void irq_chip_disable_parent(struct irq_data *data)
1320{
1321 data = data->parent_data;
1322 if (data->chip->irq_disable)
1323 data->chip->irq_disable(data);
1324 else
1325 data->chip->irq_mask(data);
1326}
1327EXPORT_SYMBOL_GPL(irq_chip_disable_parent);
1328
1329/**
1330 * irq_chip_ack_parent - Acknowledge the parent interrupt
1331 * @data: Pointer to interrupt specific data
1332 */
1333void irq_chip_ack_parent(struct irq_data *data)
1334{
1335 data = data->parent_data;
1336 data->chip->irq_ack(data);
1337}
1338EXPORT_SYMBOL_GPL(irq_chip_ack_parent);
1339
1340/**
1341 * irq_chip_mask_parent - Mask the parent interrupt
1342 * @data: Pointer to interrupt specific data
1343 */
1344void irq_chip_mask_parent(struct irq_data *data)
1345{
1346 data = data->parent_data;
1347 data->chip->irq_mask(data);
1348}
1349EXPORT_SYMBOL_GPL(irq_chip_mask_parent);
1350
1351/**
1352 * irq_chip_mask_ack_parent - Mask and acknowledge the parent interrupt
1353 * @data: Pointer to interrupt specific data
1354 */
1355void irq_chip_mask_ack_parent(struct irq_data *data)
1356{
1357 data = data->parent_data;
1358 data->chip->irq_mask_ack(data);
1359}
1360EXPORT_SYMBOL_GPL(irq_chip_mask_ack_parent);
1361
1362/**
1363 * irq_chip_unmask_parent - Unmask the parent interrupt
1364 * @data: Pointer to interrupt specific data
1365 */
1366void irq_chip_unmask_parent(struct irq_data *data)
1367{
1368 data = data->parent_data;
1369 data->chip->irq_unmask(data);
1370}
1371EXPORT_SYMBOL_GPL(irq_chip_unmask_parent);
1372
1373/**
1374 * irq_chip_eoi_parent - Invoke EOI on the parent interrupt
1375 * @data: Pointer to interrupt specific data
1376 */
1377void irq_chip_eoi_parent(struct irq_data *data)
1378{
1379 data = data->parent_data;
1380 data->chip->irq_eoi(data);
1381}
1382EXPORT_SYMBOL_GPL(irq_chip_eoi_parent);
1383
1384/**
1385 * irq_chip_set_affinity_parent - Set affinity on the parent interrupt
1386 * @data: Pointer to interrupt specific data
1387 * @dest: The affinity mask to set
1388 * @force: Flag to enforce setting (disable online checks)
1389 *
1390 * Conditional, as the underlying parent chip might not implement it.
1391 */
1392int irq_chip_set_affinity_parent(struct irq_data *data,
1393 const struct cpumask *dest, bool force)
1394{
1395 data = data->parent_data;
1396 if (data->chip->irq_set_affinity)
1397 return data->chip->irq_set_affinity(data, dest, force);
1398
1399 return -ENOSYS;
1400}
1401EXPORT_SYMBOL_GPL(irq_chip_set_affinity_parent);
1402
1403/**
1404 * irq_chip_set_type_parent - Set IRQ type on the parent interrupt
1405 * @data: Pointer to interrupt specific data
1406 * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
1407 *
1408 * Conditional, as the underlying parent chip might not implement it.
1409 */
1410int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
1411{
1412 data = data->parent_data;
1413
1414 if (data->chip->irq_set_type)
1415 return data->chip->irq_set_type(data, type);
1416
1417 return -ENOSYS;
1418}
1419EXPORT_SYMBOL_GPL(irq_chip_set_type_parent);
1420
1421/**
1422 * irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware
1423 * @data: Pointer to interrupt specific data
1424 *
1425 * Iterate through the domain hierarchy of the interrupt and check
1426 * whether a hw retrigger function exists. If yes, invoke it.
1427 */
1428int irq_chip_retrigger_hierarchy(struct irq_data *data)
1429{
1430 for (data = data->parent_data; data; data = data->parent_data)
1431 if (data->chip && data->chip->irq_retrigger)
1432 return data->chip->irq_retrigger(data);
1433
1434 return 0;
1435}
1436EXPORT_SYMBOL_GPL(irq_chip_retrigger_hierarchy);
1437
1438/**
1439 * irq_chip_set_vcpu_affinity_parent - Set vcpu affinity on the parent interrupt
1440 * @data: Pointer to interrupt specific data
1441 * @vcpu_info: The vcpu affinity information
1442 */
1443int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
1444{
1445 data = data->parent_data;
1446 if (data->chip->irq_set_vcpu_affinity)
1447 return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
1448
1449 return -ENOSYS;
1450}
1451EXPORT_SYMBOL_GPL(irq_chip_set_vcpu_affinity_parent);
1452/**
1453 * irq_chip_set_wake_parent - Set/reset wake-up on the parent interrupt
1454 * @data: Pointer to interrupt specific data
1455 * @on: Whether to set or reset the wake-up capability of this irq
1456 *
1457 * Conditional, as the underlying parent chip might not implement it.
1458 */
1459int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
1460{
1461 data = data->parent_data;
1462
1463 if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
1464 return 0;
1465
1466 if (data->chip->irq_set_wake)
1467 return data->chip->irq_set_wake(data, on);
1468
1469 return -ENOSYS;
1470}
1471EXPORT_SYMBOL_GPL(irq_chip_set_wake_parent);
1472
1473/**
1474 * irq_chip_request_resources_parent - Request resources on the parent interrupt
1475 * @data: Pointer to interrupt specific data
1476 */
1477int irq_chip_request_resources_parent(struct irq_data *data)
1478{
1479 data = data->parent_data;
1480
1481 if (data->chip->irq_request_resources)
1482 return data->chip->irq_request_resources(data);
1483
1484 /* no error on missing optional irq_chip::irq_request_resources */
1485 return 0;
1486}
1487EXPORT_SYMBOL_GPL(irq_chip_request_resources_parent);
1488
1489/**
1490 * irq_chip_release_resources_parent - Release resources on the parent interrupt
1491 * @data: Pointer to interrupt specific data
1492 */
1493void irq_chip_release_resources_parent(struct irq_data *data)
1494{
1495 data = data->parent_data;
1496 if (data->chip->irq_release_resources)
1497 data->chip->irq_release_resources(data);
1498}
1499EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
1500#endif
1501
1502/**
1503 * irq_chip_compose_msi_msg - Compose msi message for a irq chip
1504 * @data: Pointer to interrupt specific data
1505 * @msg: Pointer to the MSI message
1506 *
1507 * For hierarchical domains we find the first chip in the hierarchy
1508 * which implements the irq_compose_msi_msg callback. For non
1509 * hierarchical we use the top level chip.
1510 */
1511int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1512{
1513 struct irq_data *pos;
1514
1515 for (pos = NULL; !pos && data; data = irqd_get_parent_data(irqd: data)) {
1516 if (data->chip && data->chip->irq_compose_msi_msg)
1517 pos = data;
1518 }
1519
1520 if (!pos)
1521 return -ENOSYS;
1522
1523 pos->chip->irq_compose_msi_msg(pos, msg);
1524 return 0;
1525}
1526
1527static struct device *irq_get_pm_device(struct irq_data *data)
1528{
1529 if (data->domain)
1530 return data->domain->pm_dev;
1531
1532 return NULL;
1533}
1534
1535/**
1536 * irq_chip_pm_get - Enable power for an IRQ chip
1537 * @data: Pointer to interrupt specific data
1538 *
1539 * Enable the power to the IRQ chip referenced by the interrupt data
1540 * structure.
1541 */
1542int irq_chip_pm_get(struct irq_data *data)
1543{
1544 struct device *dev = irq_get_pm_device(data);
1545 int retval = 0;
1546
1547 if (IS_ENABLED(CONFIG_PM) && dev)
1548 retval = pm_runtime_resume_and_get(dev);
1549
1550 return retval;
1551}
1552
1553/**
1554 * irq_chip_pm_put - Disable power for an IRQ chip
1555 * @data: Pointer to interrupt specific data
1556 *
1557 * Disable the power to the IRQ chip referenced by the interrupt data
1558 * structure, belongs. Note that power will only be disabled, once this
1559 * function has been called for all IRQs that have called irq_chip_pm_get().
1560 */
1561int irq_chip_pm_put(struct irq_data *data)
1562{
1563 struct device *dev = irq_get_pm_device(data);
1564 int retval = 0;
1565
1566 if (IS_ENABLED(CONFIG_PM) && dev)
1567 retval = pm_runtime_put(dev);
1568
1569 return (retval < 0) ? retval : 0;
1570}
1571