1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_IDTENTRY_H
3#define _ASM_X86_IDTENTRY_H
4
5/* Interrupts/Exceptions */
6#include <asm/trapnr.h>
7
8#define IDT_ALIGN (8 * (1 + HAS_KERNEL_IBT))
9
10#ifndef __ASSEMBLER__
11#include <linux/entry-common.h>
12#include <linux/hardirq.h>
13
14#include <asm/irq_stack.h>
15
16typedef void (*idtentry_t)(struct pt_regs *regs);
17
18/**
19 * DECLARE_IDTENTRY - Declare functions for simple IDT entry points
20 * No error code pushed by hardware
21 * @vector: Vector number (ignored for C)
22 * @func: Function name of the entry point
23 *
24 * Declares four functions:
25 * - The ASM entry point: asm_##func
26 * - The XEN PV trap entry point: xen_##func (maybe unused)
27 * - The C handler called from the FRED event dispatcher (maybe unused)
28 * - The C handler called from the ASM entry point
29 *
30 * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it
31 * declares the entry points for usage in C code. There is an ASM variant
32 * as well which is used to emit the entry stubs in entry_32/64.S.
33 */
34#define DECLARE_IDTENTRY(vector, func) \
35 asmlinkage void asm_##func(void); \
36 asmlinkage void xen_asm_##func(void); \
37 void fred_##func(struct pt_regs *regs); \
38 __visible void func(struct pt_regs *regs)
39
40/**
41 * DEFINE_IDTENTRY - Emit code for simple IDT entry points
42 * @func: Function name of the entry point
43 *
44 * @func is called from ASM entry code with interrupts disabled.
45 *
46 * The macro is written so it acts as function definition. Append the
47 * body with a pair of curly brackets.
48 *
49 * irqentry_enter() contains common code which has to be invoked before
50 * arbitrary code in the body. irqentry_exit() contains common code
51 * which has to run before returning to the low level assembly code.
52 */
53#define DEFINE_IDTENTRY(func) \
54static __always_inline void __##func(struct pt_regs *regs); \
55 \
56__visible noinstr void func(struct pt_regs *regs) \
57{ \
58 irqentry_state_t state = irqentry_enter(regs); \
59 \
60 instrumentation_begin(); \
61 __##func (regs); \
62 instrumentation_end(); \
63 irqentry_exit(regs, state); \
64} \
65 \
66static __always_inline void __##func(struct pt_regs *regs)
67
68/* Special case for 32bit IRET 'trap' */
69#define DECLARE_IDTENTRY_SW DECLARE_IDTENTRY
70#define DEFINE_IDTENTRY_SW DEFINE_IDTENTRY
71
72/**
73 * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points
74 * Error code pushed by hardware
75 * @vector: Vector number (ignored for C)
76 * @func: Function name of the entry point
77 *
78 * Declares three functions:
79 * - The ASM entry point: asm_##func
80 * - The XEN PV trap entry point: xen_##func (maybe unused)
81 * - The C handler called from the ASM entry point
82 *
83 * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the
84 * C-handler.
85 */
86#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
87 asmlinkage void asm_##func(void); \
88 asmlinkage void xen_asm_##func(void); \
89 __visible void func(struct pt_regs *regs, unsigned long error_code)
90
91/**
92 * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points
93 * Error code pushed by hardware
94 * @func: Function name of the entry point
95 *
96 * Same as DEFINE_IDTENTRY, but has an extra error_code argument
97 */
98#define DEFINE_IDTENTRY_ERRORCODE(func) \
99static __always_inline void __##func(struct pt_regs *regs, \
100 unsigned long error_code); \
101 \
102__visible noinstr void func(struct pt_regs *regs, \
103 unsigned long error_code) \
104{ \
105 irqentry_state_t state = irqentry_enter(regs); \
106 \
107 instrumentation_begin(); \
108 __##func (regs, error_code); \
109 instrumentation_end(); \
110 irqentry_exit(regs, state); \
111} \
112 \
113static __always_inline void __##func(struct pt_regs *regs, \
114 unsigned long error_code)
115
116/**
117 * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points
118 * No error code pushed by hardware
119 * @vector: Vector number (ignored for C)
120 * @func: Function name of the entry point
121 *
122 * Maps to DECLARE_IDTENTRY().
123 */
124#define DECLARE_IDTENTRY_RAW(vector, func) \
125 DECLARE_IDTENTRY(vector, func)
126
127/**
128 * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points
129 * @func: Function name of the entry point
130 *
131 * @func is called from ASM entry code with interrupts disabled.
132 *
133 * The macro is written so it acts as function definition. Append the
134 * body with a pair of curly brackets.
135 *
136 * Contrary to DEFINE_IDTENTRY() this does not invoke the
137 * idtentry_enter/exit() helpers before and after the body invocation. This
138 * needs to be done in the body itself if applicable. Use if extra work
139 * is required before the enter/exit() helpers are invoked.
140 */
141#define DEFINE_IDTENTRY_RAW(func) \
142__visible noinstr void func(struct pt_regs *regs)
143
144/**
145 * DEFINE_FREDENTRY_RAW - Emit code for raw FRED entry points
146 * @func: Function name of the entry point
147 *
148 * @func is called from the FRED event dispatcher with interrupts disabled.
149 *
150 * See @DEFINE_IDTENTRY_RAW for further details.
151 */
152#define DEFINE_FREDENTRY_RAW(func) \
153noinstr void fred_##func(struct pt_regs *regs)
154
155/**
156 * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points
157 * Error code pushed by hardware
158 * @vector: Vector number (ignored for C)
159 * @func: Function name of the entry point
160 *
161 * Maps to DECLARE_IDTENTRY_ERRORCODE()
162 */
163#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \
164 DECLARE_IDTENTRY_ERRORCODE(vector, func)
165
166/**
167 * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points
168 * @func: Function name of the entry point
169 *
170 * @func is called from ASM entry code with interrupts disabled.
171 *
172 * The macro is written so it acts as function definition. Append the
173 * body with a pair of curly brackets.
174 *
175 * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the
176 * irqentry_enter/exit() helpers before and after the body invocation. This
177 * needs to be done in the body itself if applicable. Use if extra work
178 * is required before the enter/exit() helpers are invoked.
179 */
180#define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \
181__visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
182
183/**
184 * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry
185 * points (common/spurious)
186 * @vector: Vector number (ignored for C)
187 * @func: Function name of the entry point
188 *
189 * Maps to DECLARE_IDTENTRY_ERRORCODE()
190 */
191#define DECLARE_IDTENTRY_IRQ(vector, func) \
192 DECLARE_IDTENTRY_ERRORCODE(vector, func)
193
194/**
195 * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points
196 * @func: Function name of the entry point
197 *
198 * The vector number is pushed by the low level entry stub and handed
199 * to the function as error_code argument which needs to be truncated
200 * to an u8 because the push is sign extending.
201 *
202 * irq_enter/exit_rcu() are invoked before the function body and the
203 * KVM L1D flush request is set. Stack switching to the interrupt stack
204 * has to be done in the function body if necessary.
205 */
206#define DEFINE_IDTENTRY_IRQ(func) \
207static void __##func(struct pt_regs *regs, u32 vector); \
208 \
209__visible noinstr void func(struct pt_regs *regs, \
210 unsigned long error_code) \
211{ \
212 irqentry_state_t state = irqentry_enter(regs); \
213 u32 vector = (u32)(u8)error_code; \
214 \
215 kvm_set_cpu_l1tf_flush_l1d(); \
216 instrumentation_begin(); \
217 run_irq_on_irqstack_cond(__##func, regs, vector); \
218 instrumentation_end(); \
219 irqentry_exit(regs, state); \
220} \
221 \
222static noinline void __##func(struct pt_regs *regs, u32 vector)
223
224/**
225 * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points
226 * @vector: Vector number (ignored for C)
227 * @func: Function name of the entry point
228 *
229 * Declares three functions:
230 * - The ASM entry point: asm_##func
231 * - The XEN PV trap entry point: xen_##func (maybe unused)
232 * - The C handler called from the ASM entry point
233 *
234 * Maps to DECLARE_IDTENTRY().
235 */
236#define DECLARE_IDTENTRY_SYSVEC(vector, func) \
237 DECLARE_IDTENTRY(vector, func)
238
239/**
240 * DEFINE_IDTENTRY_SYSVEC - Emit code for system vector IDT entry points
241 * @func: Function name of the entry point
242 *
243 * irqentry_enter/exit() and irq_enter/exit_rcu() are invoked before the
244 * function body. KVM L1D flush request is set.
245 *
246 * Runs the function on the interrupt stack if the entry hit kernel mode
247 */
248#define DEFINE_IDTENTRY_SYSVEC(func) \
249static void __##func(struct pt_regs *regs); \
250 \
251static __always_inline void instr_##func(struct pt_regs *regs) \
252{ \
253 run_sysvec_on_irqstack_cond(__##func, regs); \
254} \
255 \
256__visible noinstr void func(struct pt_regs *regs) \
257{ \
258 irqentry_state_t state = irqentry_enter(regs); \
259 \
260 kvm_set_cpu_l1tf_flush_l1d(); \
261 instrumentation_begin(); \
262 instr_##func (regs); \
263 instrumentation_end(); \
264 irqentry_exit(regs, state); \
265} \
266 \
267void fred_##func(struct pt_regs *regs) \
268{ \
269 instr_##func (regs); \
270} \
271 \
272static noinline void __##func(struct pt_regs *regs)
273
274/**
275 * DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT
276 * entry points
277 * @func: Function name of the entry point
278 *
279 * Runs the function on the interrupted stack. No switch to IRQ stack and
280 * only the minimal __irq_enter/exit() handling.
281 *
282 * Only use for 'empty' vectors like reschedule IPI and KVM posted
283 * interrupt vectors.
284 */
285#define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func) \
286static __always_inline void __##func(struct pt_regs *regs); \
287 \
288static __always_inline void instr_##func(struct pt_regs *regs) \
289{ \
290 __irq_enter_raw(); \
291 __##func (regs); \
292 __irq_exit_raw(); \
293} \
294 \
295__visible noinstr void func(struct pt_regs *regs) \
296{ \
297 irqentry_state_t state = irqentry_enter(regs); \
298 \
299 kvm_set_cpu_l1tf_flush_l1d(); \
300 instrumentation_begin(); \
301 instr_##func (regs); \
302 instrumentation_end(); \
303 irqentry_exit(regs, state); \
304} \
305 \
306void fred_##func(struct pt_regs *regs) \
307{ \
308 instr_##func (regs); \
309} \
310 \
311static __always_inline void __##func(struct pt_regs *regs)
312
313/**
314 * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point
315 * @vector: Vector number (ignored for C)
316 * @func: Function name of the entry point
317 *
318 * Declares three functions:
319 * - The ASM entry point: asm_##func
320 * - The XEN PV trap entry point: xen_##func (maybe unused)
321 * - The C handler called from the ASM entry point
322 *
323 * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit
324 * difference
325 */
326#define DECLARE_IDTENTRY_XENCB(vector, func) \
327 DECLARE_IDTENTRY(vector, func)
328
329#ifdef CONFIG_X86_64
330/**
331 * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points
332 * @vector: Vector number (ignored for C)
333 * @func: Function name of the entry point
334 *
335 * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler
336 * which is called from the ASM entry point on user mode entry
337 */
338#define DECLARE_IDTENTRY_IST(vector, func) \
339 DECLARE_IDTENTRY_RAW(vector, func); \
340 __visible void noist_##func(struct pt_regs *regs)
341
342/**
343 * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point
344 * @vector: Vector number (ignored for C)
345 * @func: Function name of the entry point
346 *
347 * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the
348 * safe_stack C handler.
349 */
350#define DECLARE_IDTENTRY_VC(vector, func) \
351 DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \
352 __visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code); \
353 __visible noinstr void user_##func(struct pt_regs *regs, unsigned long error_code)
354
355/**
356 * DEFINE_IDTENTRY_IST - Emit code for IST entry points
357 * @func: Function name of the entry point
358 *
359 * Maps to DEFINE_IDTENTRY_RAW
360 */
361#define DEFINE_IDTENTRY_IST(func) \
362 DEFINE_IDTENTRY_RAW(func)
363
364/**
365 * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which
366 * belong to a IST entry point (MCE, DB)
367 * @func: Function name of the entry point. Must be the same as
368 * the function name of the corresponding IST variant
369 *
370 * Maps to DEFINE_IDTENTRY_RAW().
371 */
372#define DEFINE_IDTENTRY_NOIST(func) \
373 DEFINE_IDTENTRY_RAW(noist_##func)
374
375/**
376 * DECLARE_IDTENTRY_DF - Declare functions for double fault
377 * @vector: Vector number (ignored for C)
378 * @func: Function name of the entry point
379 *
380 * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE
381 */
382#define DECLARE_IDTENTRY_DF(vector, func) \
383 DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func)
384
385/**
386 * DEFINE_IDTENTRY_DF - Emit code for double fault
387 * @func: Function name of the entry point
388 *
389 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
390 */
391#define DEFINE_IDTENTRY_DF(func) \
392 DEFINE_IDTENTRY_RAW_ERRORCODE(func)
393
394/**
395 * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler
396 when raised from kernel mode
397 * @func: Function name of the entry point
398 *
399 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
400 */
401#define DEFINE_IDTENTRY_VC_KERNEL(func) \
402 DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func)
403
404/**
405 * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler
406 when raised from user mode
407 * @func: Function name of the entry point
408 *
409 * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
410 */
411#define DEFINE_IDTENTRY_VC_USER(func) \
412 DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func)
413
414#else /* CONFIG_X86_64 */
415
416/**
417 * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant
418 * @vector: Vector number (ignored for C)
419 * @func: Function name of the entry point
420 *
421 * Declares two functions:
422 * - The ASM entry point: asm_##func
423 * - The C handler called from the C shim
424 */
425#define DECLARE_IDTENTRY_DF(vector, func) \
426 asmlinkage void asm_##func(void); \
427 __visible void func(struct pt_regs *regs, \
428 unsigned long error_code, \
429 unsigned long address)
430
431/**
432 * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit
433 * @func: Function name of the entry point
434 *
435 * This is called through the doublefault shim which already provides
436 * cr2 in the address argument.
437 */
438#define DEFINE_IDTENTRY_DF(func) \
439__visible noinstr void func(struct pt_regs *regs, \
440 unsigned long error_code, \
441 unsigned long address)
442
443#endif /* !CONFIG_X86_64 */
444
445/* C-Code mapping */
446#define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_RAW
447#define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_RAW
448#define DEFINE_FREDENTRY_NMI DEFINE_FREDENTRY_RAW
449
450#ifdef CONFIG_X86_64
451#define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST
452#define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST
453#define DEFINE_IDTENTRY_MCE_USER DEFINE_IDTENTRY_NOIST
454#define DEFINE_FREDENTRY_MCE DEFINE_FREDENTRY_RAW
455
456#define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST
457#define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST
458#define DEFINE_IDTENTRY_DEBUG_USER DEFINE_IDTENTRY_NOIST
459#define DEFINE_FREDENTRY_DEBUG DEFINE_FREDENTRY_RAW
460#endif
461
462void idt_install_sysvec(unsigned int n, const void *function);
463void fred_install_sysvec(unsigned int vector, const idtentry_t function);
464
465#define sysvec_install(vector, function) { \
466 if (IS_ENABLED(CONFIG_X86_FRED)) \
467 fred_install_sysvec(vector, function); \
468 if (!cpu_feature_enabled(X86_FEATURE_FRED)) \
469 idt_install_sysvec(vector, asm_##function); \
470}
471
472#else /* !__ASSEMBLER__ */
473
474/*
475 * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs.
476 */
477#define DECLARE_IDTENTRY(vector, func) \
478 idtentry vector asm_##func func has_error_code=0
479
480#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
481 idtentry vector asm_##func func has_error_code=1
482
483/* Special case for 32bit IRET 'trap'. Do not emit ASM code */
484#define DECLARE_IDTENTRY_SW(vector, func)
485
486#define DECLARE_IDTENTRY_RAW(vector, func) \
487 DECLARE_IDTENTRY(vector, func)
488
489#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \
490 DECLARE_IDTENTRY_ERRORCODE(vector, func)
491
492/* Entries for common/spurious (device) interrupts */
493#define DECLARE_IDTENTRY_IRQ(vector, func) \
494 idtentry_irq vector func
495
496/* System vector entries */
497#define DECLARE_IDTENTRY_SYSVEC(vector, func) \
498 DECLARE_IDTENTRY(vector, func)
499
500#ifdef CONFIG_X86_64
501# define DECLARE_IDTENTRY_MCE(vector, func) \
502 idtentry_mce_db vector asm_##func func
503
504# define DECLARE_IDTENTRY_DEBUG(vector, func) \
505 idtentry_mce_db vector asm_##func func
506
507# define DECLARE_IDTENTRY_DF(vector, func) \
508 idtentry_df vector asm_##func func
509
510# define DECLARE_IDTENTRY_XENCB(vector, func) \
511 DECLARE_IDTENTRY(vector, func)
512
513# define DECLARE_IDTENTRY_VC(vector, func) \
514 idtentry_vc vector asm_##func func
515
516#else
517# define DECLARE_IDTENTRY_MCE(vector, func) \
518 DECLARE_IDTENTRY(vector, func)
519
520/* No ASM emitted for DF as this goes through a C shim */
521# define DECLARE_IDTENTRY_DF(vector, func)
522
523/* No ASM emitted for XEN hypervisor callback */
524# define DECLARE_IDTENTRY_XENCB(vector, func)
525
526#endif
527
528/* No ASM code emitted for NMI */
529#define DECLARE_IDTENTRY_NMI(vector, func)
530
531/*
532 * ASM code to emit the common vector entry stubs where each stub is
533 * packed into IDT_ALIGN bytes.
534 *
535 * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because
536 * GCC treats the local vector variable as unsigned int and would expand
537 * all vectors above 0x7F to a 5 byte push. The original code did an
538 * adjustment of the vector number to be in the signed byte range to avoid
539 * this. While clever it's mindboggling counterintuitive and requires the
540 * odd conversion back to a real vector number in the C entry points. Using
541 * .byte achieves the same thing and the only fixup needed in the C entry
542 * point is to mask off the bits above bit 7 because the push is sign
543 * extending.
544 */
545 .align IDT_ALIGN
546SYM_CODE_START(irq_entries_start)
547 vector=FIRST_EXTERNAL_VECTOR
548 .rept NR_EXTERNAL_VECTORS
549 UNWIND_HINT_IRET_REGS
5500 :
551 ENDBR
552 .byte 0x6a, vector
553 jmp asm_common_interrupt
554 /* Ensure that the above is IDT_ALIGN bytes max */
555 .fill 0b + IDT_ALIGN - ., 1, 0xcc
556 vector = vector+1
557 .endr
558SYM_CODE_END(irq_entries_start)
559
560#ifdef CONFIG_X86_LOCAL_APIC
561 .align IDT_ALIGN
562SYM_CODE_START(spurious_entries_start)
563 vector=FIRST_SYSTEM_VECTOR
564 .rept NR_SYSTEM_VECTORS
565 UNWIND_HINT_IRET_REGS
5660 :
567 ENDBR
568 .byte 0x6a, vector
569 jmp asm_spurious_interrupt
570 /* Ensure that the above is IDT_ALIGN bytes max */
571 .fill 0b + IDT_ALIGN - ., 1, 0xcc
572 vector = vector+1
573 .endr
574SYM_CODE_END(spurious_entries_start)
575#endif
576
577#endif /* __ASSEMBLER__ */
578
579/*
580 * The actual entry points. Note that DECLARE_IDTENTRY*() serves two
581 * purposes:
582 * - provide the function declarations when included from C-Code
583 * - emit the ASM stubs when included from entry_32/64.S
584 *
585 * This avoids duplicate defines and ensures that everything is consistent.
586 */
587
588/*
589 * Dummy trap number so the low level ASM macro vector number checks do not
590 * match which results in emitting plain IDTENTRY stubs without bells and
591 * whistles.
592 */
593#define X86_TRAP_OTHER 0xFFFF
594
595/* Simple exception entry points. No hardware error code */
596DECLARE_IDTENTRY(X86_TRAP_DE, exc_divide_error);
597DECLARE_IDTENTRY(X86_TRAP_OF, exc_overflow);
598DECLARE_IDTENTRY(X86_TRAP_BR, exc_bounds);
599DECLARE_IDTENTRY(X86_TRAP_NM, exc_device_not_available);
600DECLARE_IDTENTRY(X86_TRAP_OLD_MF, exc_coproc_segment_overrun);
601DECLARE_IDTENTRY(X86_TRAP_SPURIOUS, exc_spurious_interrupt_bug);
602DECLARE_IDTENTRY(X86_TRAP_MF, exc_coprocessor_error);
603DECLARE_IDTENTRY(X86_TRAP_XF, exc_simd_coprocessor_error);
604
605/* 32bit software IRET trap. Do not emit ASM code */
606DECLARE_IDTENTRY_SW(X86_TRAP_IRET, iret_error);
607
608/* Simple exception entries with error code pushed by hardware */
609DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS, exc_invalid_tss);
610DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP, exc_segment_not_present);
611DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS, exc_stack_segment);
612DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP, exc_general_protection);
613DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC, exc_alignment_check);
614
615/* Raw exception entries which need extra work */
616DECLARE_IDTENTRY_RAW(X86_TRAP_UD, exc_invalid_op);
617DECLARE_IDTENTRY_RAW(X86_TRAP_BP, exc_int3);
618DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF, exc_page_fault);
619
620#if defined(CONFIG_IA32_EMULATION)
621DECLARE_IDTENTRY_RAW(IA32_SYSCALL_VECTOR, int80_emulation);
622#endif
623
624#ifdef CONFIG_X86_MCE
625#ifdef CONFIG_X86_64
626DECLARE_IDTENTRY_MCE(X86_TRAP_MC, exc_machine_check);
627#else
628DECLARE_IDTENTRY_RAW(X86_TRAP_MC, exc_machine_check);
629#endif
630#ifdef CONFIG_XEN_PV
631DECLARE_IDTENTRY_RAW(X86_TRAP_MC, xenpv_exc_machine_check);
632#endif
633#endif
634
635/* NMI */
636
637#if IS_ENABLED(CONFIG_KVM_INTEL)
638/*
639 * Special entry point for VMX which invokes this on the kernel stack, even for
640 * 64-bit, i.e. without using an IST. asm_exc_nmi() requires an IST to work
641 * correctly vs. the NMI 'executing' marker. Used for 32-bit kernels as well
642 * to avoid more ifdeffery.
643 */
644DECLARE_IDTENTRY(X86_TRAP_NMI, exc_nmi_kvm_vmx);
645#endif
646
647DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi);
648#ifdef CONFIG_XEN_PV
649DECLARE_IDTENTRY_RAW(X86_TRAP_NMI, xenpv_exc_nmi);
650#endif
651
652/* #DB */
653#ifdef CONFIG_X86_64
654DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB, exc_debug);
655#else
656DECLARE_IDTENTRY_RAW(X86_TRAP_DB, exc_debug);
657#endif
658#ifdef CONFIG_XEN_PV
659DECLARE_IDTENTRY_RAW(X86_TRAP_DB, xenpv_exc_debug);
660#endif
661
662/* #DF */
663DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault);
664#ifdef CONFIG_XEN_PV
665DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault);
666#endif
667
668/* #CP */
669#ifdef CONFIG_X86_CET
670DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP, exc_control_protection);
671#endif
672
673/* #VC */
674#ifdef CONFIG_AMD_MEM_ENCRYPT
675DECLARE_IDTENTRY_VC(X86_TRAP_VC, exc_vmm_communication);
676#endif
677
678#ifdef CONFIG_XEN_PV
679DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback);
680DECLARE_IDTENTRY_RAW(X86_TRAP_OTHER, exc_xen_unknown_trap);
681#endif
682
683#ifdef CONFIG_INTEL_TDX_GUEST
684DECLARE_IDTENTRY(X86_TRAP_VE, exc_virtualization_exception);
685#endif
686
687/* Device interrupts common/spurious */
688DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, common_interrupt);
689#ifdef CONFIG_X86_LOCAL_APIC
690DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, spurious_interrupt);
691#endif
692
693/* System vector entry points */
694#ifdef CONFIG_X86_LOCAL_APIC
695DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR, sysvec_error_interrupt);
696DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR, sysvec_spurious_apic_interrupt);
697DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR, sysvec_apic_timer_interrupt);
698DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi);
699#endif
700
701#ifdef CONFIG_SMP
702DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi);
703DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot);
704DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single);
705DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function);
706#else
707# define fred_sysvec_reschedule_ipi NULL
708# define fred_sysvec_reboot NULL
709# define fred_sysvec_call_function_single NULL
710# define fred_sysvec_call_function NULL
711#endif
712
713#ifdef CONFIG_X86_LOCAL_APIC
714# ifdef CONFIG_X86_MCE_THRESHOLD
715DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR, sysvec_threshold);
716# else
717# define fred_sysvec_threshold NULL
718# endif
719
720# ifdef CONFIG_X86_MCE_AMD
721DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR, sysvec_deferred_error);
722# else
723# define fred_sysvec_deferred_error NULL
724# endif
725
726# ifdef CONFIG_X86_THERMAL_VECTOR
727DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR, sysvec_thermal);
728# else
729# define fred_sysvec_thermal NULL
730# endif
731
732# ifdef CONFIG_IRQ_WORK
733DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR, sysvec_irq_work);
734# else
735# define fred_sysvec_irq_work NULL
736# endif
737#endif
738
739#if IS_ENABLED(CONFIG_KVM)
740DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR, sysvec_kvm_posted_intr_ipi);
741DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR, sysvec_kvm_posted_intr_wakeup_ipi);
742DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR, sysvec_kvm_posted_intr_nested_ipi);
743#else
744# define fred_sysvec_kvm_posted_intr_ipi NULL
745# define fred_sysvec_kvm_posted_intr_wakeup_ipi NULL
746# define fred_sysvec_kvm_posted_intr_nested_ipi NULL
747#endif
748
749# ifdef CONFIG_X86_POSTED_MSI
750DECLARE_IDTENTRY_SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, sysvec_posted_msi_notification);
751#else
752# define fred_sysvec_posted_msi_notification NULL
753# endif
754
755#if IS_ENABLED(CONFIG_HYPERV)
756DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback);
757DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment);
758DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0);
759#endif
760
761#if IS_ENABLED(CONFIG_ACRN_GUEST)
762DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_acrn_hv_callback);
763#endif
764
765#ifdef CONFIG_XEN_PVHVM
766DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback);
767#endif
768
769#ifdef CONFIG_KVM_GUEST
770DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt);
771#endif
772
773#undef X86_TRAP_OTHER
774
775#endif
776