1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_SYNC_CORE_H
3#define _ASM_X86_SYNC_CORE_H
4
5#include <linux/preempt.h>
6#include <asm/processor.h>
7#include <asm/cpufeature.h>
8#include <asm/special_insns.h>
9
10#ifdef CONFIG_X86_32
11static __always_inline void iret_to_self(void)
12{
13 asm volatile (
14 "pushfl\n\t"
15 "pushl %%cs\n\t"
16 "pushl $1f\n\t"
17 "iret\n\t"
18 "1:"
19 : ASM_CALL_CONSTRAINT : : "memory");
20}
21#else
22static __always_inline void iret_to_self(void)
23{
24 unsigned int tmp;
25
26 asm volatile (
27 "mov %%ss, %0\n\t"
28 "pushq %q0\n\t"
29 "pushq %%rsp\n\t"
30 "addq $8, (%%rsp)\n\t"
31 "pushfq\n\t"
32 "mov %%cs, %0\n\t"
33 "pushq %q0\n\t"
34 "pushq $1f\n\t"
35 "iretq\n\t"
36 "1:"
37 : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
38}
39#endif /* CONFIG_X86_32 */
40
41/*
42 * This function forces the icache and prefetched instruction stream to
43 * catch up with reality in two very specific cases:
44 *
45 * a) Text was modified using one virtual address and is about to be executed
46 * from the same physical page at a different virtual address.
47 *
48 * b) Text was modified on a different CPU, may subsequently be
49 * executed on this CPU, and you want to make sure the new version
50 * gets executed. This generally means you're calling this in an IPI.
51 *
52 * If you're calling this for a different reason, you're probably doing
53 * it wrong.
54 *
55 * Like all of Linux's memory ordering operations, this is a
56 * compiler barrier as well.
57 */
58static __always_inline void sync_core(void)
59{
60 /*
61 * The SERIALIZE instruction is the most straightforward way to
62 * do this, but it is not universally available.
63 */
64 if (static_cpu_has(X86_FEATURE_SERIALIZE)) {
65 serialize();
66 return;
67 }
68
69 /*
70 * For all other processors, there are quite a few ways to do this.
71 * IRET-to-self is nice because it works on every CPU, at any CPL
72 * (so it's compatible with paravirtualization), and it never exits
73 * to a hypervisor. The only downsides are that it's a bit slow
74 * (it seems to be a bit more than 2x slower than the fastest
75 * options) and that it unmasks NMIs. The "push %cs" is needed,
76 * because in paravirtual environments __KERNEL_CS may not be a
77 * valid CS value when we do IRET directly.
78 *
79 * In case NMI unmasking or performance ever becomes a problem,
80 * the next best option appears to be MOV-to-CR2 and an
81 * unconditional jump. That sequence also works on all CPUs,
82 * but it will fault at CPL3 (i.e. Xen PV).
83 *
84 * CPUID is the conventional way, but it's nasty: it doesn't
85 * exist on some 486-like CPUs, and it usually exits to a
86 * hypervisor.
87 */
88 iret_to_self();
89}
90
91/*
92 * Ensure that a core serializing instruction is issued before returning
93 * to user-mode. x86 implements return to user-space through sysexit,
94 * sysrel, and sysretq, which are not core serializing.
95 */
96static inline void sync_core_before_usermode(void)
97{
98 /* With PTI, we unconditionally serialize before running user code. */
99 if (static_cpu_has(X86_FEATURE_PTI))
100 return;
101
102 /*
103 * Even if we're in an interrupt, we might reschedule before returning,
104 * in which case we could switch to a different thread in the same mm
105 * and return using SYSRET or SYSEXIT. Instead of trying to keep
106 * track of our need to sync the core, just sync right away.
107 */
108 sync_core();
109}
110
111#endif /* _ASM_X86_SYNC_CORE_H */
112