1// SPDX-License-Identifier: GPL-2.0-only
2#define pr_fmt(fmt) "SMP alternatives: " fmt
3
4#include <linux/mmu_context.h>
5#include <linux/perf_event.h>
6#include <linux/vmalloc.h>
7#include <linux/memory.h>
8#include <linux/execmem.h>
9
10#include <asm/text-patching.h>
11#include <asm/insn.h>
12#include <asm/ibt.h>
13#include <asm/set_memory.h>
14#include <asm/nmi.h>
15
16int __read_mostly alternatives_patched;
17
18EXPORT_SYMBOL_GPL(alternatives_patched);
19
20#define MAX_PATCH_LEN (255-1)
21
22#define DA_ALL (~0)
23#define DA_ALT 0x01
24#define DA_RET 0x02
25#define DA_RETPOLINE 0x04
26#define DA_ENDBR 0x08
27#define DA_SMP 0x10
28
29static unsigned int debug_alternative;
30
31static int __init debug_alt(char *str)
32{
33 if (str && *str == '=')
34 str++;
35
36 if (!str || kstrtouint(s: str, base: 0, res: &debug_alternative))
37 debug_alternative = DA_ALL;
38
39 return 1;
40}
41__setup("debug-alternative", debug_alt);
42
43static int noreplace_smp;
44
45static int __init setup_noreplace_smp(char *str)
46{
47 noreplace_smp = 1;
48 return 1;
49}
50__setup("noreplace-smp", setup_noreplace_smp);
51
52#define DPRINTK(type, fmt, args...) \
53do { \
54 if (debug_alternative & DA_##type) \
55 printk(KERN_DEBUG pr_fmt(fmt) "\n", ##args); \
56} while (0)
57
58#define DUMP_BYTES(type, buf, len, fmt, args...) \
59do { \
60 if (unlikely(debug_alternative & DA_##type)) { \
61 int j; \
62 \
63 if (!(len)) \
64 break; \
65 \
66 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
67 for (j = 0; j < (len) - 1; j++) \
68 printk(KERN_CONT "%02hhx ", buf[j]); \
69 printk(KERN_CONT "%02hhx\n", buf[j]); \
70 } \
71} while (0)
72
73static const unsigned char x86nops[] =
74{
75 BYTES_NOP1,
76 BYTES_NOP2,
77 BYTES_NOP3,
78 BYTES_NOP4,
79 BYTES_NOP5,
80 BYTES_NOP6,
81 BYTES_NOP7,
82 BYTES_NOP8,
83#ifdef CONFIG_64BIT
84 BYTES_NOP9,
85 BYTES_NOP10,
86 BYTES_NOP11,
87#endif
88};
89
90const unsigned char * const x86_nops[ASM_NOP_MAX+1] =
91{
92 NULL,
93 x86nops,
94 x86nops + 1,
95 x86nops + 1 + 2,
96 x86nops + 1 + 2 + 3,
97 x86nops + 1 + 2 + 3 + 4,
98 x86nops + 1 + 2 + 3 + 4 + 5,
99 x86nops + 1 + 2 + 3 + 4 + 5 + 6,
100 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
101#ifdef CONFIG_64BIT
102 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
103 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9,
104 x86nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10,
105#endif
106};
107
108#ifdef CONFIG_FINEIBT
109static bool cfi_paranoid __ro_after_init;
110#endif
111
112#ifdef CONFIG_MITIGATION_ITS
113
114#ifdef CONFIG_MODULES
115static struct module *its_mod;
116#endif
117static void *its_page;
118static unsigned int its_offset;
119struct its_array its_pages;
120
121static void *__its_alloc(struct its_array *pages)
122{
123 void *page __free(execmem) = execmem_alloc_rw(type: EXECMEM_MODULE_TEXT, PAGE_SIZE);
124 if (!page)
125 return NULL;
126
127 void *tmp = krealloc(pages->pages, (pages->num+1) * sizeof(void *),
128 GFP_KERNEL);
129 if (!tmp)
130 return NULL;
131
132 pages->pages = tmp;
133 pages->pages[pages->num++] = page;
134
135 return no_free_ptr(page);
136}
137
138/* Initialize a thunk with the "jmp *reg; int3" instructions. */
139static void *its_init_thunk(void *thunk, int reg)
140{
141 u8 *bytes = thunk;
142 int offset = 0;
143 int i = 0;
144
145#ifdef CONFIG_FINEIBT
146 if (cfi_paranoid) {
147 /*
148 * When ITS uses indirect branch thunk the fineibt_paranoid
149 * caller sequence doesn't fit in the caller site. So put the
150 * remaining part of the sequence (UDB + JNE) into the ITS
151 * thunk.
152 */
153 bytes[i++] = 0xd6; /* UDB */
154 bytes[i++] = 0x75; /* JNE */
155 bytes[i++] = 0xfd;
156
157 offset = 1;
158 }
159#endif
160
161 if (reg >= 8) {
162 bytes[i++] = 0x41; /* REX.B prefix */
163 reg -= 8;
164 }
165 bytes[i++] = 0xff;
166 bytes[i++] = 0xe0 + reg; /* JMP *reg */
167 bytes[i++] = 0xcc;
168
169 return thunk + offset;
170}
171
172static void its_pages_protect(struct its_array *pages)
173{
174 for (int i = 0; i < pages->num; i++) {
175 void *page = pages->pages[i];
176 execmem_restore_rox(ptr: page, PAGE_SIZE);
177 }
178}
179
180static void its_fini_core(void)
181{
182 if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX))
183 its_pages_protect(pages: &its_pages);
184 kfree(objp: its_pages.pages);
185}
186
187#ifdef CONFIG_MODULES
188void its_init_mod(struct module *mod)
189{
190 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
191 return;
192
193 mutex_lock(lock: &text_mutex);
194 its_mod = mod;
195 its_page = NULL;
196}
197
198void its_fini_mod(struct module *mod)
199{
200 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
201 return;
202
203 WARN_ON_ONCE(its_mod != mod);
204
205 its_mod = NULL;
206 its_page = NULL;
207 mutex_unlock(lock: &text_mutex);
208
209 if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
210 its_pages_protect(pages: &mod->arch.its_pages);
211}
212
213void its_free_mod(struct module *mod)
214{
215 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
216 return;
217
218 for (int i = 0; i < mod->arch.its_pages.num; i++) {
219 void *page = mod->arch.its_pages.pages[i];
220 execmem_free(ptr: page);
221 }
222 kfree(objp: mod->arch.its_pages.pages);
223}
224#endif /* CONFIG_MODULES */
225
226static void *its_alloc(void)
227{
228 struct its_array *pages = &its_pages;
229 void *page;
230
231#ifdef CONFIG_MODULES
232 if (its_mod)
233 pages = &its_mod->arch.its_pages;
234#endif
235
236 page = __its_alloc(pages);
237 if (!page)
238 return NULL;
239
240 if (pages == &its_pages)
241 set_memory_x(addr: (unsigned long)page, numpages: 1);
242
243 return page;
244}
245
246static void *its_allocate_thunk(int reg)
247{
248 int size = 3 + (reg / 8);
249 void *thunk;
250
251#ifdef CONFIG_FINEIBT
252 /*
253 * The ITS thunk contains an indirect jump and an int3 instruction so
254 * its size is 3 or 4 bytes depending on the register used. If CFI
255 * paranoid is used then 3 extra bytes are added in the ITS thunk to
256 * complete the fineibt_paranoid caller sequence.
257 */
258 if (cfi_paranoid)
259 size += 3;
260#endif
261
262 if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) {
263 its_page = its_alloc();
264 if (!its_page) {
265 pr_err("ITS page allocation failed\n");
266 return NULL;
267 }
268 memset(s: its_page, INT3_INSN_OPCODE, PAGE_SIZE);
269 its_offset = 32;
270 }
271
272 /*
273 * If the indirect branch instruction will be in the lower half
274 * of a cacheline, then update the offset to reach the upper half.
275 */
276 if ((its_offset + size - 1) % 64 < 32)
277 its_offset = ((its_offset - 1) | 0x3F) + 33;
278
279 thunk = its_page + its_offset;
280 its_offset += size;
281
282 return its_init_thunk(thunk, reg);
283}
284
285u8 *its_static_thunk(int reg)
286{
287 u8 *thunk = __x86_indirect_its_thunk_array[reg];
288
289#ifdef CONFIG_FINEIBT
290 /* Paranoid thunk starts 2 bytes before */
291 if (cfi_paranoid)
292 return thunk - 2;
293#endif
294 return thunk;
295}
296
297#else
298static inline void its_fini_core(void) {}
299#endif /* CONFIG_MITIGATION_ITS */
300
301/*
302 * Nomenclature for variable names to simplify and clarify this code and ease
303 * any potential staring at it:
304 *
305 * @instr: source address of the original instructions in the kernel text as
306 * generated by the compiler.
307 *
308 * @buf: temporary buffer on which the patching operates. This buffer is
309 * eventually text-poked into the kernel image.
310 *
311 * @replacement/@repl: pointer to the opcodes which are replacing @instr, located
312 * in the .altinstr_replacement section.
313 */
314
315/*
316 * Fill the buffer with a single effective instruction of size @len.
317 *
318 * In order not to issue an ORC stack depth tracking CFI entry (Call Frame Info)
319 * for every single-byte NOP, try to generate the maximally available NOP of
320 * size <= ASM_NOP_MAX such that only a single CFI entry is generated (vs one for
321 * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and
322 * *jump* over instead of executing long and daft NOPs.
323 */
324static void add_nop(u8 *buf, unsigned int len)
325{
326 u8 *target = buf + len;
327
328 if (!len)
329 return;
330
331 if (len <= ASM_NOP_MAX) {
332 memcpy(to: buf, from: x86_nops[len], len);
333 return;
334 }
335
336 if (len < 128) {
337 __text_gen_insn(buf, JMP8_INSN_OPCODE, addr: buf, dest: target, JMP8_INSN_SIZE);
338 buf += JMP8_INSN_SIZE;
339 } else {
340 __text_gen_insn(buf, JMP32_INSN_OPCODE, addr: buf, dest: target, JMP32_INSN_SIZE);
341 buf += JMP32_INSN_SIZE;
342 }
343
344 for (;buf < target; buf++)
345 *buf = INT3_INSN_OPCODE;
346}
347
348/*
349 * Matches NOP and NOPL, not any of the other possible NOPs.
350 */
351static bool insn_is_nop(struct insn *insn)
352{
353 /* Anything NOP, but no REP NOP */
354 if (insn->opcode.bytes[0] == 0x90 &&
355 (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3))
356 return true;
357
358 /* NOPL */
359 if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F)
360 return true;
361
362 /* TODO: more nops */
363
364 return false;
365}
366
367/*
368 * Find the offset of the first non-NOP instruction starting at @offset
369 * but no further than @len.
370 */
371static int skip_nops(u8 *buf, int offset, int len)
372{
373 struct insn insn;
374
375 for (; offset < len; offset += insn.length) {
376 if (insn_decode_kernel(&insn, &buf[offset]))
377 break;
378
379 if (!insn_is_nop(insn: &insn))
380 break;
381 }
382
383 return offset;
384}
385
386/*
387 * "noinline" to cause control flow change and thus invalidate I$ and
388 * cause refetch after modification.
389 */
390static void noinline optimize_nops(const u8 * const instr, u8 *buf, size_t len)
391{
392 for (int next, i = 0; i < len; i = next) {
393 struct insn insn;
394
395 if (insn_decode_kernel(&insn, &buf[i]))
396 return;
397
398 next = i + insn.length;
399
400 if (insn_is_nop(insn: &insn)) {
401 int nop = i;
402
403 /* Has the NOP already been optimized? */
404 if (i + insn.length == len)
405 return;
406
407 next = skip_nops(buf, offset: next, len);
408
409 add_nop(buf: buf + nop, len: next - nop);
410 DUMP_BYTES(ALT, buf, len, "%px: [%d:%d) optimized NOPs: ", instr, nop, next);
411 }
412 }
413}
414
415/*
416 * In this context, "source" is where the instructions are placed in the
417 * section .altinstr_replacement, for example during kernel build by the
418 * toolchain.
419 * "Destination" is where the instructions are being patched in by this
420 * machinery.
421 *
422 * The source offset is:
423 *
424 * src_imm = target - src_next_ip (1)
425 *
426 * and the target offset is:
427 *
428 * dst_imm = target - dst_next_ip (2)
429 *
430 * so rework (1) as an expression for target like:
431 *
432 * target = src_imm + src_next_ip (1a)
433 *
434 * and substitute in (2) to get:
435 *
436 * dst_imm = (src_imm + src_next_ip) - dst_next_ip (3)
437 *
438 * Now, since the instruction stream is 'identical' at src and dst (it
439 * is being copied after all) it can be stated that:
440 *
441 * src_next_ip = src + ip_offset
442 * dst_next_ip = dst + ip_offset (4)
443 *
444 * Substitute (4) in (3) and observe ip_offset being cancelled out to
445 * obtain:
446 *
447 * dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset)
448 * = src_imm + src - dst + ip_offset - ip_offset
449 * = src_imm + src - dst (5)
450 *
451 * IOW, only the relative displacement of the code block matters.
452 */
453
454#define apply_reloc_n(n_, p_, d_) \
455 do { \
456 s32 v = *(s##n_ *)(p_); \
457 v += (d_); \
458 BUG_ON((v >> 31) != (v >> (n_-1))); \
459 *(s##n_ *)(p_) = (s##n_)v; \
460 } while (0)
461
462
463static __always_inline
464void apply_reloc(int n, void *ptr, uintptr_t diff)
465{
466 switch (n) {
467 case 1: apply_reloc_n(8, ptr, diff); break;
468 case 2: apply_reloc_n(16, ptr, diff); break;
469 case 4: apply_reloc_n(32, ptr, diff); break;
470 default: BUG();
471 }
472}
473
474static __always_inline
475bool need_reloc(unsigned long offset, u8 *src, size_t src_len)
476{
477 u8 *target = src + offset;
478 /*
479 * If the target is inside the patched block, it's relative to the
480 * block itself and does not need relocation.
481 */
482 return (target < src || target > src + src_len);
483}
484
485static void __apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len)
486{
487 for (int next, i = 0; i < instrlen; i = next) {
488 struct insn insn;
489
490 if (WARN_ON_ONCE(insn_decode_kernel(&insn, &buf[i])))
491 return;
492
493 next = i + insn.length;
494
495 switch (insn.opcode.bytes[0]) {
496 case 0x0f:
497 if (insn.opcode.bytes[1] < 0x80 ||
498 insn.opcode.bytes[1] > 0x8f)
499 break;
500
501 fallthrough; /* Jcc.d32 */
502 case 0x70 ... 0x7f: /* Jcc.d8 */
503 case JMP8_INSN_OPCODE:
504 case JMP32_INSN_OPCODE:
505 case CALL_INSN_OPCODE:
506 if (need_reloc(offset: next + insn.immediate.value, src: repl, src_len: repl_len)) {
507 apply_reloc(n: insn.immediate.nbytes,
508 ptr: buf + i + insn_offset_immediate(insn: &insn),
509 diff: repl - instr);
510 }
511
512 /*
513 * Where possible, convert JMP.d32 into JMP.d8.
514 */
515 if (insn.opcode.bytes[0] == JMP32_INSN_OPCODE) {
516 s32 imm = insn.immediate.value;
517 imm += repl - instr;
518 imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE;
519 if ((imm >> 31) == (imm >> 7)) {
520 buf[i+0] = JMP8_INSN_OPCODE;
521 buf[i+1] = (s8)imm;
522
523 memset(s: &buf[i+2], INT3_INSN_OPCODE, n: insn.length - 2);
524 }
525 }
526 break;
527 }
528
529 if (insn_rip_relative(insn: &insn)) {
530 if (need_reloc(offset: next + insn.displacement.value, src: repl, src_len: repl_len)) {
531 apply_reloc(n: insn.displacement.nbytes,
532 ptr: buf + i + insn_offset_displacement(insn: &insn),
533 diff: repl - instr);
534 }
535 }
536 }
537}
538
539void text_poke_apply_relocation(u8 *buf, const u8 * const instr, size_t instrlen, u8 *repl, size_t repl_len)
540{
541 __apply_relocation(buf, instr, instrlen, repl, repl_len);
542 optimize_nops(instr, buf, len: instrlen);
543}
544
545/* Low-level backend functions usable from alternative code replacements. */
546DEFINE_ASM_FUNC(nop_func, "", .entry.text);
547EXPORT_SYMBOL_GPL(nop_func);
548
549noinstr void BUG_func(void)
550{
551 BUG();
552}
553EXPORT_SYMBOL(BUG_func);
554
555#define CALL_RIP_REL_OPCODE 0xff
556#define CALL_RIP_REL_MODRM 0x15
557
558/*
559 * Rewrite the "call BUG_func" replacement to point to the target of the
560 * indirect pv_ops call "call *disp(%ip)".
561 */
562static int alt_replace_call(u8 *instr, u8 *insn_buff, struct alt_instr *a)
563{
564 void *target, *bug = &BUG_func;
565 s32 disp;
566
567 if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) {
568 pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n");
569 BUG();
570 }
571
572 if (a->instrlen != 6 ||
573 instr[0] != CALL_RIP_REL_OPCODE ||
574 instr[1] != CALL_RIP_REL_MODRM) {
575 pr_err("ALT_FLAG_DIRECT_CALL set for unrecognized indirect call\n");
576 BUG();
577 }
578
579 /* Skip CALL_RIP_REL_OPCODE and CALL_RIP_REL_MODRM */
580 disp = *(s32 *)(instr + 2);
581#ifdef CONFIG_X86_64
582 /* ff 15 00 00 00 00 call *0x0(%rip) */
583 /* target address is stored at "next instruction + disp". */
584 target = *(void **)(instr + a->instrlen + disp);
585#else
586 /* ff 15 00 00 00 00 call *0x0 */
587 /* target address is stored at disp. */
588 target = *(void **)disp;
589#endif
590 if (!target)
591 target = bug;
592
593 /* (BUG_func - .) + (target - BUG_func) := target - . */
594 *(s32 *)(insn_buff + 1) += target - bug;
595
596 if (target == &nop_func)
597 return 0;
598
599 return 5;
600}
601
602static inline u8 * instr_va(struct alt_instr *i)
603{
604 return (u8 *)&i->instr_offset + i->instr_offset;
605}
606
607/*
608 * Replace instructions with better alternatives for this CPU type. This runs
609 * before SMP is initialized to avoid SMP problems with self modifying code.
610 * This implies that asymmetric systems where APs have less capabilities than
611 * the boot processor are not handled. Tough. Make sure you disable such
612 * features by hand.
613 *
614 * Marked "noinline" to cause control flow change and thus insn cache
615 * to refetch changed I$ lines.
616 */
617void __init_or_module noinline apply_alternatives(struct alt_instr *start,
618 struct alt_instr *end)
619{
620 u8 insn_buff[MAX_PATCH_LEN];
621 u8 *instr, *replacement;
622 struct alt_instr *a, *b;
623
624 DPRINTK(ALT, "alt table %px, -> %px", start, end);
625
626 /*
627 * KASAN_SHADOW_START is defined using
628 * cpu_feature_enabled(X86_FEATURE_LA57) and is therefore patched here.
629 * During the process, KASAN becomes confused seeing partial LA57
630 * conversion and triggers a false-positive out-of-bound report.
631 *
632 * Disable KASAN until the patching is complete.
633 */
634 kasan_disable_current();
635
636 /*
637 * The scan order should be from start to end. A later scanned
638 * alternative code can overwrite previously scanned alternative code.
639 * Some kernel functions (e.g. memcpy, memset, etc) use this order to
640 * patch code.
641 *
642 * So be careful if you want to change the scan order to any other
643 * order.
644 */
645 for (a = start; a < end; a++) {
646 int insn_buff_sz = 0;
647
648 /*
649 * In case of nested ALTERNATIVE()s the outer alternative might
650 * add more padding. To ensure consistent patching find the max
651 * padding for all alt_instr entries for this site (nested
652 * alternatives result in consecutive entries).
653 */
654 for (b = a+1; b < end && instr_va(i: b) == instr_va(i: a); b++) {
655 u8 len = max(a->instrlen, b->instrlen);
656 a->instrlen = b->instrlen = len;
657 }
658
659 instr = instr_va(i: a);
660 replacement = (u8 *)&a->repl_offset + a->repl_offset;
661 BUG_ON(a->instrlen > sizeof(insn_buff));
662 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
663
664 /*
665 * Patch if either:
666 * - feature is present
667 * - feature not present but ALT_FLAG_NOT is set to mean,
668 * patch if feature is *NOT* present.
669 */
670 if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) {
671 memcpy(to: insn_buff, from: instr, len: a->instrlen);
672 optimize_nops(instr, buf: insn_buff, len: a->instrlen);
673 text_poke_early(addr: instr, opcode: insn_buff, len: a->instrlen);
674 continue;
675 }
676
677 DPRINTK(ALT, "feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d) flags: 0x%x",
678 a->cpuid >> 5,
679 a->cpuid & 0x1f,
680 instr, instr, a->instrlen,
681 replacement, a->replacementlen, a->flags);
682
683 memcpy(to: insn_buff, from: replacement, len: a->replacementlen);
684 insn_buff_sz = a->replacementlen;
685
686 if (a->flags & ALT_FLAG_DIRECT_CALL) {
687 insn_buff_sz = alt_replace_call(instr, insn_buff, a);
688 if (insn_buff_sz < 0)
689 continue;
690 }
691
692 for (; insn_buff_sz < a->instrlen; insn_buff_sz++)
693 insn_buff[insn_buff_sz] = 0x90;
694
695 text_poke_apply_relocation(buf: insn_buff, instr, instrlen: a->instrlen, repl: replacement, repl_len: a->replacementlen);
696
697 DUMP_BYTES(ALT, instr, a->instrlen, "%px: old_insn: ", instr);
698 DUMP_BYTES(ALT, replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
699 DUMP_BYTES(ALT, insn_buff, insn_buff_sz, "%px: final_insn: ", instr);
700
701 text_poke_early(addr: instr, opcode: insn_buff, len: insn_buff_sz);
702 }
703
704 kasan_enable_current();
705}
706
707static inline bool is_jcc32(struct insn *insn)
708{
709 /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */
710 return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80;
711}
712
713#if defined(CONFIG_MITIGATION_RETPOLINE) && defined(CONFIG_OBJTOOL)
714
715/*
716 * [CS]{,3} CALL/JMP *%\reg [INT3]*
717 */
718static int emit_indirect(int op, int reg, u8 *bytes, int len)
719{
720 int cs = 0, bp = 0;
721 int i = 0;
722 u8 modrm;
723
724 /*
725 * Set @len to the excess bytes after writing the instruction.
726 */
727 len -= 2 + (reg >= 8);
728 WARN_ON_ONCE(len < 0);
729
730 switch (op) {
731 case CALL_INSN_OPCODE:
732 modrm = 0x10; /* Reg = 2; CALL r/m */
733 /*
734 * Additional NOP is better than prefix decode penalty.
735 */
736 if (len <= 3)
737 cs = len;
738 break;
739
740 case JMP32_INSN_OPCODE:
741 modrm = 0x20; /* Reg = 4; JMP r/m */
742 bp = len;
743 break;
744
745 default:
746 WARN_ON_ONCE(1);
747 return -1;
748 }
749
750 while (cs--)
751 bytes[i++] = 0x2e; /* CS-prefix */
752
753 if (reg >= 8) {
754 bytes[i++] = 0x41; /* REX.B prefix */
755 reg -= 8;
756 }
757
758 modrm |= 0xc0; /* Mod = 3 */
759 modrm += reg;
760
761 bytes[i++] = 0xff; /* opcode */
762 bytes[i++] = modrm;
763
764 while (bp--)
765 bytes[i++] = 0xcc; /* INT3 */
766
767 return i;
768}
769
770static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes,
771 void *call_dest, void *jmp_dest)
772{
773 u8 op = insn->opcode.bytes[0];
774 int i = 0;
775
776 /*
777 * Clang does 'weird' Jcc __x86_indirect_thunk_r11 conditional
778 * tail-calls. Deal with them.
779 */
780 if (is_jcc32(insn)) {
781 bytes[i++] = op;
782 op = insn->opcode.bytes[1];
783 goto clang_jcc;
784 }
785
786 if (insn->length == 6)
787 bytes[i++] = 0x2e; /* CS-prefix */
788
789 switch (op) {
790 case CALL_INSN_OPCODE:
791 __text_gen_insn(buf: bytes+i, opcode: op, addr: addr+i,
792 dest: call_dest,
793 CALL_INSN_SIZE);
794 i += CALL_INSN_SIZE;
795 break;
796
797 case JMP32_INSN_OPCODE:
798clang_jcc:
799 __text_gen_insn(buf: bytes+i, opcode: op, addr: addr+i,
800 dest: jmp_dest,
801 JMP32_INSN_SIZE);
802 i += JMP32_INSN_SIZE;
803 break;
804
805 default:
806 WARN(1, "%pS %px %*ph\n", addr, addr, 6, addr);
807 return -1;
808 }
809
810 WARN_ON_ONCE(i != insn->length);
811
812 return i;
813}
814
815static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
816{
817 return __emit_trampoline(addr, insn, bytes,
818 call_dest: __x86_indirect_call_thunk_array[reg],
819 jmp_dest: __x86_indirect_jump_thunk_array[reg]);
820}
821
822#ifdef CONFIG_MITIGATION_ITS
823static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
824{
825 u8 *thunk = __x86_indirect_its_thunk_array[reg];
826 u8 *tmp = its_allocate_thunk(reg);
827
828 if (tmp)
829 thunk = tmp;
830
831 return __emit_trampoline(addr, insn, bytes, call_dest: thunk, jmp_dest: thunk);
832}
833
834/* Check if an indirect branch is at ITS-unsafe address */
835static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
836{
837 if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
838 return false;
839
840 /* Indirect branch opcode is 2 or 3 bytes depending on reg */
841 addr += 1 + reg / 8;
842
843 /* Lower-half of the cacheline? */
844 return !(addr & 0x20);
845}
846#else /* CONFIG_MITIGATION_ITS */
847
848#ifdef CONFIG_FINEIBT
849static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
850{
851 return false;
852}
853#endif
854
855#endif /* CONFIG_MITIGATION_ITS */
856
857/*
858 * Rewrite the compiler generated retpoline thunk calls.
859 *
860 * For spectre_v2=off (!X86_FEATURE_RETPOLINE), rewrite them into immediate
861 * indirect instructions, avoiding the extra indirection.
862 *
863 * For example, convert:
864 *
865 * CALL __x86_indirect_thunk_\reg
866 *
867 * into:
868 *
869 * CALL *%\reg
870 *
871 * It also tries to inline spectre_v2=retpoline,lfence when size permits.
872 */
873static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
874{
875 retpoline_thunk_t *target;
876 int reg, ret, i = 0;
877 u8 op, cc;
878
879 target = addr + insn->length + insn->immediate.value;
880 reg = target - __x86_indirect_thunk_array;
881
882 if (WARN_ON_ONCE(reg & ~0xf))
883 return -1;
884
885 /* If anyone ever does: CALL/JMP *%rsp, we're in deep trouble. */
886 BUG_ON(reg == 4);
887
888 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE) &&
889 !cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
890 if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
891 return emit_call_track_retpoline(addr, insn, reg, bytes);
892
893 return -1;
894 }
895
896 op = insn->opcode.bytes[0];
897
898 /*
899 * Convert:
900 *
901 * Jcc.d32 __x86_indirect_thunk_\reg
902 *
903 * into:
904 *
905 * Jncc.d8 1f
906 * [ LFENCE ]
907 * JMP *%\reg
908 * [ NOP ]
909 * 1:
910 */
911 if (is_jcc32(insn)) {
912 cc = insn->opcode.bytes[1] & 0xf;
913 cc ^= 1; /* invert condition */
914
915 bytes[i++] = 0x70 + cc; /* Jcc.d8 */
916 bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */
917
918 /* Continue as if: JMP.d32 __x86_indirect_thunk_\reg */
919 op = JMP32_INSN_OPCODE;
920 }
921
922 /*
923 * For RETPOLINE_LFENCE: prepend the indirect CALL/JMP with an LFENCE.
924 */
925 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
926 bytes[i++] = 0x0f;
927 bytes[i++] = 0xae;
928 bytes[i++] = 0xe8; /* LFENCE */
929 }
930
931#ifdef CONFIG_MITIGATION_ITS
932 /*
933 * Check if the address of last byte of emitted-indirect is in
934 * lower-half of the cacheline. Such branches need ITS mitigation.
935 */
936 if (cpu_wants_indirect_its_thunk_at(addr: (unsigned long)addr + i, reg))
937 return emit_its_trampoline(addr, insn, reg, bytes);
938#endif
939
940 ret = emit_indirect(op, reg, bytes: bytes + i, len: insn->length - i);
941 if (ret < 0)
942 return ret;
943 i += ret;
944
945 for (; i < insn->length;)
946 bytes[i++] = BYTES_NOP1;
947
948 return i;
949}
950
951/*
952 * Generated by 'objtool --retpoline'.
953 */
954void __init_or_module noinline apply_retpolines(s32 *start, s32 *end)
955{
956 s32 *s;
957
958 for (s = start; s < end; s++) {
959 void *addr = (void *)s + *s;
960 struct insn insn;
961 int len, ret;
962 u8 bytes[16];
963 u8 op1, op2;
964 u8 *dest;
965
966 ret = insn_decode_kernel(&insn, addr);
967 if (WARN_ON_ONCE(ret < 0))
968 continue;
969
970 op1 = insn.opcode.bytes[0];
971 op2 = insn.opcode.bytes[1];
972
973 switch (op1) {
974 case 0x70 ... 0x7f: /* Jcc.d8 */
975 /* See cfi_paranoid. */
976 WARN_ON_ONCE(cfi_mode != CFI_FINEIBT);
977 continue;
978
979 case CALL_INSN_OPCODE:
980 case JMP32_INSN_OPCODE:
981 /* Check for cfi_paranoid + ITS */
982 dest = addr + insn.length + insn.immediate.value;
983 if (dest[-1] == 0xd6 && (dest[0] & 0xf0) == 0x70) {
984 WARN_ON_ONCE(cfi_mode != CFI_FINEIBT);
985 continue;
986 }
987 break;
988
989 case 0x0f: /* escape */
990 if (op2 >= 0x80 && op2 <= 0x8f)
991 break;
992 fallthrough;
993 default:
994 WARN_ON_ONCE(1);
995 continue;
996 }
997
998 DPRINTK(RETPOLINE, "retpoline at: %pS (%px) len: %d to: %pS",
999 addr, addr, insn.length,
1000 addr + insn.length + insn.immediate.value);
1001
1002 len = patch_retpoline(addr, insn: &insn, bytes);
1003 if (len == insn.length) {
1004 optimize_nops(instr: addr, buf: bytes, len);
1005 DUMP_BYTES(RETPOLINE, ((u8*)addr), len, "%px: orig: ", addr);
1006 DUMP_BYTES(RETPOLINE, ((u8*)bytes), len, "%px: repl: ", addr);
1007 text_poke_early(addr, opcode: bytes, len);
1008 }
1009 }
1010}
1011
1012#ifdef CONFIG_MITIGATION_RETHUNK
1013
1014bool cpu_wants_rethunk(void)
1015{
1016 return cpu_feature_enabled(X86_FEATURE_RETHUNK);
1017}
1018
1019bool cpu_wants_rethunk_at(void *addr)
1020{
1021 if (!cpu_feature_enabled(X86_FEATURE_RETHUNK))
1022 return false;
1023 if (x86_return_thunk != its_return_thunk)
1024 return true;
1025
1026 return !((unsigned long)addr & 0x20);
1027}
1028
1029/*
1030 * Rewrite the compiler generated return thunk tail-calls.
1031 *
1032 * For example, convert:
1033 *
1034 * JMP __x86_return_thunk
1035 *
1036 * into:
1037 *
1038 * RET
1039 */
1040static int patch_return(void *addr, struct insn *insn, u8 *bytes)
1041{
1042 int i = 0;
1043
1044 /* Patch the custom return thunks... */
1045 if (cpu_wants_rethunk_at(addr)) {
1046 i = JMP32_INSN_SIZE;
1047 __text_gen_insn(buf: bytes, JMP32_INSN_OPCODE, addr, dest: x86_return_thunk, size: i);
1048 } else {
1049 /* ... or patch them out if not needed. */
1050 bytes[i++] = RET_INSN_OPCODE;
1051 }
1052
1053 for (; i < insn->length;)
1054 bytes[i++] = INT3_INSN_OPCODE;
1055 return i;
1056}
1057
1058void __init_or_module noinline apply_returns(s32 *start, s32 *end)
1059{
1060 s32 *s;
1061
1062 if (cpu_wants_rethunk())
1063 static_call_force_reinit();
1064
1065 for (s = start; s < end; s++) {
1066 void *dest = NULL, *addr = (void *)s + *s;
1067 struct insn insn;
1068 int len, ret;
1069 u8 bytes[16];
1070 u8 op;
1071
1072 ret = insn_decode_kernel(&insn, addr);
1073 if (WARN_ON_ONCE(ret < 0))
1074 continue;
1075
1076 op = insn.opcode.bytes[0];
1077 if (op == JMP32_INSN_OPCODE)
1078 dest = addr + insn.length + insn.immediate.value;
1079
1080 if (__static_call_fixup(tramp: addr, op, dest) ||
1081 WARN_ONCE(dest != &__x86_return_thunk,
1082 "missing return thunk: %pS-%pS: %*ph",
1083 addr, dest, 5, addr))
1084 continue;
1085
1086 DPRINTK(RET, "return thunk at: %pS (%px) len: %d to: %pS",
1087 addr, addr, insn.length,
1088 addr + insn.length + insn.immediate.value);
1089
1090 len = patch_return(addr, insn: &insn, bytes);
1091 if (len == insn.length) {
1092 DUMP_BYTES(RET, ((u8*)addr), len, "%px: orig: ", addr);
1093 DUMP_BYTES(RET, ((u8*)bytes), len, "%px: repl: ", addr);
1094 text_poke_early(addr, opcode: bytes, len);
1095 }
1096 }
1097}
1098#else /* !CONFIG_MITIGATION_RETHUNK: */
1099void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
1100#endif /* !CONFIG_MITIGATION_RETHUNK */
1101
1102#else /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */
1103
1104void __init_or_module noinline apply_retpolines(s32 *start, s32 *end) { }
1105void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
1106
1107#endif /* !CONFIG_MITIGATION_RETPOLINE || !CONFIG_OBJTOOL */
1108
1109#ifdef CONFIG_X86_KERNEL_IBT
1110
1111__noendbr bool is_endbr(u32 *val)
1112{
1113 u32 endbr;
1114
1115 __get_kernel_nofault(&endbr, val, u32, Efault);
1116 return __is_endbr(val: endbr);
1117
1118Efault:
1119 return false;
1120}
1121
1122#ifdef CONFIG_FINEIBT
1123
1124static __noendbr bool exact_endbr(u32 *val)
1125{
1126 u32 endbr;
1127
1128 __get_kernel_nofault(&endbr, val, u32, Efault);
1129 return endbr == gen_endbr();
1130
1131Efault:
1132 return false;
1133}
1134
1135#endif
1136
1137static void poison_cfi(void *addr);
1138
1139static void __init_or_module poison_endbr(void *addr)
1140{
1141 u32 poison = gen_endbr_poison();
1142
1143 if (WARN_ON_ONCE(!is_endbr(addr)))
1144 return;
1145
1146 DPRINTK(ENDBR, "ENDBR at: %pS (%px)", addr, addr);
1147
1148 /*
1149 * When we have IBT, the lack of ENDBR will trigger #CP
1150 */
1151 DUMP_BYTES(ENDBR, ((u8*)addr), 4, "%px: orig: ", addr);
1152 DUMP_BYTES(ENDBR, ((u8*)&poison), 4, "%px: repl: ", addr);
1153 text_poke_early(addr, opcode: &poison, len: 4);
1154}
1155
1156/*
1157 * Generated by: objtool --ibt
1158 *
1159 * Seal the functions for indirect calls by clobbering the ENDBR instructions
1160 * and the kCFI hash value.
1161 */
1162void __init_or_module noinline apply_seal_endbr(s32 *start, s32 *end)
1163{
1164 s32 *s;
1165
1166 for (s = start; s < end; s++) {
1167 void *addr = (void *)s + *s;
1168
1169 poison_endbr(addr);
1170 if (IS_ENABLED(CONFIG_FINEIBT))
1171 poison_cfi(addr: addr - 16);
1172 }
1173}
1174
1175#else /* !CONFIG_X86_KERNEL_IBT: */
1176
1177void __init_or_module apply_seal_endbr(s32 *start, s32 *end) { }
1178
1179#endif /* !CONFIG_X86_KERNEL_IBT */
1180
1181#ifdef CONFIG_CFI_AUTO_DEFAULT
1182# define __CFI_DEFAULT CFI_AUTO
1183#elif defined(CONFIG_CFI)
1184# define __CFI_DEFAULT CFI_KCFI
1185#else
1186# define __CFI_DEFAULT CFI_OFF
1187#endif
1188
1189enum cfi_mode cfi_mode __ro_after_init = __CFI_DEFAULT;
1190static bool cfi_debug __ro_after_init;
1191
1192#ifdef CONFIG_FINEIBT_BHI
1193bool cfi_bhi __ro_after_init = false;
1194#endif
1195
1196#ifdef CONFIG_CFI
1197u32 cfi_get_func_hash(void *func)
1198{
1199 u32 hash;
1200
1201 func -= cfi_get_offset();
1202 switch (cfi_mode) {
1203 case CFI_FINEIBT:
1204 func += 7;
1205 break;
1206 case CFI_KCFI:
1207 func += 1;
1208 break;
1209 default:
1210 return 0;
1211 }
1212
1213 if (get_kernel_nofault(hash, func))
1214 return 0;
1215
1216 return hash;
1217}
1218
1219int cfi_get_func_arity(void *func)
1220{
1221 bhi_thunk *target;
1222 s32 disp;
1223
1224 if (cfi_mode != CFI_FINEIBT && !cfi_bhi)
1225 return 0;
1226
1227 if (get_kernel_nofault(disp, func - 4))
1228 return 0;
1229
1230 target = func + disp;
1231 return target - __bhi_args;
1232}
1233#endif
1234
1235#ifdef CONFIG_FINEIBT
1236
1237static bool cfi_rand __ro_after_init = true;
1238static u32 cfi_seed __ro_after_init;
1239
1240/*
1241 * Re-hash the CFI hash with a boot-time seed while making sure the result is
1242 * not a valid ENDBR instruction.
1243 */
1244static u32 cfi_rehash(u32 hash)
1245{
1246 hash ^= cfi_seed;
1247 while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) {
1248 bool lsb = hash & 1;
1249 hash >>= 1;
1250 if (lsb)
1251 hash ^= 0x80200003;
1252 }
1253 return hash;
1254}
1255
1256static __init int cfi_parse_cmdline(char *str)
1257{
1258 if (!str)
1259 return -EINVAL;
1260
1261 while (str) {
1262 char *next = strchr(str, ',');
1263 if (next) {
1264 *next = 0;
1265 next++;
1266 }
1267
1268 if (!strcmp(str, "auto")) {
1269 cfi_mode = CFI_AUTO;
1270 } else if (!strcmp(str, "off")) {
1271 cfi_mode = CFI_OFF;
1272 cfi_rand = false;
1273 } else if (!strcmp(str, "debug")) {
1274 cfi_debug = true;
1275 } else if (!strcmp(str, "kcfi")) {
1276 cfi_mode = CFI_KCFI;
1277 } else if (!strcmp(str, "fineibt")) {
1278 cfi_mode = CFI_FINEIBT;
1279 } else if (!strcmp(str, "norand")) {
1280 cfi_rand = false;
1281 } else if (!strcmp(str, "warn")) {
1282 pr_alert("CFI: mismatch non-fatal!\n");
1283 cfi_warn = true;
1284 } else if (!strcmp(str, "paranoid")) {
1285 if (cfi_mode == CFI_FINEIBT) {
1286 cfi_paranoid = true;
1287 } else {
1288 pr_err("CFI: ignoring paranoid; depends on fineibt.\n");
1289 }
1290 } else if (!strcmp(str, "bhi")) {
1291#ifdef CONFIG_FINEIBT_BHI
1292 if (cfi_mode == CFI_FINEIBT) {
1293 cfi_bhi = true;
1294 } else {
1295 pr_err("CFI: ignoring bhi; depends on fineibt.\n");
1296 }
1297#else
1298 pr_err("CFI: ignoring bhi; depends on FINEIBT_BHI=y.\n");
1299#endif
1300 } else {
1301 pr_err("CFI: Ignoring unknown option (%s).", str);
1302 }
1303
1304 str = next;
1305 }
1306
1307 return 0;
1308}
1309early_param("cfi", cfi_parse_cmdline);
1310
1311/*
1312 * kCFI FineIBT
1313 *
1314 * __cfi_\func: __cfi_\func:
1315 * movl $0x12345678,%eax // 5 endbr64 // 4
1316 * nop subl $0x12345678,%eax // 5
1317 * nop jne.d32,pn \func+3 // 7
1318 * nop
1319 * nop
1320 * nop
1321 * nop
1322 * nop
1323 * nop
1324 * nop
1325 * nop
1326 * nop
1327 * \func: \func:
1328 * endbr64 nopl -42(%rax)
1329 *
1330 *
1331 * caller: caller:
1332 * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%eax // 5
1333 * addl $-15(%r11),%r10d // 4 lea -0x10(%r11),%r11 // 4
1334 * je 1f // 2 nop5 // 5
1335 * ud2 // 2
1336 * 1: cs call __x86_indirect_thunk_r11 // 6 call *%r11; nop3; // 6
1337 *
1338 *
1339 * Notably, the FineIBT sequences are crafted such that branches are presumed
1340 * non-taken. This is based on Agner Fog's optimization manual, which states:
1341 *
1342 * "Make conditional jumps most often not taken: The efficiency and throughput
1343 * for not-taken branches is better than for taken branches on most
1344 * processors. Therefore, it is good to place the most frequent branch first"
1345 */
1346
1347/*
1348 * <fineibt_preamble_start>:
1349 * 0: f3 0f 1e fa endbr64
1350 * 4: 2d 78 56 34 12 sub $0x12345678, %eax
1351 * 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
1352 * 10: 0f 1f 40 d6 nopl -0x2a(%rax)
1353 *
1354 * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
1355 * UDB on x86_64 and raises #UD.
1356 */
1357asm( ".pushsection .rodata \n"
1358 "fineibt_preamble_start: \n"
1359 " endbr64 \n"
1360 " subl $0x12345678, %eax \n"
1361 "fineibt_preamble_bhi: \n"
1362 " cs jne.d32 fineibt_preamble_start+0x13 \n"
1363 "#fineibt_func: \n"
1364 " nopl -42(%rax) \n"
1365 "fineibt_preamble_end: \n"
1366 ".popsection\n"
1367);
1368
1369extern u8 fineibt_preamble_start[];
1370extern u8 fineibt_preamble_bhi[];
1371extern u8 fineibt_preamble_end[];
1372
1373#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
1374#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
1375#define fineibt_preamble_ud 0x13
1376#define fineibt_preamble_hash 5
1377
1378/*
1379 * <fineibt_caller_start>:
1380 * 0: b8 78 56 34 12 mov $0x12345678, %eax
1381 * 5: 4d 8d 5b f0 lea -0x10(%r11), %r11
1382 * 9: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)
1383 */
1384asm( ".pushsection .rodata \n"
1385 "fineibt_caller_start: \n"
1386 " movl $0x12345678, %eax \n"
1387 " lea -0x10(%r11), %r11 \n"
1388 ASM_NOP5
1389 "fineibt_caller_end: \n"
1390 ".popsection \n"
1391);
1392
1393extern u8 fineibt_caller_start[];
1394extern u8 fineibt_caller_end[];
1395
1396#define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
1397#define fineibt_caller_hash 1
1398
1399#define fineibt_caller_jmp (fineibt_caller_size - 2)
1400
1401/*
1402 * Since FineIBT does hash validation on the callee side it is prone to
1403 * circumvention attacks where a 'naked' ENDBR instruction exists that
1404 * is not part of the fineibt_preamble sequence.
1405 *
1406 * Notably the x86 entry points must be ENDBR and equally cannot be
1407 * fineibt_preamble.
1408 *
1409 * The fineibt_paranoid caller sequence adds additional caller side
1410 * hash validation. This stops such circumvention attacks dead, but at the cost
1411 * of adding a load.
1412 *
1413 * <fineibt_paranoid_start>:
1414 * 0: b8 78 56 34 12 mov $0x12345678, %eax
1415 * 5: 41 3b 43 f5 cmp -0x11(%r11), %eax
1416 * 9: 2e 4d 8d 5b <f0> cs lea -0x10(%r11), %r11
1417 * e: 75 fd jne d <fineibt_paranoid_start+0xd>
1418 * 10: 41 ff d3 call *%r11
1419 * 13: 90 nop
1420 *
1421 * Notably LEA does not modify flags and can be reordered with the CMP,
1422 * avoiding a dependency. Again, using a non-taken (backwards) branch
1423 * for the failure case, abusing LEA's immediate 0xf0 as LOCK prefix for the
1424 * Jcc.d8, causing #UD.
1425 */
1426asm( ".pushsection .rodata \n"
1427 "fineibt_paranoid_start: \n"
1428 " mov $0x12345678, %eax \n"
1429 " cmpl -11(%r11), %eax \n"
1430 " cs lea -0x10(%r11), %r11 \n"
1431 "#fineibt_caller_size: \n"
1432 " jne fineibt_paranoid_start+0xd \n"
1433 "fineibt_paranoid_ind: \n"
1434 " cs call *%r11 \n"
1435 "fineibt_paranoid_end: \n"
1436 ".popsection \n"
1437);
1438
1439extern u8 fineibt_paranoid_start[];
1440extern u8 fineibt_paranoid_ind[];
1441extern u8 fineibt_paranoid_end[];
1442
1443#define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start)
1444#define fineibt_paranoid_ind (fineibt_paranoid_ind - fineibt_paranoid_start)
1445#define fineibt_paranoid_ud 0xd
1446
1447static u32 decode_preamble_hash(void *addr, int *reg)
1448{
1449 u8 *p = addr;
1450
1451 /* b8+reg 78 56 34 12 movl $0x12345678,\reg */
1452 if (p[0] >= 0xb8 && p[0] < 0xc0) {
1453 if (reg)
1454 *reg = p[0] - 0xb8;
1455 return *(u32 *)(addr + 1);
1456 }
1457
1458 return 0; /* invalid hash value */
1459}
1460
1461static u32 decode_caller_hash(void *addr)
1462{
1463 u8 *p = addr;
1464
1465 /* 41 ba 88 a9 cb ed mov $(-0x12345678),%r10d */
1466 if (p[0] == 0x41 && p[1] == 0xba)
1467 return -*(u32 *)(addr + 2);
1468
1469 /* e8 0c 88 a9 cb ed jmp.d8 +12 */
1470 if (p[0] == JMP8_INSN_OPCODE && p[1] == fineibt_caller_jmp)
1471 return -*(u32 *)(addr + 2);
1472
1473 return 0; /* invalid hash value */
1474}
1475
1476/* .retpoline_sites */
1477static int cfi_disable_callers(s32 *start, s32 *end)
1478{
1479 /*
1480 * Disable kCFI by patching in a JMP.d8, this leaves the hash immediate
1481 * in tact for later usage. Also see decode_caller_hash() and
1482 * cfi_rewrite_callers().
1483 */
1484 const u8 jmp[] = { JMP8_INSN_OPCODE, fineibt_caller_jmp };
1485 s32 *s;
1486
1487 for (s = start; s < end; s++) {
1488 void *addr = (void *)s + *s;
1489 u32 hash;
1490
1491 addr -= fineibt_caller_size;
1492 hash = decode_caller_hash(addr);
1493 if (!hash) /* nocfi callers */
1494 continue;
1495
1496 text_poke_early(addr, jmp, 2);
1497 }
1498
1499 return 0;
1500}
1501
1502static int cfi_enable_callers(s32 *start, s32 *end)
1503{
1504 /*
1505 * Re-enable kCFI, undo what cfi_disable_callers() did.
1506 */
1507 const u8 mov[] = { 0x41, 0xba };
1508 s32 *s;
1509
1510 for (s = start; s < end; s++) {
1511 void *addr = (void *)s + *s;
1512 u32 hash;
1513
1514 addr -= fineibt_caller_size;
1515 hash = decode_caller_hash(addr);
1516 if (!hash) /* nocfi callers */
1517 continue;
1518
1519 text_poke_early(addr, mov, 2);
1520 }
1521
1522 return 0;
1523}
1524
1525/* .cfi_sites */
1526static int cfi_rand_preamble(s32 *start, s32 *end)
1527{
1528 s32 *s;
1529
1530 for (s = start; s < end; s++) {
1531 void *addr = (void *)s + *s;
1532 u32 hash;
1533
1534 hash = decode_preamble_hash(addr, NULL);
1535 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
1536 addr, addr, 5, addr))
1537 return -EINVAL;
1538
1539 hash = cfi_rehash(hash);
1540 text_poke_early(addr + 1, &hash, 4);
1541 }
1542
1543 return 0;
1544}
1545
1546/*
1547 * Inline the bhi-arity 1 case:
1548 *
1549 * __cfi_foo:
1550 * 0: f3 0f 1e fa endbr64
1551 * 4: 2d 78 56 34 12 sub $0x12345678, %eax
1552 * 9: 49 0f 45 fa cmovne %rax, %rdi
1553 * d: 2e 75 03 jne,pn foo+0x3
1554 *
1555 * foo:
1556 * 10: 0f 1f 40 <d6> nopl -42(%rax)
1557 *
1558 * Notably, this scheme is incompatible with permissive CFI
1559 * because the CMOVcc is unconditional and RDI will have been
1560 * clobbered.
1561 */
1562asm( ".pushsection .rodata \n"
1563 "fineibt_bhi1_start: \n"
1564 " cmovne %rax, %rdi \n"
1565 " cs jne fineibt_bhi1_func + 0x3 \n"
1566 "fineibt_bhi1_func: \n"
1567 " nopl -42(%rax) \n"
1568 "fineibt_bhi1_end: \n"
1569 ".popsection \n"
1570);
1571
1572extern u8 fineibt_bhi1_start[];
1573extern u8 fineibt_bhi1_end[];
1574
1575#define fineibt_bhi1_size (fineibt_bhi1_end - fineibt_bhi1_start)
1576
1577static void cfi_fineibt_bhi_preamble(void *addr, int arity)
1578{
1579 u8 bytes[MAX_INSN_SIZE];
1580
1581 if (!arity)
1582 return;
1583
1584 if (!cfi_warn && arity == 1) {
1585 text_poke_early(addr + fineibt_preamble_bhi,
1586 fineibt_bhi1_start, fineibt_bhi1_size);
1587 return;
1588 }
1589
1590 /*
1591 * Replace the bytes at fineibt_preamble_bhi with a CALL instruction
1592 * that lines up exactly with the end of the preamble, such that the
1593 * return address will be foo+0.
1594 *
1595 * __cfi_foo:
1596 * 0: f3 0f 1e fa endbr64
1597 * 4: 2d 78 56 34 12 sub $0x12345678, %eax
1598 * 9: 2e 2e e8 DD DD DD DD cs cs call __bhi_args[arity]
1599 */
1600 bytes[0] = 0x2e;
1601 bytes[1] = 0x2e;
1602 __text_gen_insn(bytes + 2, CALL_INSN_OPCODE,
1603 addr + fineibt_preamble_bhi + 2,
1604 __bhi_args[arity], CALL_INSN_SIZE);
1605
1606 text_poke_early(addr + fineibt_preamble_bhi, bytes, 7);
1607}
1608
1609static int cfi_rewrite_preamble(s32 *start, s32 *end)
1610{
1611 s32 *s;
1612
1613 for (s = start; s < end; s++) {
1614 void *addr = (void *)s + *s;
1615 int arity;
1616 u32 hash;
1617
1618 /*
1619 * When the function doesn't start with ENDBR the compiler will
1620 * have determined there are no indirect calls to it and we
1621 * don't need no CFI either.
1622 */
1623 if (!is_endbr(addr + 16))
1624 continue;
1625
1626 hash = decode_preamble_hash(addr, &arity);
1627 if (WARN(!hash, "no CFI hash found at: %pS %px %*ph\n",
1628 addr, addr, 5, addr))
1629 return -EINVAL;
1630
1631 text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size);
1632 WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678);
1633 text_poke_early(addr + fineibt_preamble_hash, &hash, 4);
1634
1635 WARN_ONCE(!IS_ENABLED(CONFIG_FINEIBT_BHI) && arity,
1636 "kCFI preamble has wrong register at: %pS %*ph\n",
1637 addr, 5, addr);
1638
1639 if (cfi_bhi)
1640 cfi_fineibt_bhi_preamble(addr, arity);
1641 }
1642
1643 return 0;
1644}
1645
1646static void cfi_rewrite_endbr(s32 *start, s32 *end)
1647{
1648 s32 *s;
1649
1650 for (s = start; s < end; s++) {
1651 void *addr = (void *)s + *s;
1652
1653 if (!exact_endbr(addr + 16))
1654 continue;
1655
1656 poison_endbr(addr + 16);
1657 }
1658}
1659
1660/* .retpoline_sites */
1661static int cfi_rand_callers(s32 *start, s32 *end)
1662{
1663 s32 *s;
1664
1665 for (s = start; s < end; s++) {
1666 void *addr = (void *)s + *s;
1667 u32 hash;
1668
1669 addr -= fineibt_caller_size;
1670 hash = decode_caller_hash(addr);
1671 if (hash) {
1672 hash = -cfi_rehash(hash);
1673 text_poke_early(addr + 2, &hash, 4);
1674 }
1675 }
1676
1677 return 0;
1678}
1679
1680static int emit_paranoid_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
1681{
1682 u8 *thunk = (void *)__x86_indirect_its_thunk_array[reg] - 2;
1683
1684#ifdef CONFIG_MITIGATION_ITS
1685 u8 *tmp = its_allocate_thunk(reg);
1686 if (tmp)
1687 thunk = tmp;
1688#endif
1689
1690 return __emit_trampoline(addr, insn, bytes, thunk, thunk);
1691}
1692
1693static int cfi_rewrite_callers(s32 *start, s32 *end)
1694{
1695 s32 *s;
1696
1697 for (s = start; s < end; s++) {
1698 void *addr = (void *)s + *s;
1699 struct insn insn;
1700 u8 bytes[20];
1701 u32 hash;
1702 int ret;
1703 u8 op;
1704
1705 addr -= fineibt_caller_size;
1706 hash = decode_caller_hash(addr);
1707 if (!hash)
1708 continue;
1709
1710 if (!cfi_paranoid) {
1711 text_poke_early(addr, fineibt_caller_start, fineibt_caller_size);
1712 WARN_ON(*(u32 *)(addr + fineibt_caller_hash) != 0x12345678);
1713 text_poke_early(addr + fineibt_caller_hash, &hash, 4);
1714 /* rely on apply_retpolines() */
1715 continue;
1716 }
1717
1718 /* cfi_paranoid */
1719 ret = insn_decode_kernel(&insn, addr + fineibt_caller_size);
1720 if (WARN_ON_ONCE(ret < 0))
1721 continue;
1722
1723 op = insn.opcode.bytes[0];
1724 if (op != CALL_INSN_OPCODE && op != JMP32_INSN_OPCODE) {
1725 WARN_ON_ONCE(1);
1726 continue;
1727 }
1728
1729 memcpy(bytes, fineibt_paranoid_start, fineibt_paranoid_size);
1730 memcpy(bytes + fineibt_caller_hash, &hash, 4);
1731
1732 if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + fineibt_paranoid_ind, 11)) {
1733 emit_paranoid_trampoline(addr + fineibt_caller_size,
1734 &insn, 11, bytes + fineibt_caller_size);
1735 } else {
1736 int len = fineibt_paranoid_size - fineibt_paranoid_ind;
1737 ret = emit_indirect(op, 11, bytes + fineibt_paranoid_ind, len);
1738 if (WARN_ON_ONCE(ret != len))
1739 continue;
1740 }
1741
1742 text_poke_early(addr, bytes, fineibt_paranoid_size);
1743 }
1744
1745 return 0;
1746}
1747
1748#define pr_cfi_debug(X...) if (cfi_debug) pr_info(X)
1749
1750#define FINEIBT_WARN(_f, _v) \
1751 WARN_ONCE((_f) != (_v), "FineIBT: " #_f " %ld != %d\n", _f, _v)
1752
1753static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
1754 s32 *start_cfi, s32 *end_cfi, bool builtin)
1755{
1756 int ret;
1757
1758 if (FINEIBT_WARN(fineibt_preamble_size, 20) ||
1759 FINEIBT_WARN(fineibt_preamble_bhi + fineibt_bhi1_size, 20) ||
1760 FINEIBT_WARN(fineibt_caller_size, 14) ||
1761 FINEIBT_WARN(fineibt_paranoid_size, 20))
1762 return;
1763
1764 if (cfi_mode == CFI_AUTO) {
1765 cfi_mode = CFI_KCFI;
1766 if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) {
1767 /*
1768 * FRED has much saner context on exception entry and
1769 * is less easy to take advantage of.
1770 */
1771 if (!cpu_feature_enabled(X86_FEATURE_FRED))
1772 cfi_paranoid = true;
1773 cfi_mode = CFI_FINEIBT;
1774 }
1775 }
1776
1777 /*
1778 * Rewrite the callers to not use the __cfi_ stubs, such that we might
1779 * rewrite them. This disables all CFI. If this succeeds but any of the
1780 * later stages fails, we're without CFI.
1781 */
1782 pr_cfi_debug("CFI: disabling all indirect call checking\n");
1783 ret = cfi_disable_callers(start_retpoline, end_retpoline);
1784 if (ret)
1785 goto err;
1786
1787 if (cfi_rand) {
1788 if (builtin) {
1789 cfi_seed = get_random_u32();
1790 cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
1791 cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
1792 }
1793 pr_cfi_debug("CFI: cfi_seed: 0x%08x\n", cfi_seed);
1794
1795 pr_cfi_debug("CFI: rehashing all preambles\n");
1796 ret = cfi_rand_preamble(start_cfi, end_cfi);
1797 if (ret)
1798 goto err;
1799
1800 pr_cfi_debug("CFI: rehashing all indirect calls\n");
1801 ret = cfi_rand_callers(start_retpoline, end_retpoline);
1802 if (ret)
1803 goto err;
1804 } else {
1805 pr_cfi_debug("CFI: rehashing disabled\n");
1806 }
1807
1808 switch (cfi_mode) {
1809 case CFI_OFF:
1810 if (builtin)
1811 pr_info("CFI: disabled\n");
1812 return;
1813
1814 case CFI_KCFI:
1815 pr_cfi_debug("CFI: re-enabling all indirect call checking\n");
1816 ret = cfi_enable_callers(start_retpoline, end_retpoline);
1817 if (ret)
1818 goto err;
1819
1820 if (builtin)
1821 pr_info("CFI: Using %sretpoline kCFI\n",
1822 cfi_rand ? "rehashed " : "");
1823 return;
1824
1825 case CFI_FINEIBT:
1826 pr_cfi_debug("CFI: adding FineIBT to all preambles\n");
1827 /* place the FineIBT preamble at func()-16 */
1828 ret = cfi_rewrite_preamble(start_cfi, end_cfi);
1829 if (ret)
1830 goto err;
1831
1832 /* rewrite the callers to target func()-16 */
1833 pr_cfi_debug("CFI: rewriting indirect call sites to use FineIBT\n");
1834 ret = cfi_rewrite_callers(start_retpoline, end_retpoline);
1835 if (ret)
1836 goto err;
1837
1838 /* now that nobody targets func()+0, remove ENDBR there */
1839 pr_cfi_debug("CFI: removing old endbr insns\n");
1840 cfi_rewrite_endbr(start_cfi, end_cfi);
1841
1842 if (builtin) {
1843 pr_info("Using %sFineIBT%s CFI\n",
1844 cfi_paranoid ? "paranoid " : "",
1845 cfi_bhi ? "+BHI" : "");
1846 }
1847 return;
1848
1849 default:
1850 break;
1851 }
1852
1853err:
1854 pr_err("Something went horribly wrong trying to rewrite the CFI implementation.\n");
1855}
1856
1857static inline void poison_hash(void *addr)
1858{
1859 *(u32 *)addr = 0;
1860}
1861
1862static void poison_cfi(void *addr)
1863{
1864 /*
1865 * Compilers manage to be inconsistent with ENDBR vs __cfi prefixes,
1866 * some (static) functions for which they can determine the address
1867 * is never taken do not get a __cfi prefix, but *DO* get an ENDBR.
1868 *
1869 * As such, these functions will get sealed, but we need to be careful
1870 * to not unconditionally scribble the previous function.
1871 */
1872 switch (cfi_mode) {
1873 case CFI_FINEIBT:
1874 /*
1875 * FineIBT prefix should start with an ENDBR.
1876 */
1877 if (!is_endbr(addr))
1878 break;
1879
1880 /*
1881 * __cfi_\func:
1882 * nopl -42(%rax)
1883 * sub $0, %eax
1884 * jne \func+3
1885 * \func:
1886 * nopl -42(%rax)
1887 */
1888 poison_endbr(addr);
1889 poison_hash(addr + fineibt_preamble_hash);
1890 break;
1891
1892 case CFI_KCFI:
1893 /*
1894 * kCFI prefix should start with a valid hash.
1895 */
1896 if (!decode_preamble_hash(addr, NULL))
1897 break;
1898
1899 /*
1900 * __cfi_\func:
1901 * movl $0, %eax
1902 * .skip 11, 0x90
1903 */
1904 poison_hash(addr + 1);
1905 break;
1906
1907 default:
1908 break;
1909 }
1910}
1911
1912#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
1913
1914/*
1915 * When regs->ip points to a 0xD6 byte in the FineIBT preamble,
1916 * return true and fill out target and type.
1917 *
1918 * We check the preamble by checking for the ENDBR instruction relative to the
1919 * UDB instruction.
1920 */
1921static bool decode_fineibt_preamble(struct pt_regs *regs, unsigned long *target, u32 *type)
1922{
1923 unsigned long addr = regs->ip - fineibt_preamble_ud;
1924 u32 hash;
1925
1926 if (!exact_endbr((void *)addr))
1927 return false;
1928
1929 *target = addr + fineibt_prefix_size;
1930
1931 __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
1932 *type = (u32)regs->ax + hash;
1933
1934 /*
1935 * Since regs->ip points to the middle of an instruction; it cannot
1936 * continue with the normal fixup.
1937 */
1938 regs->ip = *target;
1939
1940 return true;
1941
1942Efault:
1943 return false;
1944}
1945
1946/*
1947 * regs->ip points to one of the UD2 in __bhi_args[].
1948 */
1949static bool decode_fineibt_bhi(struct pt_regs *regs, unsigned long *target, u32 *type)
1950{
1951 unsigned long addr;
1952 u32 hash;
1953
1954 if (!cfi_bhi)
1955 return false;
1956
1957 if (regs->ip < (unsigned long)__bhi_args ||
1958 regs->ip >= (unsigned long)__bhi_args_end)
1959 return false;
1960
1961 /*
1962 * Fetch the return address from the stack, this points to the
1963 * FineIBT preamble. Since the CALL instruction is in the 5 last
1964 * bytes of the preamble, the return address is in fact the target
1965 * address.
1966 */
1967 __get_kernel_nofault(&addr, regs->sp, unsigned long, Efault);
1968 *target = addr;
1969
1970 addr -= fineibt_prefix_size;
1971 if (!exact_endbr((void *)addr))
1972 return false;
1973
1974 __get_kernel_nofault(&hash, addr + fineibt_preamble_hash, u32, Efault);
1975 *type = (u32)regs->ax + hash;
1976
1977 /*
1978 * The UD2 sites are constructed with a RET immediately following,
1979 * as such the non-fatal case can use the regular fixup.
1980 */
1981 return true;
1982
1983Efault:
1984 return false;
1985}
1986
1987static bool is_paranoid_thunk(unsigned long addr)
1988{
1989 u32 thunk;
1990
1991 __get_kernel_nofault(&thunk, (u32 *)addr, u32, Efault);
1992 return (thunk & 0x00FFFFFF) == 0xfd75d6;
1993
1994Efault:
1995 return false;
1996}
1997
1998/*
1999 * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[]
2000 * sequence, or to UDB + Jcc.d8 for cfi_paranoid + ITS thunk.
2001 */
2002static bool decode_fineibt_paranoid(struct pt_regs *regs, unsigned long *target, u32 *type)
2003{
2004 unsigned long addr = regs->ip - fineibt_paranoid_ud;
2005
2006 if (!cfi_paranoid)
2007 return false;
2008
2009 if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) {
2010 *target = regs->r11 + fineibt_prefix_size;
2011 *type = regs->ax;
2012
2013 /*
2014 * Since the trapping instruction is the exact, but LOCK prefixed,
2015 * Jcc.d8 that got us here, the normal fixup will work.
2016 */
2017 return true;
2018 }
2019
2020 /*
2021 * The cfi_paranoid + ITS thunk combination results in:
2022 *
2023 * 0: b8 78 56 34 12 mov $0x12345678, %eax
2024 * 5: 41 3b 43 f7 cmp -11(%r11), %eax
2025 * a: 2e 3d 8d 5b f0 cs lea -0x10(%r11), %r11
2026 * e: 2e e8 XX XX XX XX cs call __x86_indirect_paranoid_thunk_r11
2027 *
2028 * Where the paranoid_thunk looks like:
2029 *
2030 * 1d: <d6> udb
2031 * __x86_indirect_paranoid_thunk_r11:
2032 * 1e: 75 fd jne 1d
2033 * __x86_indirect_its_thunk_r11:
2034 * 20: 41 ff eb jmp *%r11
2035 * 23: cc int3
2036 *
2037 */
2038 if (is_paranoid_thunk(regs->ip)) {
2039 *target = regs->r11 + fineibt_prefix_size;
2040 *type = regs->ax;
2041
2042 regs->ip = *target;
2043 return true;
2044 }
2045
2046 return false;
2047}
2048
2049bool decode_fineibt_insn(struct pt_regs *regs, unsigned long *target, u32 *type)
2050{
2051 if (decode_fineibt_paranoid(regs, target, type))
2052 return true;
2053
2054 if (decode_fineibt_bhi(regs, target, type))
2055 return true;
2056
2057 return decode_fineibt_preamble(regs, target, type);
2058}
2059
2060#else /* !CONFIG_FINEIBT: */
2061
2062static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
2063 s32 *start_cfi, s32 *end_cfi, bool builtin)
2064{
2065 if (IS_ENABLED(CONFIG_CFI) && builtin)
2066 pr_info("CFI: Using standard kCFI\n");
2067}
2068
2069#ifdef CONFIG_X86_KERNEL_IBT
2070static void poison_cfi(void *addr) { }
2071#endif
2072
2073#endif /* !CONFIG_FINEIBT */
2074
2075void apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
2076 s32 *start_cfi, s32 *end_cfi)
2077{
2078 return __apply_fineibt(start_retpoline, end_retpoline,
2079 start_cfi, end_cfi,
2080 /* .builtin = */ false);
2081}
2082
2083#ifdef CONFIG_SMP
2084static void alternatives_smp_lock(const s32 *start, const s32 *end,
2085 u8 *text, u8 *text_end)
2086{
2087 const s32 *poff;
2088
2089 for (poff = start; poff < end; poff++) {
2090 u8 *ptr = (u8 *)poff + *poff;
2091
2092 if (!*poff || ptr < text || ptr >= text_end)
2093 continue;
2094 /* turn DS segment override prefix into lock prefix */
2095 if (*ptr == 0x3e)
2096 text_poke(addr: ptr, opcode: ((unsigned char []){0xf0}), len: 1);
2097 }
2098}
2099
2100static void alternatives_smp_unlock(const s32 *start, const s32 *end,
2101 u8 *text, u8 *text_end)
2102{
2103 const s32 *poff;
2104
2105 for (poff = start; poff < end; poff++) {
2106 u8 *ptr = (u8 *)poff + *poff;
2107
2108 if (!*poff || ptr < text || ptr >= text_end)
2109 continue;
2110 /* turn lock prefix into DS segment override prefix */
2111 if (*ptr == 0xf0)
2112 text_poke(addr: ptr, opcode: ((unsigned char []){0x3E}), len: 1);
2113 }
2114}
2115
2116struct smp_alt_module {
2117 /* what is this ??? */
2118 struct module *mod;
2119 char *name;
2120
2121 /* ptrs to lock prefixes */
2122 const s32 *locks;
2123 const s32 *locks_end;
2124
2125 /* .text segment, needed to avoid patching init code ;) */
2126 u8 *text;
2127 u8 *text_end;
2128
2129 struct list_head next;
2130};
2131static LIST_HEAD(smp_alt_modules);
2132static bool uniproc_patched = false; /* protected by text_mutex */
2133
2134void __init_or_module alternatives_smp_module_add(struct module *mod,
2135 char *name,
2136 void *locks, void *locks_end,
2137 void *text, void *text_end)
2138{
2139 struct smp_alt_module *smp;
2140
2141 mutex_lock(lock: &text_mutex);
2142 if (!uniproc_patched)
2143 goto unlock;
2144
2145 if (num_possible_cpus() == 1)
2146 /* Don't bother remembering, we'll never have to undo it. */
2147 goto smp_unlock;
2148
2149 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
2150 if (NULL == smp)
2151 /* we'll run the (safe but slow) SMP code then ... */
2152 goto unlock;
2153
2154 smp->mod = mod;
2155 smp->name = name;
2156 smp->locks = locks;
2157 smp->locks_end = locks_end;
2158 smp->text = text;
2159 smp->text_end = text_end;
2160 DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n",
2161 smp->locks, smp->locks_end,
2162 smp->text, smp->text_end, smp->name);
2163
2164 list_add_tail(new: &smp->next, head: &smp_alt_modules);
2165smp_unlock:
2166 alternatives_smp_unlock(start: locks, end: locks_end, text, text_end);
2167unlock:
2168 mutex_unlock(lock: &text_mutex);
2169}
2170
2171void __init_or_module alternatives_smp_module_del(struct module *mod)
2172{
2173 struct smp_alt_module *item;
2174
2175 mutex_lock(lock: &text_mutex);
2176 list_for_each_entry(item, &smp_alt_modules, next) {
2177 if (mod != item->mod)
2178 continue;
2179 list_del(entry: &item->next);
2180 kfree(objp: item);
2181 break;
2182 }
2183 mutex_unlock(lock: &text_mutex);
2184}
2185
2186void alternatives_enable_smp(void)
2187{
2188 struct smp_alt_module *mod;
2189
2190 /* Why bother if there are no other CPUs? */
2191 BUG_ON(num_possible_cpus() == 1);
2192
2193 mutex_lock(lock: &text_mutex);
2194
2195 if (uniproc_patched) {
2196 pr_info("switching to SMP code\n");
2197 BUG_ON(num_online_cpus() != 1);
2198 clear_cpu_cap(c: &boot_cpu_data, X86_FEATURE_UP);
2199 clear_cpu_cap(c: &cpu_data(0), X86_FEATURE_UP);
2200 list_for_each_entry(mod, &smp_alt_modules, next)
2201 alternatives_smp_lock(start: mod->locks, end: mod->locks_end,
2202 text: mod->text, text_end: mod->text_end);
2203 uniproc_patched = false;
2204 }
2205 mutex_unlock(lock: &text_mutex);
2206}
2207
2208/*
2209 * Return 1 if the address range is reserved for SMP-alternatives.
2210 * Must hold text_mutex.
2211 */
2212int alternatives_text_reserved(void *start, void *end)
2213{
2214 struct smp_alt_module *mod;
2215 const s32 *poff;
2216 u8 *text_start = start;
2217 u8 *text_end = end;
2218
2219 lockdep_assert_held(&text_mutex);
2220
2221 list_for_each_entry(mod, &smp_alt_modules, next) {
2222 if (mod->text > text_end || mod->text_end < text_start)
2223 continue;
2224 for (poff = mod->locks; poff < mod->locks_end; poff++) {
2225 const u8 *ptr = (const u8 *)poff + *poff;
2226
2227 if (text_start <= ptr && text_end > ptr)
2228 return 1;
2229 }
2230 }
2231
2232 return 0;
2233}
2234#endif /* CONFIG_SMP */
2235
2236/*
2237 * Self-test for the INT3 based CALL emulation code.
2238 *
2239 * This exercises int3_emulate_call() to make sure INT3 pt_regs are set up
2240 * properly and that there is a stack gap between the INT3 frame and the
2241 * previous context. Without this gap doing a virtual PUSH on the interrupted
2242 * stack would corrupt the INT3 IRET frame.
2243 *
2244 * See entry_{32,64}.S for more details.
2245 */
2246
2247/*
2248 * We define the int3_magic() function in assembly to control the calling
2249 * convention such that we can 'call' it from assembly.
2250 */
2251
2252extern void int3_magic(unsigned int *ptr); /* defined in asm */
2253
2254asm (
2255" .pushsection .init.text, \"ax\", @progbits\n"
2256" .type int3_magic, @function\n"
2257"int3_magic:\n"
2258 ANNOTATE_NOENDBR
2259" movl $1, (%" _ASM_ARG1 ")\n"
2260 ASM_RET
2261" .size int3_magic, .-int3_magic\n"
2262" .popsection\n"
2263);
2264
2265extern void int3_selftest_ip(void); /* defined in asm below */
2266
2267static int __init
2268int3_exception_notify(struct notifier_block *self, unsigned long val, void *data)
2269{
2270 unsigned long selftest = (unsigned long)&int3_selftest_ip;
2271 struct die_args *args = data;
2272 struct pt_regs *regs = args->regs;
2273
2274 OPTIMIZER_HIDE_VAR(selftest);
2275
2276 if (!regs || user_mode(regs))
2277 return NOTIFY_DONE;
2278
2279 if (val != DIE_INT3)
2280 return NOTIFY_DONE;
2281
2282 if (regs->ip - INT3_INSN_SIZE != selftest)
2283 return NOTIFY_DONE;
2284
2285 int3_emulate_call(regs, func: (unsigned long)&int3_magic);
2286 return NOTIFY_STOP;
2287}
2288
2289/* Must be noinline to ensure uniqueness of int3_selftest_ip. */
2290static noinline void __init int3_selftest(void)
2291{
2292 static __initdata struct notifier_block int3_exception_nb = {
2293 .notifier_call = int3_exception_notify,
2294 .priority = INT_MAX-1, /* last */
2295 };
2296 unsigned int val = 0;
2297
2298 BUG_ON(register_die_notifier(&int3_exception_nb));
2299
2300 /*
2301 * Basically: int3_magic(&val); but really complicated :-)
2302 *
2303 * INT3 padded with NOP to CALL_INSN_SIZE. The int3_exception_nb
2304 * notifier above will emulate CALL for us.
2305 */
2306 asm volatile ("int3_selftest_ip:\n\t"
2307 ANNOTATE_NOENDBR
2308 " int3; nop; nop; nop; nop\n\t"
2309 : ASM_CALL_CONSTRAINT
2310 : __ASM_SEL_RAW(a, D) (&val)
2311 : "memory");
2312
2313 BUG_ON(val != 1);
2314
2315 unregister_die_notifier(nb: &int3_exception_nb);
2316}
2317
2318static __initdata int __alt_reloc_selftest_addr;
2319
2320extern void __init __alt_reloc_selftest(void *arg);
2321__visible noinline void __init __alt_reloc_selftest(void *arg)
2322{
2323 WARN_ON(arg != &__alt_reloc_selftest_addr);
2324}
2325
2326static noinline void __init alt_reloc_selftest(void)
2327{
2328 /*
2329 * Tests text_poke_apply_relocation().
2330 *
2331 * This has a relative immediate (CALL) in a place other than the first
2332 * instruction and additionally on x86_64 we get a RIP-relative LEA:
2333 *
2334 * lea 0x0(%rip),%rdi # 5d0: R_X86_64_PC32 .init.data+0x5566c
2335 * call +0 # 5d5: R_X86_64_PLT32 __alt_reloc_selftest-0x4
2336 *
2337 * Getting this wrong will either crash and burn or tickle the WARN
2338 * above.
2339 */
2340 asm_inline volatile (
2341 ALTERNATIVE("", "lea %[mem], %%" _ASM_ARG1 "; call __alt_reloc_selftest;", X86_FEATURE_ALWAYS)
2342 : ASM_CALL_CONSTRAINT
2343 : [mem] "m" (__alt_reloc_selftest_addr)
2344 : _ASM_ARG1
2345 );
2346}
2347
2348void __init alternative_instructions(void)
2349{
2350 u64 ibt;
2351
2352 int3_selftest();
2353
2354 /*
2355 * The patching is not fully atomic, so try to avoid local
2356 * interruptions that might execute the to be patched code.
2357 * Other CPUs are not running.
2358 */
2359 stop_nmi();
2360
2361 /*
2362 * Don't stop machine check exceptions while patching.
2363 * MCEs only happen when something got corrupted and in this
2364 * case we must do something about the corruption.
2365 * Ignoring it is worse than an unlikely patching race.
2366 * Also machine checks tend to be broadcast and if one CPU
2367 * goes into machine check the others follow quickly, so we don't
2368 * expect a machine check to cause undue problems during to code
2369 * patching.
2370 */
2371
2372 /*
2373 * Make sure to set (artificial) features depending on used paravirt
2374 * functions which can later influence alternative patching.
2375 */
2376 paravirt_set_cap();
2377
2378 /* Keep CET-IBT disabled until caller/callee are patched */
2379 ibt = ibt_save(/*disable*/ true);
2380
2381 __apply_fineibt(start_retpoline: __retpoline_sites, end_retpoline: __retpoline_sites_end,
2382 start_cfi: __cfi_sites, end_cfi: __cfi_sites_end, builtin: true);
2383 cfi_debug = false;
2384
2385 /*
2386 * Rewrite the retpolines, must be done before alternatives since
2387 * those can rewrite the retpoline thunks.
2388 */
2389 apply_retpolines(start: __retpoline_sites, end: __retpoline_sites_end);
2390 apply_returns(start: __return_sites, end: __return_sites_end);
2391
2392 its_fini_core();
2393
2394 /*
2395 * Adjust all CALL instructions to point to func()-10, including
2396 * those in .altinstr_replacement.
2397 */
2398 callthunks_patch_builtin_calls();
2399
2400 apply_alternatives(start: __alt_instructions, end: __alt_instructions_end);
2401
2402 /*
2403 * Seal all functions that do not have their address taken.
2404 */
2405 apply_seal_endbr(start: __ibt_endbr_seal, end: __ibt_endbr_seal_end);
2406
2407 ibt_restore(save: ibt);
2408
2409#ifdef CONFIG_SMP
2410 /* Patch to UP if other cpus not imminent. */
2411 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
2412 uniproc_patched = true;
2413 alternatives_smp_module_add(NULL, name: "core kernel",
2414 locks: __smp_locks, locks_end: __smp_locks_end,
2415 text: _text, text_end: _etext);
2416 }
2417
2418 if (!uniproc_patched || num_possible_cpus() == 1) {
2419 free_init_pages(what: "SMP alternatives",
2420 begin: (unsigned long)__smp_locks,
2421 end: (unsigned long)__smp_locks_end);
2422 }
2423#endif
2424
2425 restart_nmi();
2426 alternatives_patched = 1;
2427
2428 alt_reloc_selftest();
2429}
2430
2431/**
2432 * text_poke_early - Update instructions on a live kernel at boot time
2433 * @addr: address to modify
2434 * @opcode: source of the copy
2435 * @len: length to copy
2436 *
2437 * When you use this code to patch more than one byte of an instruction
2438 * you need to make sure that other CPUs cannot execute this code in parallel.
2439 * Also no thread must be currently preempted in the middle of these
2440 * instructions. And on the local CPU you need to be protected against NMI or
2441 * MCE handlers seeing an inconsistent instruction while you patch.
2442 */
2443void __init_or_module text_poke_early(void *addr, const void *opcode,
2444 size_t len)
2445{
2446 unsigned long flags;
2447
2448 if (boot_cpu_has(X86_FEATURE_NX) &&
2449 is_module_text_address(addr: (unsigned long)addr)) {
2450 /*
2451 * Modules text is marked initially as non-executable, so the
2452 * code cannot be running and speculative code-fetches are
2453 * prevented. Just change the code.
2454 */
2455 memcpy(to: addr, from: opcode, len);
2456 } else {
2457 local_irq_save(flags);
2458 memcpy(to: addr, from: opcode, len);
2459 sync_core();
2460 local_irq_restore(flags);
2461
2462 /*
2463 * Could also do a CLFLUSH here to speed up CPU recovery; but
2464 * that causes hangs on some VIA CPUs.
2465 */
2466 }
2467}
2468
2469__ro_after_init struct mm_struct *text_poke_mm;
2470__ro_after_init unsigned long text_poke_mm_addr;
2471
2472static void text_poke_memcpy(void *dst, const void *src, size_t len)
2473{
2474 memcpy(to: dst, from: src, len);
2475}
2476
2477static void text_poke_memset(void *dst, const void *src, size_t len)
2478{
2479 int c = *(const int *)src;
2480
2481 memset(s: dst, c, n: len);
2482}
2483
2484typedef void text_poke_f(void *dst, const void *src, size_t len);
2485
2486static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len)
2487{
2488 bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
2489 struct page *pages[2] = {NULL};
2490 struct mm_struct *prev_mm;
2491 unsigned long flags;
2492 pte_t pte, *ptep;
2493 spinlock_t *ptl;
2494 pgprot_t pgprot;
2495
2496 /*
2497 * While boot memory allocator is running we cannot use struct pages as
2498 * they are not yet initialized. There is no way to recover.
2499 */
2500 BUG_ON(!after_bootmem);
2501
2502 if (!core_kernel_text(addr: (unsigned long)addr)) {
2503 pages[0] = vmalloc_to_page(addr);
2504 if (cross_page_boundary)
2505 pages[1] = vmalloc_to_page(addr: addr + PAGE_SIZE);
2506 } else {
2507 pages[0] = virt_to_page(addr);
2508 WARN_ON(!PageReserved(pages[0]));
2509 if (cross_page_boundary)
2510 pages[1] = virt_to_page(addr + PAGE_SIZE);
2511 }
2512 /*
2513 * If something went wrong, crash and burn since recovery paths are not
2514 * implemented.
2515 */
2516 BUG_ON(!pages[0] || (cross_page_boundary && !pages[1]));
2517
2518 /*
2519 * Map the page without the global bit, as TLB flushing is done with
2520 * flush_tlb_mm_range(), which is intended for non-global PTEs.
2521 */
2522 pgprot = __pgprot(pgprot_val(PAGE_KERNEL) & ~_PAGE_GLOBAL);
2523
2524 /*
2525 * The lock is not really needed, but this allows to avoid open-coding.
2526 */
2527 ptep = get_locked_pte(mm: text_poke_mm, addr: text_poke_mm_addr, ptl: &ptl);
2528
2529 /*
2530 * This must not fail; preallocated in poking_init().
2531 */
2532 VM_BUG_ON(!ptep);
2533
2534 local_irq_save(flags);
2535
2536 pte = mk_pte(page: pages[0], pgprot);
2537 set_pte_at(text_poke_mm, text_poke_mm_addr, ptep, pte);
2538
2539 if (cross_page_boundary) {
2540 pte = mk_pte(page: pages[1], pgprot);
2541 set_pte_at(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1, pte);
2542 }
2543
2544 /*
2545 * Loading the temporary mm behaves as a compiler barrier, which
2546 * guarantees that the PTE will be set at the time memcpy() is done.
2547 */
2548 prev_mm = use_temporary_mm(temp_mm: text_poke_mm);
2549
2550 kasan_disable_current();
2551 func((u8 *)text_poke_mm_addr + offset_in_page(addr), src, len);
2552 kasan_enable_current();
2553
2554 /*
2555 * Ensure that the PTE is only cleared after the instructions of memcpy
2556 * were issued by using a compiler barrier.
2557 */
2558 barrier();
2559
2560 pte_clear(text_poke_mm, text_poke_mm_addr, ptep);
2561 if (cross_page_boundary)
2562 pte_clear(text_poke_mm, text_poke_mm_addr + PAGE_SIZE, ptep + 1);
2563
2564 /*
2565 * Loading the previous page-table hierarchy requires a serializing
2566 * instruction that already allows the core to see the updated version.
2567 * Xen-PV is assumed to serialize execution in a similar manner.
2568 */
2569 unuse_temporary_mm(prev_mm);
2570
2571 /*
2572 * Flushing the TLB might involve IPIs, which would require enabled
2573 * IRQs, but not if the mm is not used, as it is in this point.
2574 */
2575 flush_tlb_mm_range(mm: text_poke_mm, start: text_poke_mm_addr, end: text_poke_mm_addr +
2576 (cross_page_boundary ? 2 : 1) * PAGE_SIZE,
2577 PAGE_SHIFT, freed_tables: false);
2578
2579 if (func == text_poke_memcpy) {
2580 /*
2581 * If the text does not match what we just wrote then something is
2582 * fundamentally screwy; there's nothing we can really do about that.
2583 */
2584 BUG_ON(memcmp(addr, src, len));
2585 }
2586
2587 local_irq_restore(flags);
2588 pte_unmap_unlock(ptep, ptl);
2589 return addr;
2590}
2591
2592/**
2593 * text_poke - Update instructions on a live kernel
2594 * @addr: address to modify
2595 * @opcode: source of the copy
2596 * @len: length to copy
2597 *
2598 * Only atomic text poke/set should be allowed when not doing early patching.
2599 * It means the size must be writable atomically and the address must be aligned
2600 * in a way that permits an atomic write. It also makes sure we fit on a single
2601 * page.
2602 *
2603 * Note that the caller must ensure that if the modified code is part of a
2604 * module, the module would not be removed during poking. This can be achieved
2605 * by registering a module notifier, and ordering module removal and patching
2606 * through a mutex.
2607 */
2608void *text_poke(void *addr, const void *opcode, size_t len)
2609{
2610 lockdep_assert_held(&text_mutex);
2611
2612 return __text_poke(func: text_poke_memcpy, addr, src: opcode, len);
2613}
2614
2615/**
2616 * text_poke_kgdb - Update instructions on a live kernel by kgdb
2617 * @addr: address to modify
2618 * @opcode: source of the copy
2619 * @len: length to copy
2620 *
2621 * Only atomic text poke/set should be allowed when not doing early patching.
2622 * It means the size must be writable atomically and the address must be aligned
2623 * in a way that permits an atomic write. It also makes sure we fit on a single
2624 * page.
2625 *
2626 * Context: should only be used by kgdb, which ensures no other core is running,
2627 * despite the fact it does not hold the text_mutex.
2628 */
2629void *text_poke_kgdb(void *addr, const void *opcode, size_t len)
2630{
2631 return __text_poke(func: text_poke_memcpy, addr, src: opcode, len);
2632}
2633
2634void *text_poke_copy_locked(void *addr, const void *opcode, size_t len,
2635 bool core_ok)
2636{
2637 unsigned long start = (unsigned long)addr;
2638 size_t patched = 0;
2639
2640 if (WARN_ON_ONCE(!core_ok && core_kernel_text(start)))
2641 return NULL;
2642
2643 while (patched < len) {
2644 unsigned long ptr = start + patched;
2645 size_t s;
2646
2647 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);
2648
2649 __text_poke(func: text_poke_memcpy, addr: (void *)ptr, src: opcode + patched, len: s);
2650 patched += s;
2651 }
2652 return addr;
2653}
2654
2655/**
2656 * text_poke_copy - Copy instructions into (an unused part of) RX memory
2657 * @addr: address to modify
2658 * @opcode: source of the copy
2659 * @len: length to copy, could be more than 2x PAGE_SIZE
2660 *
2661 * Not safe against concurrent execution; useful for JITs to dump
2662 * new code blocks into unused regions of RX memory. Can be used in
2663 * conjunction with synchronize_rcu_tasks() to wait for existing
2664 * execution to quiesce after having made sure no existing functions
2665 * pointers are live.
2666 */
2667void *text_poke_copy(void *addr, const void *opcode, size_t len)
2668{
2669 mutex_lock(lock: &text_mutex);
2670 addr = text_poke_copy_locked(addr, opcode, len, core_ok: false);
2671 mutex_unlock(lock: &text_mutex);
2672 return addr;
2673}
2674
2675/**
2676 * text_poke_set - memset into (an unused part of) RX memory
2677 * @addr: address to modify
2678 * @c: the byte to fill the area with
2679 * @len: length to copy, could be more than 2x PAGE_SIZE
2680 *
2681 * This is useful to overwrite unused regions of RX memory with illegal
2682 * instructions.
2683 */
2684void *text_poke_set(void *addr, int c, size_t len)
2685{
2686 unsigned long start = (unsigned long)addr;
2687 size_t patched = 0;
2688
2689 if (WARN_ON_ONCE(core_kernel_text(start)))
2690 return NULL;
2691
2692 mutex_lock(lock: &text_mutex);
2693 while (patched < len) {
2694 unsigned long ptr = start + patched;
2695 size_t s;
2696
2697 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched);
2698
2699 __text_poke(func: text_poke_memset, addr: (void *)ptr, src: (void *)&c, len: s);
2700 patched += s;
2701 }
2702 mutex_unlock(lock: &text_mutex);
2703 return addr;
2704}
2705
2706static void do_sync_core(void *info)
2707{
2708 sync_core();
2709}
2710
2711void smp_text_poke_sync_each_cpu(void)
2712{
2713 on_each_cpu(func: do_sync_core, NULL, wait: 1);
2714}
2715
2716/*
2717 * NOTE: crazy scheme to allow patching Jcc.d32 but not increase the size of
2718 * this thing. When len == 6 everything is prefixed with 0x0f and we map
2719 * opcode to Jcc.d8, using len to distinguish.
2720 */
2721struct smp_text_poke_loc {
2722 /* addr := _stext + rel_addr */
2723 s32 rel_addr;
2724 s32 disp;
2725 u8 len;
2726 u8 opcode;
2727 const u8 text[TEXT_POKE_MAX_OPCODE_SIZE];
2728 /* see smp_text_poke_batch_finish() */
2729 u8 old;
2730};
2731
2732#define TEXT_POKE_ARRAY_MAX (PAGE_SIZE / sizeof(struct smp_text_poke_loc))
2733
2734static struct smp_text_poke_array {
2735 struct smp_text_poke_loc vec[TEXT_POKE_ARRAY_MAX];
2736 int nr_entries;
2737} text_poke_array;
2738
2739static DEFINE_PER_CPU(atomic_t, text_poke_array_refs);
2740
2741/*
2742 * These four __always_inline annotations imply noinstr, necessary
2743 * due to smp_text_poke_int3_handler() being noinstr:
2744 */
2745
2746static __always_inline bool try_get_text_poke_array(void)
2747{
2748 atomic_t *refs = this_cpu_ptr(&text_poke_array_refs);
2749
2750 if (!raw_atomic_inc_not_zero(v: refs))
2751 return false;
2752
2753 return true;
2754}
2755
2756static __always_inline void put_text_poke_array(void)
2757{
2758 atomic_t *refs = this_cpu_ptr(&text_poke_array_refs);
2759
2760 smp_mb__before_atomic();
2761 raw_atomic_dec(v: refs);
2762}
2763
2764static __always_inline void *text_poke_addr(const struct smp_text_poke_loc *tpl)
2765{
2766 return _stext + tpl->rel_addr;
2767}
2768
2769static __always_inline int patch_cmp(const void *tpl_a, const void *tpl_b)
2770{
2771 if (tpl_a < text_poke_addr(tpl: tpl_b))
2772 return -1;
2773 if (tpl_a > text_poke_addr(tpl: tpl_b))
2774 return 1;
2775 return 0;
2776}
2777
2778noinstr int smp_text_poke_int3_handler(struct pt_regs *regs)
2779{
2780 struct smp_text_poke_loc *tpl;
2781 int ret = 0;
2782 void *ip;
2783
2784 if (user_mode(regs))
2785 return 0;
2786
2787 /*
2788 * Having observed our INT3 instruction, we now must observe
2789 * text_poke_array with non-zero refcount:
2790 *
2791 * text_poke_array_refs = 1 INT3
2792 * WMB RMB
2793 * write INT3 if (text_poke_array_refs != 0)
2794 */
2795 smp_rmb();
2796
2797 if (!try_get_text_poke_array())
2798 return 0;
2799
2800 /*
2801 * Discount the INT3. See smp_text_poke_batch_finish().
2802 */
2803 ip = (void *) regs->ip - INT3_INSN_SIZE;
2804
2805 /*
2806 * Skip the binary search if there is a single member in the vector.
2807 */
2808 if (unlikely(text_poke_array.nr_entries > 1)) {
2809 tpl = __inline_bsearch(key: ip, base: text_poke_array.vec, num: text_poke_array.nr_entries,
2810 size: sizeof(struct smp_text_poke_loc),
2811 cmp: patch_cmp);
2812 if (!tpl)
2813 goto out_put;
2814 } else {
2815 tpl = text_poke_array.vec;
2816 if (text_poke_addr(tpl) != ip)
2817 goto out_put;
2818 }
2819
2820 ip += tpl->len;
2821
2822 switch (tpl->opcode) {
2823 case INT3_INSN_OPCODE:
2824 /*
2825 * Someone poked an explicit INT3, they'll want to handle it,
2826 * do not consume.
2827 */
2828 goto out_put;
2829
2830 case RET_INSN_OPCODE:
2831 int3_emulate_ret(regs);
2832 break;
2833
2834 case CALL_INSN_OPCODE:
2835 int3_emulate_call(regs, func: (long)ip + tpl->disp);
2836 break;
2837
2838 case JMP32_INSN_OPCODE:
2839 case JMP8_INSN_OPCODE:
2840 int3_emulate_jmp(regs, ip: (long)ip + tpl->disp);
2841 break;
2842
2843 case 0x70 ... 0x7f: /* Jcc */
2844 int3_emulate_jcc(regs, cc: tpl->opcode & 0xf, ip: (long)ip, disp: tpl->disp);
2845 break;
2846
2847 default:
2848 BUG();
2849 }
2850
2851 ret = 1;
2852
2853out_put:
2854 put_text_poke_array();
2855 return ret;
2856}
2857
2858/**
2859 * smp_text_poke_batch_finish() -- update instructions on live kernel on SMP
2860 *
2861 * Input state:
2862 * text_poke_array.vec: vector of instructions to patch
2863 * text_poke_array.nr_entries: number of entries in the vector
2864 *
2865 * Modify multi-byte instructions by using INT3 breakpoints on SMP.
2866 * We completely avoid using stop_machine() here, and achieve the
2867 * synchronization using INT3 breakpoints and SMP cross-calls.
2868 *
2869 * The way it is done:
2870 * - For each entry in the vector:
2871 * - add an INT3 trap to the address that will be patched
2872 * - SMP sync all CPUs
2873 * - For each entry in the vector:
2874 * - update all but the first byte of the patched range
2875 * - SMP sync all CPUs
2876 * - For each entry in the vector:
2877 * - replace the first byte (INT3) by the first byte of the
2878 * replacing opcode
2879 * - SMP sync all CPUs
2880 */
2881void smp_text_poke_batch_finish(void)
2882{
2883 unsigned char int3 = INT3_INSN_OPCODE;
2884 unsigned int i;
2885 int do_sync;
2886
2887 if (!text_poke_array.nr_entries)
2888 return;
2889
2890 lockdep_assert_held(&text_mutex);
2891
2892 /*
2893 * Corresponds to the implicit memory barrier in try_get_text_poke_array() to
2894 * ensure reading a non-zero refcount provides up to date text_poke_array data.
2895 */
2896 for_each_possible_cpu(i)
2897 atomic_set_release(per_cpu_ptr(&text_poke_array_refs, i), i: 1);
2898
2899 /*
2900 * Function tracing can enable thousands of places that need to be
2901 * updated. This can take quite some time, and with full kernel debugging
2902 * enabled, this could cause the softlockup watchdog to trigger.
2903 * This function gets called every 256 entries added to be patched.
2904 * Call cond_resched() here to make sure that other tasks can get scheduled
2905 * while processing all the functions being patched.
2906 */
2907 cond_resched();
2908
2909 /*
2910 * Corresponding read barrier in INT3 notifier for making sure the
2911 * text_poke_array.nr_entries and handler are correctly ordered wrt. patching.
2912 */
2913 smp_wmb();
2914
2915 /*
2916 * First step: add a INT3 trap to the address that will be patched.
2917 */
2918 for (i = 0; i < text_poke_array.nr_entries; i++) {
2919 text_poke_array.vec[i].old = *(u8 *)text_poke_addr(tpl: &text_poke_array.vec[i]);
2920 text_poke(addr: text_poke_addr(tpl: &text_poke_array.vec[i]), opcode: &int3, INT3_INSN_SIZE);
2921 }
2922
2923 smp_text_poke_sync_each_cpu();
2924
2925 /*
2926 * Second step: update all but the first byte of the patched range.
2927 */
2928 for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) {
2929 u8 old[TEXT_POKE_MAX_OPCODE_SIZE+1] = { text_poke_array.vec[i].old, };
2930 u8 _new[TEXT_POKE_MAX_OPCODE_SIZE+1];
2931 const u8 *new = text_poke_array.vec[i].text;
2932 int len = text_poke_array.vec[i].len;
2933
2934 if (len - INT3_INSN_SIZE > 0) {
2935 memcpy(to: old + INT3_INSN_SIZE,
2936 from: text_poke_addr(tpl: &text_poke_array.vec[i]) + INT3_INSN_SIZE,
2937 len: len - INT3_INSN_SIZE);
2938
2939 if (len == 6) {
2940 _new[0] = 0x0f;
2941 memcpy(to: _new + 1, from: new, len: 5);
2942 new = _new;
2943 }
2944
2945 text_poke(addr: text_poke_addr(tpl: &text_poke_array.vec[i]) + INT3_INSN_SIZE,
2946 opcode: new + INT3_INSN_SIZE,
2947 len: len - INT3_INSN_SIZE);
2948
2949 do_sync++;
2950 }
2951
2952 /*
2953 * Emit a perf event to record the text poke, primarily to
2954 * support Intel PT decoding which must walk the executable code
2955 * to reconstruct the trace. The flow up to here is:
2956 * - write INT3 byte
2957 * - IPI-SYNC
2958 * - write instruction tail
2959 * At this point the actual control flow will be through the
2960 * INT3 and handler and not hit the old or new instruction.
2961 * Intel PT outputs FUP/TIP packets for the INT3, so the flow
2962 * can still be decoded. Subsequently:
2963 * - emit RECORD_TEXT_POKE with the new instruction
2964 * - IPI-SYNC
2965 * - write first byte
2966 * - IPI-SYNC
2967 * So before the text poke event timestamp, the decoder will see
2968 * either the old instruction flow or FUP/TIP of INT3. After the
2969 * text poke event timestamp, the decoder will see either the
2970 * new instruction flow or FUP/TIP of INT3. Thus decoders can
2971 * use the timestamp as the point at which to modify the
2972 * executable code.
2973 * The old instruction is recorded so that the event can be
2974 * processed forwards or backwards.
2975 */
2976 perf_event_text_poke(addr: text_poke_addr(tpl: &text_poke_array.vec[i]), old_bytes: old, old_len: len, new_bytes: new, new_len: len);
2977 }
2978
2979 if (do_sync) {
2980 /*
2981 * According to Intel, this core syncing is very likely
2982 * not necessary and we'd be safe even without it. But
2983 * better safe than sorry (plus there's not only Intel).
2984 */
2985 smp_text_poke_sync_each_cpu();
2986 }
2987
2988 /*
2989 * Third step: replace the first byte (INT3) by the first byte of the
2990 * replacing opcode.
2991 */
2992 for (do_sync = 0, i = 0; i < text_poke_array.nr_entries; i++) {
2993 u8 byte = text_poke_array.vec[i].text[0];
2994
2995 if (text_poke_array.vec[i].len == 6)
2996 byte = 0x0f;
2997
2998 if (byte == INT3_INSN_OPCODE)
2999 continue;
3000
3001 text_poke(addr: text_poke_addr(tpl: &text_poke_array.vec[i]), opcode: &byte, INT3_INSN_SIZE);
3002 do_sync++;
3003 }
3004
3005 if (do_sync)
3006 smp_text_poke_sync_each_cpu();
3007
3008 /*
3009 * Remove and wait for refs to be zero.
3010 *
3011 * Notably, if after step-3 above the INT3 got removed, then the
3012 * smp_text_poke_sync_each_cpu() will have serialized against any running INT3
3013 * handlers and the below spin-wait will not happen.
3014 *
3015 * IOW. unless the replacement instruction is INT3, this case goes
3016 * unused.
3017 */
3018 for_each_possible_cpu(i) {
3019 atomic_t *refs = per_cpu_ptr(&text_poke_array_refs, i);
3020
3021 if (unlikely(!atomic_dec_and_test(refs)))
3022 atomic_cond_read_acquire(refs, !VAL);
3023 }
3024
3025 /* They are all completed: */
3026 text_poke_array.nr_entries = 0;
3027}
3028
3029static void __smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate)
3030{
3031 struct smp_text_poke_loc *tpl;
3032 struct insn insn;
3033 int ret, i = 0;
3034
3035 tpl = &text_poke_array.vec[text_poke_array.nr_entries++];
3036
3037 if (len == 6)
3038 i = 1;
3039 memcpy(to: (void *)tpl->text, from: opcode+i, len: len-i);
3040 if (!emulate)
3041 emulate = opcode;
3042
3043 ret = insn_decode_kernel(&insn, emulate);
3044 BUG_ON(ret < 0);
3045
3046 tpl->rel_addr = addr - (void *)_stext;
3047 tpl->len = len;
3048 tpl->opcode = insn.opcode.bytes[0];
3049
3050 if (is_jcc32(insn: &insn)) {
3051 /*
3052 * Map Jcc.d32 onto Jcc.d8 and use len to distinguish.
3053 */
3054 tpl->opcode = insn.opcode.bytes[1] - 0x10;
3055 }
3056
3057 switch (tpl->opcode) {
3058 case RET_INSN_OPCODE:
3059 case JMP32_INSN_OPCODE:
3060 case JMP8_INSN_OPCODE:
3061 /*
3062 * Control flow instructions without implied execution of the
3063 * next instruction can be padded with INT3.
3064 */
3065 for (i = insn.length; i < len; i++)
3066 BUG_ON(tpl->text[i] != INT3_INSN_OPCODE);
3067 break;
3068
3069 default:
3070 BUG_ON(len != insn.length);
3071 }
3072
3073 switch (tpl->opcode) {
3074 case INT3_INSN_OPCODE:
3075 case RET_INSN_OPCODE:
3076 break;
3077
3078 case CALL_INSN_OPCODE:
3079 case JMP32_INSN_OPCODE:
3080 case JMP8_INSN_OPCODE:
3081 case 0x70 ... 0x7f: /* Jcc */
3082 tpl->disp = insn.immediate.value;
3083 break;
3084
3085 default: /* assume NOP */
3086 switch (len) {
3087 case 2: /* NOP2 -- emulate as JMP8+0 */
3088 BUG_ON(memcmp(emulate, x86_nops[len], len));
3089 tpl->opcode = JMP8_INSN_OPCODE;
3090 tpl->disp = 0;
3091 break;
3092
3093 case 5: /* NOP5 -- emulate as JMP32+0 */
3094 BUG_ON(memcmp(emulate, x86_nops[len], len));
3095 tpl->opcode = JMP32_INSN_OPCODE;
3096 tpl->disp = 0;
3097 break;
3098
3099 default: /* unknown instruction */
3100 BUG();
3101 }
3102 break;
3103 }
3104}
3105
3106/*
3107 * We hard rely on the text_poke_array.vec being ordered; ensure this is so by flushing
3108 * early if needed.
3109 */
3110static bool text_poke_addr_ordered(void *addr)
3111{
3112 WARN_ON_ONCE(!addr);
3113
3114 if (!text_poke_array.nr_entries)
3115 return true;
3116
3117 /*
3118 * If the last current entry's address is higher than the
3119 * new entry's address we'd like to add, then ordering
3120 * is violated and we must first flush all pending patching
3121 * requests:
3122 */
3123 if (text_poke_addr(tpl: text_poke_array.vec + text_poke_array.nr_entries-1) > addr)
3124 return false;
3125
3126 return true;
3127}
3128
3129/**
3130 * smp_text_poke_batch_add() -- update instruction on live kernel on SMP, batched
3131 * @addr: address to patch
3132 * @opcode: opcode of new instruction
3133 * @len: length to copy
3134 * @emulate: instruction to be emulated
3135 *
3136 * Add a new instruction to the current queue of to-be-patched instructions
3137 * the kernel maintains. The patching request will not be executed immediately,
3138 * but becomes part of an array of patching requests, optimized for batched
3139 * execution. All pending patching requests will be executed on the next
3140 * smp_text_poke_batch_finish() call.
3141 */
3142void __ref smp_text_poke_batch_add(void *addr, const void *opcode, size_t len, const void *emulate)
3143{
3144 if (text_poke_array.nr_entries == TEXT_POKE_ARRAY_MAX || !text_poke_addr_ordered(addr))
3145 smp_text_poke_batch_finish();
3146 __smp_text_poke_batch_add(addr, opcode, len, emulate);
3147}
3148
3149/**
3150 * smp_text_poke_single() -- update instruction on live kernel on SMP immediately
3151 * @addr: address to patch
3152 * @opcode: opcode of new instruction
3153 * @len: length to copy
3154 * @emulate: instruction to be emulated
3155 *
3156 * Update a single instruction with the vector in the stack, avoiding
3157 * dynamically allocated memory. This function should be used when it is
3158 * not possible to allocate memory for a vector. The single instruction
3159 * is patched in immediately.
3160 */
3161void __ref smp_text_poke_single(void *addr, const void *opcode, size_t len, const void *emulate)
3162{
3163 smp_text_poke_batch_add(addr, opcode, len, emulate);
3164 smp_text_poke_batch_finish();
3165}
3166