1// SPDX-License-Identifier: GPL-2.0-only
2
3#define pr_fmt(fmt) "callthunks: " fmt
4
5#include <linux/debugfs.h>
6#include <linux/kallsyms.h>
7#include <linux/memory.h>
8#include <linux/moduleloader.h>
9#include <linux/static_call.h>
10
11#include <asm/alternative.h>
12#include <asm/asm-offsets.h>
13#include <asm/cpu.h>
14#include <asm/ftrace.h>
15#include <asm/insn.h>
16#include <asm/kexec.h>
17#include <asm/nospec-branch.h>
18#include <asm/paravirt.h>
19#include <asm/sections.h>
20#include <asm/switch_to.h>
21#include <asm/sync_core.h>
22#include <asm/text-patching.h>
23#include <asm/xen/hypercall.h>
24
25static int __initdata_or_module debug_callthunks;
26
27#define MAX_PATCH_LEN (255-1)
28
29#define prdbg(fmt, args...) \
30do { \
31 if (debug_callthunks) \
32 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
33} while(0)
34
35static int __init debug_thunks(char *str)
36{
37 debug_callthunks = 1;
38 return 1;
39}
40__setup("debug-callthunks", debug_thunks);
41
42#ifdef CONFIG_CALL_THUNKS_DEBUG
43DEFINE_PER_CPU(u64, __x86_call_count);
44DEFINE_PER_CPU(u64, __x86_ret_count);
45DEFINE_PER_CPU(u64, __x86_stuffs_count);
46DEFINE_PER_CPU(u64, __x86_ctxsw_count);
47EXPORT_PER_CPU_SYMBOL_GPL(__x86_ctxsw_count);
48EXPORT_PER_CPU_SYMBOL_GPL(__x86_call_count);
49#endif
50
51extern s32 __call_sites[], __call_sites_end[];
52
53struct core_text {
54 unsigned long base;
55 unsigned long end;
56 const char *name;
57};
58
59static bool thunks_initialized __ro_after_init;
60
61static const struct core_text builtin_coretext = {
62 .base = (unsigned long)_text,
63 .end = (unsigned long)_etext,
64 .name = "builtin",
65};
66
67asm (
68 ".pushsection .rodata \n"
69 ".global skl_call_thunk_template \n"
70 "skl_call_thunk_template: \n"
71 __stringify(INCREMENT_CALL_DEPTH)" \n"
72 ".global skl_call_thunk_tail \n"
73 "skl_call_thunk_tail: \n"
74 ".popsection \n"
75);
76
77extern u8 skl_call_thunk_template[];
78extern u8 skl_call_thunk_tail[];
79
80#define SKL_TMPL_SIZE \
81 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
82
83extern void error_entry(void);
84extern void xen_error_entry(void);
85extern void paranoid_entry(void);
86
87static inline bool within_coretext(const struct core_text *ct, void *addr)
88{
89 unsigned long p = (unsigned long)addr;
90
91 return ct->base <= p && p < ct->end;
92}
93
94static inline bool within_module_coretext(void *addr)
95{
96 bool ret = false;
97
98#ifdef CONFIG_MODULES
99 struct module *mod;
100
101 guard(rcu)();
102 mod = __module_address(addr: (unsigned long)addr);
103 if (mod && within_module_core(addr: (unsigned long)addr, mod))
104 ret = true;
105#endif
106 return ret;
107}
108
109static bool is_coretext(const struct core_text *ct, void *addr)
110{
111 if (ct && within_coretext(ct, addr))
112 return true;
113 if (within_coretext(ct: &builtin_coretext, addr))
114 return true;
115 return within_module_coretext(addr);
116}
117
118static bool skip_addr(void *dest)
119{
120 if (dest == error_entry)
121 return true;
122 if (dest == paranoid_entry)
123 return true;
124 if (dest == xen_error_entry)
125 return true;
126 /* Does FILL_RSB... */
127 if (dest == __switch_to_asm)
128 return true;
129 /* Accounts directly */
130 if (dest == ret_from_fork)
131 return true;
132#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
133 if (dest == soft_restart_cpu)
134 return true;
135#endif
136#ifdef CONFIG_FUNCTION_TRACER
137 if (dest == __fentry__)
138 return true;
139#endif
140#ifdef CONFIG_KEXEC_CORE
141# ifdef CONFIG_X86_64
142 if (dest >= (void *)__relocate_kernel_start &&
143 dest < (void *)__relocate_kernel_end)
144 return true;
145# else
146 if (dest >= (void *)relocate_kernel &&
147 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
148 return true;
149# endif
150#endif
151 return false;
152}
153
154static __init_or_module void *call_get_dest(void *addr)
155{
156 struct insn insn;
157 void *dest;
158 int ret;
159
160 ret = insn_decode_kernel(&insn, addr);
161 if (ret)
162 return ERR_PTR(error: ret);
163
164 /* Patched out call? */
165 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
166 return NULL;
167
168 dest = addr + insn.length + insn.immediate.value;
169 if (skip_addr(dest))
170 return NULL;
171 return dest;
172}
173
174static const u8 nops[] = {
175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
177 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
178 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
179};
180
181static void *patch_dest(void *dest, bool direct)
182{
183 unsigned int tsize = SKL_TMPL_SIZE;
184 u8 insn_buff[MAX_PATCH_LEN];
185 u8 *pad = dest - tsize;
186
187 memcpy(to: insn_buff, from: skl_call_thunk_template, len: tsize);
188 text_poke_apply_relocation(buf: insn_buff, instr: pad, instrlen: tsize, repl: skl_call_thunk_template, repl_len: tsize);
189
190 /* Already patched? */
191 if (!bcmp(pad, insn_buff, tsize))
192 return pad;
193
194 /* Ensure there are nops */
195 if (bcmp(pad, nops, tsize)) {
196 pr_warn_once("Invalid padding area for %pS\n", dest);
197 return NULL;
198 }
199
200 if (direct)
201 memcpy(to: pad, from: insn_buff, len: tsize);
202 else
203 text_poke_copy_locked(addr: pad, opcode: insn_buff, len: tsize, core_ok: true);
204 return pad;
205}
206
207static __init_or_module void patch_call(void *addr, const struct core_text *ct)
208{
209 void *pad, *dest;
210 u8 bytes[8];
211
212 if (!within_coretext(ct, addr))
213 return;
214
215 dest = call_get_dest(addr);
216 if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
217 return;
218
219 if (!is_coretext(ct, addr: dest))
220 return;
221
222 pad = patch_dest(dest, direct: within_coretext(ct, addr: dest));
223 if (!pad)
224 return;
225
226 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
227 dest, dest, pad);
228 __text_gen_insn(buf: bytes, CALL_INSN_OPCODE, addr, dest: pad, CALL_INSN_SIZE);
229 text_poke_early(addr, opcode: bytes, CALL_INSN_SIZE);
230}
231
232static __init_or_module void
233patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
234{
235 s32 *s;
236
237 for (s = start; s < end; s++)
238 patch_call(addr: (void *)s + *s, ct);
239}
240
241static __init_or_module void
242callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
243{
244 prdbg("Patching call sites %s\n", ct->name);
245 patch_call_sites(start: cs->call_start, end: cs->call_end, ct);
246 prdbg("Patching call sites done%s\n", ct->name);
247}
248
249void __init callthunks_patch_builtin_calls(void)
250{
251 struct callthunk_sites cs = {
252 .call_start = __call_sites,
253 .call_end = __call_sites_end,
254 };
255
256 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
257 return;
258
259 pr_info("Setting up call depth tracking\n");
260 mutex_lock(lock: &text_mutex);
261 callthunks_setup(cs: &cs, ct: &builtin_coretext);
262 thunks_initialized = true;
263 mutex_unlock(lock: &text_mutex);
264}
265
266void *callthunks_translate_call_dest(void *dest)
267{
268 void *target;
269
270 lockdep_assert_held(&text_mutex);
271
272 if (!thunks_initialized || skip_addr(dest))
273 return dest;
274
275 if (!is_coretext(NULL, addr: dest))
276 return dest;
277
278 target = patch_dest(dest, direct: false);
279 return target ? : dest;
280}
281
282#ifdef CONFIG_BPF_JIT
283static bool is_callthunk(void *addr)
284{
285 unsigned int tmpl_size = SKL_TMPL_SIZE;
286 u8 insn_buff[MAX_PATCH_LEN];
287 unsigned long dest;
288 u8 *pad;
289
290 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
291 if (!thunks_initialized || skip_addr((void *)dest))
292 return false;
293
294 pad = (void *)(dest - tmpl_size);
295
296 memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
297 text_poke_apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size);
298
299 return !bcmp(pad, insn_buff, tmpl_size);
300}
301
302int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip)
303{
304 unsigned int tmpl_size = SKL_TMPL_SIZE;
305 u8 insn_buff[MAX_PATCH_LEN];
306
307 if (!thunks_initialized)
308 return 0;
309
310 /* Is function call target a thunk? */
311 if (func && is_callthunk(func))
312 return 0;
313
314 memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
315 text_poke_apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size);
316
317 memcpy(*pprog, insn_buff, tmpl_size);
318 *pprog += tmpl_size;
319 return tmpl_size;
320}
321#endif
322
323#ifdef CONFIG_MODULES
324void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
325 struct module *mod)
326{
327 struct core_text ct = {
328 .base = (unsigned long)mod->mem[MOD_TEXT].base,
329 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
330 .name = mod->name,
331 };
332
333 if (!thunks_initialized)
334 return;
335
336 mutex_lock(lock: &text_mutex);
337 callthunks_setup(cs, ct: &ct);
338 mutex_unlock(lock: &text_mutex);
339}
340#endif /* CONFIG_MODULES */
341
342#if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
343static int callthunks_debug_show(struct seq_file *m, void *p)
344{
345 unsigned long cpu = (unsigned long)m->private;
346
347 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
348 per_cpu(__x86_call_count, cpu),
349 per_cpu(__x86_ret_count, cpu),
350 per_cpu(__x86_stuffs_count, cpu),
351 per_cpu(__x86_ctxsw_count, cpu));
352 return 0;
353}
354
355static int callthunks_debug_open(struct inode *inode, struct file *file)
356{
357 return single_open(file, callthunks_debug_show, inode->i_private);
358}
359
360static const struct file_operations dfs_ops = {
361 .open = callthunks_debug_open,
362 .read = seq_read,
363 .llseek = seq_lseek,
364 .release = single_release,
365};
366
367static int __init callthunks_debugfs_init(void)
368{
369 struct dentry *dir;
370 unsigned long cpu;
371
372 dir = debugfs_create_dir("callthunks", NULL);
373 for_each_possible_cpu(cpu) {
374 void *arg = (void *)cpu;
375 char name [10];
376
377 sprintf(name, "cpu%lu", cpu);
378 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
379 }
380 return 0;
381}
382__initcall(callthunks_debugfs_init);
383#endif
384