1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2002 Richard Henderson
4 * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
5 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
6 * Copyright (C) 2024 Mike Rapoport IBM.
7 */
8
9#define pr_fmt(fmt) "execmem: " fmt
10
11#include <linux/mm.h>
12#include <linux/mutex.h>
13#include <linux/vmalloc.h>
14#include <linux/execmem.h>
15#include <linux/maple_tree.h>
16#include <linux/set_memory.h>
17#include <linux/moduleloader.h>
18#include <linux/text-patching.h>
19
20#include <asm/tlbflush.h>
21
22#include "internal.h"
23
24static struct execmem_info *execmem_info __ro_after_init;
25static struct execmem_info default_execmem_info __ro_after_init;
26
27#ifdef CONFIG_MMU
28static void *execmem_vmalloc(struct execmem_range *range, size_t size,
29 pgprot_t pgprot, unsigned long vm_flags)
30{
31 bool kasan = range->flags & EXECMEM_KASAN_SHADOW;
32 gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN;
33 unsigned int align = range->alignment;
34 unsigned long start = range->start;
35 unsigned long end = range->end;
36 void *p;
37
38 if (kasan)
39 vm_flags |= VM_DEFER_KMEMLEAK;
40
41 p = __vmalloc_node_range(size, align, start, end, gfp_flags,
42 pgprot, vm_flags, NUMA_NO_NODE,
43 __builtin_return_address(0));
44 if (!p && range->fallback_start) {
45 start = range->fallback_start;
46 end = range->fallback_end;
47 p = __vmalloc_node_range(size, align, start, end, gfp_flags,
48 pgprot, vm_flags, NUMA_NO_NODE,
49 __builtin_return_address(0));
50 }
51
52 if (!p) {
53 pr_warn_ratelimited("unable to allocate memory\n");
54 return NULL;
55 }
56
57 if (kasan && (kasan_alloc_module_shadow(addr: p, size, GFP_KERNEL) < 0)) {
58 vfree(addr: p);
59 return NULL;
60 }
61
62 return p;
63}
64
65struct vm_struct *execmem_vmap(size_t size)
66{
67 struct execmem_range *range = &execmem_info->ranges[EXECMEM_MODULE_DATA];
68 struct vm_struct *area;
69
70 area = __get_vm_area_node(size, align: range->alignment, PAGE_SHIFT, VM_ALLOC,
71 start: range->start, end: range->end, NUMA_NO_NODE,
72 GFP_KERNEL, caller: __builtin_return_address(0));
73 if (!area && range->fallback_start)
74 area = __get_vm_area_node(size, align: range->alignment, PAGE_SHIFT, VM_ALLOC,
75 start: range->fallback_start, end: range->fallback_end,
76 NUMA_NO_NODE, GFP_KERNEL, caller: __builtin_return_address(0));
77
78 return area;
79}
80#else
81static void *execmem_vmalloc(struct execmem_range *range, size_t size,
82 pgprot_t pgprot, unsigned long vm_flags)
83{
84 return vmalloc(size);
85}
86#endif /* CONFIG_MMU */
87
88#ifdef CONFIG_ARCH_HAS_EXECMEM_ROX
89struct execmem_cache {
90 struct mutex mutex;
91 struct maple_tree busy_areas;
92 struct maple_tree free_areas;
93 unsigned int pending_free_cnt; /* protected by mutex */
94};
95
96/* delay to schedule asynchronous free if fast path free fails */
97#define FREE_DELAY (msecs_to_jiffies(10))
98
99/* mark entries in busy_areas that should be freed asynchronously */
100#define PENDING_FREE_MASK (1 << (PAGE_SHIFT - 1))
101
102static struct execmem_cache execmem_cache = {
103 .mutex = __MUTEX_INITIALIZER(execmem_cache.mutex),
104 .busy_areas = MTREE_INIT_EXT(busy_areas, MT_FLAGS_LOCK_EXTERN,
105 execmem_cache.mutex),
106 .free_areas = MTREE_INIT_EXT(free_areas, MT_FLAGS_LOCK_EXTERN,
107 execmem_cache.mutex),
108};
109
110static inline unsigned long mas_range_len(struct ma_state *mas)
111{
112 return mas->last - mas->index + 1;
113}
114
115static int execmem_set_direct_map_valid(struct vm_struct *vm, bool valid)
116{
117 unsigned int nr = (1 << get_vm_area_page_order(vm));
118 unsigned int updated = 0;
119 int err = 0;
120
121 for (int i = 0; i < vm->nr_pages; i += nr) {
122 err = set_direct_map_valid_noflush(page: vm->pages[i], nr, valid);
123 if (err)
124 goto err_restore;
125 updated += nr;
126 }
127
128 return 0;
129
130err_restore:
131 for (int i = 0; i < updated; i += nr)
132 set_direct_map_valid_noflush(page: vm->pages[i], nr, valid: !valid);
133
134 return err;
135}
136
137static int execmem_force_rw(void *ptr, size_t size)
138{
139 unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT;
140 unsigned long addr = (unsigned long)ptr;
141 int ret;
142
143 ret = set_memory_nx(addr, numpages: nr);
144 if (ret)
145 return ret;
146
147 return set_memory_rw(addr, numpages: nr);
148}
149
150int execmem_restore_rox(void *ptr, size_t size)
151{
152 unsigned int nr = PAGE_ALIGN(size) >> PAGE_SHIFT;
153 unsigned long addr = (unsigned long)ptr;
154
155 return set_memory_rox(addr, numpages: nr);
156}
157
158static void execmem_cache_clean(struct work_struct *work)
159{
160 struct maple_tree *free_areas = &execmem_cache.free_areas;
161 struct mutex *mutex = &execmem_cache.mutex;
162 MA_STATE(mas, free_areas, 0, ULONG_MAX);
163 void *area;
164
165 mutex_lock(lock: mutex);
166 mas_for_each(&mas, area, ULONG_MAX) {
167 size_t size = mas_range_len(mas: &mas);
168
169 if (IS_ALIGNED(size, PMD_SIZE) &&
170 IS_ALIGNED(mas.index, PMD_SIZE)) {
171 struct vm_struct *vm = find_vm_area(addr: area);
172
173 execmem_set_direct_map_valid(vm, valid: true);
174 mas_store_gfp(mas: &mas, NULL, GFP_KERNEL);
175 vfree(addr: area);
176 }
177 }
178 mutex_unlock(lock: mutex);
179}
180
181static DECLARE_WORK(execmem_cache_clean_work, execmem_cache_clean);
182
183static int execmem_cache_add_locked(void *ptr, size_t size, gfp_t gfp_mask)
184{
185 struct maple_tree *free_areas = &execmem_cache.free_areas;
186 unsigned long addr = (unsigned long)ptr;
187 MA_STATE(mas, free_areas, addr - 1, addr + 1);
188 unsigned long lower, upper;
189 void *area = NULL;
190
191 lower = addr;
192 upper = addr + size - 1;
193
194 area = mas_walk(mas: &mas);
195 if (area && mas.last == addr - 1)
196 lower = mas.index;
197
198 area = mas_next(mas: &mas, ULONG_MAX);
199 if (area && mas.index == addr + size)
200 upper = mas.last;
201
202 mas_set_range(mas: &mas, start: lower, last: upper);
203 return mas_store_gfp(mas: &mas, entry: (void *)lower, gfp: gfp_mask);
204}
205
206static int execmem_cache_add(void *ptr, size_t size, gfp_t gfp_mask)
207{
208 guard(mutex)(T: &execmem_cache.mutex);
209
210 return execmem_cache_add_locked(ptr, size, gfp_mask);
211}
212
213static bool within_range(struct execmem_range *range, struct ma_state *mas,
214 size_t size)
215{
216 unsigned long addr = mas->index;
217
218 if (addr >= range->start && addr + size < range->end)
219 return true;
220
221 if (range->fallback_start &&
222 addr >= range->fallback_start && addr + size < range->fallback_end)
223 return true;
224
225 return false;
226}
227
228static void *__execmem_cache_alloc(struct execmem_range *range, size_t size)
229{
230 struct maple_tree *free_areas = &execmem_cache.free_areas;
231 struct maple_tree *busy_areas = &execmem_cache.busy_areas;
232 MA_STATE(mas_free, free_areas, 0, ULONG_MAX);
233 MA_STATE(mas_busy, busy_areas, 0, ULONG_MAX);
234 struct mutex *mutex = &execmem_cache.mutex;
235 unsigned long addr, last, area_size = 0;
236 void *area, *ptr = NULL;
237 int err;
238
239 mutex_lock(lock: mutex);
240 mas_for_each(&mas_free, area, ULONG_MAX) {
241 area_size = mas_range_len(mas: &mas_free);
242
243 if (area_size >= size && within_range(range, mas: &mas_free, size))
244 break;
245 }
246
247 if (area_size < size)
248 goto out_unlock;
249
250 addr = mas_free.index;
251 last = mas_free.last;
252
253 /* insert allocated size to busy_areas at range [addr, addr + size) */
254 mas_set_range(mas: &mas_busy, start: addr, last: addr + size - 1);
255 err = mas_store_gfp(mas: &mas_busy, entry: (void *)addr, GFP_KERNEL);
256 if (err)
257 goto out_unlock;
258
259 mas_store_gfp(mas: &mas_free, NULL, GFP_KERNEL);
260 if (area_size > size) {
261 void *ptr = (void *)(addr + size);
262
263 /*
264 * re-insert remaining free size to free_areas at range
265 * [addr + size, last]
266 */
267 mas_set_range(mas: &mas_free, start: addr + size, last);
268 err = mas_store_gfp(mas: &mas_free, entry: ptr, GFP_KERNEL);
269 if (err) {
270 mas_store_gfp(mas: &mas_busy, NULL, GFP_KERNEL);
271 goto out_unlock;
272 }
273 }
274 ptr = (void *)addr;
275
276out_unlock:
277 mutex_unlock(lock: mutex);
278 return ptr;
279}
280
281static int execmem_cache_populate(struct execmem_range *range, size_t size)
282{
283 unsigned long vm_flags = VM_ALLOW_HUGE_VMAP;
284 struct vm_struct *vm;
285 size_t alloc_size;
286 int err = -ENOMEM;
287 void *p;
288
289 alloc_size = round_up(size, PMD_SIZE);
290 p = execmem_vmalloc(range, size: alloc_size, PAGE_KERNEL, vm_flags);
291 if (!p) {
292 alloc_size = size;
293 p = execmem_vmalloc(range, size: alloc_size, PAGE_KERNEL, vm_flags);
294 }
295
296 if (!p)
297 return err;
298
299 vm = find_vm_area(addr: p);
300 if (!vm)
301 goto err_free_mem;
302
303 /* fill memory with instructions that will trap */
304 execmem_fill_trapping_insns(ptr: p, size: alloc_size);
305
306 err = set_memory_rox(addr: (unsigned long)p, numpages: vm->nr_pages);
307 if (err)
308 goto err_free_mem;
309
310 err = execmem_cache_add(ptr: p, size: alloc_size, GFP_KERNEL);
311 if (err)
312 goto err_reset_direct_map;
313
314 return 0;
315
316err_reset_direct_map:
317 execmem_set_direct_map_valid(vm, valid: true);
318err_free_mem:
319 vfree(addr: p);
320 return err;
321}
322
323static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
324{
325 void *p;
326 int err;
327
328 p = __execmem_cache_alloc(range, size);
329 if (p)
330 return p;
331
332 err = execmem_cache_populate(range, size);
333 if (err)
334 return NULL;
335
336 return __execmem_cache_alloc(range, size);
337}
338
339static inline bool is_pending_free(void *ptr)
340{
341 return ((unsigned long)ptr & PENDING_FREE_MASK);
342}
343
344static inline void *pending_free_set(void *ptr)
345{
346 return (void *)((unsigned long)ptr | PENDING_FREE_MASK);
347}
348
349static inline void *pending_free_clear(void *ptr)
350{
351 return (void *)((unsigned long)ptr & ~PENDING_FREE_MASK);
352}
353
354static int __execmem_cache_free(struct ma_state *mas, void *ptr, gfp_t gfp_mask)
355{
356 size_t size = mas_range_len(mas);
357 int err;
358
359 err = execmem_force_rw(ptr, size);
360 if (err)
361 return err;
362
363 execmem_fill_trapping_insns(ptr, size);
364 execmem_restore_rox(ptr, size);
365
366 err = execmem_cache_add_locked(ptr, size, gfp_mask);
367 if (err)
368 return err;
369
370 mas_store_gfp(mas, NULL, gfp: gfp_mask);
371 return 0;
372}
373
374static void execmem_cache_free_slow(struct work_struct *work);
375static DECLARE_DELAYED_WORK(execmem_cache_free_work, execmem_cache_free_slow);
376
377static void execmem_cache_free_slow(struct work_struct *work)
378{
379 struct maple_tree *busy_areas = &execmem_cache.busy_areas;
380 MA_STATE(mas, busy_areas, 0, ULONG_MAX);
381 void *area;
382
383 guard(mutex)(T: &execmem_cache.mutex);
384
385 if (!execmem_cache.pending_free_cnt)
386 return;
387
388 mas_for_each(&mas, area, ULONG_MAX) {
389 if (!is_pending_free(ptr: area))
390 continue;
391
392 area = pending_free_clear(ptr: area);
393 if (__execmem_cache_free(mas: &mas, ptr: area, GFP_KERNEL))
394 continue;
395
396 execmem_cache.pending_free_cnt--;
397 }
398
399 if (execmem_cache.pending_free_cnt)
400 schedule_delayed_work(dwork: &execmem_cache_free_work, FREE_DELAY);
401 else
402 schedule_work(work: &execmem_cache_clean_work);
403}
404
405static bool execmem_cache_free(void *ptr)
406{
407 struct maple_tree *busy_areas = &execmem_cache.busy_areas;
408 unsigned long addr = (unsigned long)ptr;
409 MA_STATE(mas, busy_areas, addr, addr);
410 void *area;
411 int err;
412
413 guard(mutex)(T: &execmem_cache.mutex);
414
415 area = mas_walk(mas: &mas);
416 if (!area)
417 return false;
418
419 err = __execmem_cache_free(mas: &mas, ptr: area, GFP_KERNEL | __GFP_NORETRY);
420 if (err) {
421 /*
422 * mas points to exact slot we've got the area from, nothing
423 * else can modify the tree because of the mutex, so there
424 * won't be any allocations in mas_store_gfp() and it will just
425 * change the pointer.
426 */
427 area = pending_free_set(ptr: area);
428 mas_store_gfp(mas: &mas, entry: area, GFP_KERNEL);
429 execmem_cache.pending_free_cnt++;
430 schedule_delayed_work(dwork: &execmem_cache_free_work, FREE_DELAY);
431 return true;
432 }
433
434 schedule_work(work: &execmem_cache_clean_work);
435
436 return true;
437}
438
439#else /* CONFIG_ARCH_HAS_EXECMEM_ROX */
440/*
441 * when ROX cache is not used the permissions defined by architectures for
442 * execmem ranges that are updated before use (e.g. EXECMEM_MODULE_TEXT) must
443 * be writable anyway
444 */
445static inline int execmem_force_rw(void *ptr, size_t size)
446{
447 return 0;
448}
449
450static void *execmem_cache_alloc(struct execmem_range *range, size_t size)
451{
452 return NULL;
453}
454
455static bool execmem_cache_free(void *ptr)
456{
457 return false;
458}
459#endif /* CONFIG_ARCH_HAS_EXECMEM_ROX */
460
461void *execmem_alloc(enum execmem_type type, size_t size)
462{
463 struct execmem_range *range = &execmem_info->ranges[type];
464 bool use_cache = range->flags & EXECMEM_ROX_CACHE;
465 unsigned long vm_flags = VM_FLUSH_RESET_PERMS;
466 pgprot_t pgprot = range->pgprot;
467 void *p = NULL;
468
469 size = PAGE_ALIGN(size);
470
471 if (use_cache)
472 p = execmem_cache_alloc(range, size);
473 else
474 p = execmem_vmalloc(range, size, pgprot, vm_flags);
475
476 return kasan_reset_tag(addr: p);
477}
478
479void *execmem_alloc_rw(enum execmem_type type, size_t size)
480{
481 void *p __free(execmem) = execmem_alloc(type, size);
482 int err;
483
484 if (!p)
485 return NULL;
486
487 err = execmem_force_rw(ptr: p, size);
488 if (err)
489 return NULL;
490
491 return no_free_ptr(p);
492}
493
494void execmem_free(void *ptr)
495{
496 /*
497 * This memory may be RO, and freeing RO memory in an interrupt is not
498 * supported by vmalloc.
499 */
500 WARN_ON(in_interrupt());
501
502 if (!execmem_cache_free(ptr))
503 vfree(addr: ptr);
504}
505
506bool execmem_is_rox(enum execmem_type type)
507{
508 return !!(execmem_info->ranges[type].flags & EXECMEM_ROX_CACHE);
509}
510
511static bool execmem_validate(struct execmem_info *info)
512{
513 struct execmem_range *r = &info->ranges[EXECMEM_DEFAULT];
514
515 if (!r->alignment || !r->start || !r->end || !pgprot_val(r->pgprot)) {
516 pr_crit("Invalid parameters for execmem allocator, module loading will fail");
517 return false;
518 }
519
520 if (!IS_ENABLED(CONFIG_ARCH_HAS_EXECMEM_ROX)) {
521 for (int i = EXECMEM_DEFAULT; i < EXECMEM_TYPE_MAX; i++) {
522 r = &info->ranges[i];
523
524 if (r->flags & EXECMEM_ROX_CACHE) {
525 pr_warn_once("ROX cache is not supported\n");
526 r->flags &= ~EXECMEM_ROX_CACHE;
527 }
528 }
529 }
530
531 return true;
532}
533
534static void execmem_init_missing(struct execmem_info *info)
535{
536 struct execmem_range *default_range = &info->ranges[EXECMEM_DEFAULT];
537
538 for (int i = EXECMEM_DEFAULT + 1; i < EXECMEM_TYPE_MAX; i++) {
539 struct execmem_range *r = &info->ranges[i];
540
541 if (!r->start) {
542 if (i == EXECMEM_MODULE_DATA)
543 r->pgprot = PAGE_KERNEL;
544 else
545 r->pgprot = default_range->pgprot;
546 r->alignment = default_range->alignment;
547 r->start = default_range->start;
548 r->end = default_range->end;
549 r->flags = default_range->flags;
550 r->fallback_start = default_range->fallback_start;
551 r->fallback_end = default_range->fallback_end;
552 }
553 }
554}
555
556struct execmem_info * __weak execmem_arch_setup(void)
557{
558 return NULL;
559}
560
561static void __init __execmem_init(void)
562{
563 struct execmem_info *info = execmem_arch_setup();
564
565 if (!info) {
566 info = execmem_info = &default_execmem_info;
567 info->ranges[EXECMEM_DEFAULT].start = VMALLOC_START;
568 info->ranges[EXECMEM_DEFAULT].end = VMALLOC_END;
569 info->ranges[EXECMEM_DEFAULT].pgprot = PAGE_KERNEL_EXEC;
570 info->ranges[EXECMEM_DEFAULT].alignment = 1;
571 }
572
573 if (!execmem_validate(info))
574 return;
575
576 execmem_init_missing(info);
577
578 execmem_info = info;
579}
580
581#ifdef CONFIG_ARCH_WANTS_EXECMEM_LATE
582static int __init execmem_late_init(void)
583{
584 __execmem_init();
585 return 0;
586}
587core_initcall(execmem_late_init);
588#else
589void __init execmem_init(void)
590{
591 __execmem_init();
592}
593#endif
594