1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Kernel Probes (KProbes)
4 *
5 * Copyright (C) IBM Corporation, 2002, 2004
6 *
7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
8 * Probes initial implementation (includes suggestions from
9 * Rusty Russell).
10 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
11 * hlists and exceptions notifier as suggested by Andi Kleen.
12 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
13 * interface to access function arguments.
14 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
15 * exceptions notifier to be first on the priority list.
16 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
17 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
18 * <prasanna@in.ibm.com> added function-return probes.
19 */
20
21#define pr_fmt(fmt) "kprobes: " fmt
22
23#include <linux/kprobes.h>
24#include <linux/hash.h>
25#include <linux/init.h>
26#include <linux/slab.h>
27#include <linux/stddef.h>
28#include <linux/export.h>
29#include <linux/kallsyms.h>
30#include <linux/freezer.h>
31#include <linux/seq_file.h>
32#include <linux/debugfs.h>
33#include <linux/sysctl.h>
34#include <linux/kdebug.h>
35#include <linux/memory.h>
36#include <linux/ftrace.h>
37#include <linux/cpu.h>
38#include <linux/jump_label.h>
39#include <linux/static_call.h>
40#include <linux/perf_event.h>
41#include <linux/execmem.h>
42#include <linux/cleanup.h>
43
44#include <asm/sections.h>
45#include <asm/cacheflush.h>
46#include <asm/errno.h>
47#include <linux/uaccess.h>
48
49#define KPROBE_HASH_BITS 6
50#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
51
52#if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL)
53#define kprobe_sysctls_init() do { } while (0)
54#endif
55
56static int kprobes_initialized;
57/* kprobe_table can be accessed by
58 * - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held.
59 * Or
60 * - RCU hlist traversal under disabling preempt (breakpoint handlers)
61 */
62static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
63
64/* NOTE: change this value only with 'kprobe_mutex' held */
65static bool kprobes_all_disarmed;
66
67/* This protects 'kprobe_table' and 'optimizing_list' */
68static DEFINE_MUTEX(kprobe_mutex);
69static DEFINE_PER_CPU(struct kprobe *, kprobe_instance);
70
71kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
72 unsigned int __unused)
73{
74 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
75}
76
77/*
78 * Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where
79 * kprobes can not probe.
80 */
81static LIST_HEAD(kprobe_blacklist);
82
83#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
84/*
85 * 'kprobe::ainsn.insn' points to the copy of the instruction to be
86 * single-stepped. x86_64, POWER4 and above have no-exec support and
87 * stepping on the instruction on a vmalloced/kmalloced/data page
88 * is a recipe for disaster
89 */
90struct kprobe_insn_page {
91 struct list_head list;
92 kprobe_opcode_t *insns; /* Page of instruction slots */
93 struct kprobe_insn_cache *cache;
94 int nused;
95 int ngarbage;
96 char slot_used[];
97};
98
99static int slots_per_page(struct kprobe_insn_cache *c)
100{
101 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
102}
103
104enum kprobe_slot_state {
105 SLOT_CLEAN = 0,
106 SLOT_DIRTY = 1,
107 SLOT_USED = 2,
108};
109
110void __weak *alloc_insn_page(void)
111{
112 /*
113 * Use execmem_alloc() so this page is within +/- 2GB of where the
114 * kernel image and loaded module images reside. This is required
115 * for most of the architectures.
116 * (e.g. x86-64 needs this to handle the %rip-relative fixups.)
117 */
118 return execmem_alloc(type: EXECMEM_KPROBES, PAGE_SIZE);
119}
120
121static void free_insn_page(void *page)
122{
123 execmem_free(ptr: page);
124}
125
126struct kprobe_insn_cache kprobe_insn_slots = {
127 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
128 .alloc = alloc_insn_page,
129 .free = free_insn_page,
130 .sym = KPROBE_INSN_PAGE_SYM,
131 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
132 .insn_size = MAX_INSN_SIZE,
133 .nr_garbage = 0,
134};
135static int collect_garbage_slots(struct kprobe_insn_cache *c);
136
137/**
138 * __get_insn_slot - Find a slot on an executable page for an instruction.
139 * @c: Pointer to kprobe instruction cache
140 *
141 * Description: Locates available slot on existing executable pages,
142 * allocates an executable page if there's no room on existing ones.
143 * Return: Pointer to instruction slot on success, NULL on failure.
144 */
145kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
146{
147 struct kprobe_insn_page *kip;
148
149 /* Since the slot array is not protected by rcu, we need a mutex */
150 guard(mutex)(T: &c->mutex);
151 do {
152 guard(rcu)();
153 list_for_each_entry_rcu(kip, &c->pages, list) {
154 if (kip->nused < slots_per_page(c)) {
155 int i;
156
157 for (i = 0; i < slots_per_page(c); i++) {
158 if (kip->slot_used[i] == SLOT_CLEAN) {
159 kip->slot_used[i] = SLOT_USED;
160 kip->nused++;
161 return kip->insns + (i * c->insn_size);
162 }
163 }
164 /* kip->nused is broken. Fix it. */
165 kip->nused = slots_per_page(c);
166 WARN_ON(1);
167 }
168 }
169 /* If there are any garbage slots, collect it and try again. */
170 } while (c->nr_garbage && collect_garbage_slots(c) == 0);
171
172 /* All out of space. Need to allocate a new page. */
173 kip = kmalloc(struct_size(kip, slot_used, slots_per_page(c)), GFP_KERNEL);
174 if (!kip)
175 return NULL;
176
177 kip->insns = c->alloc();
178 if (!kip->insns) {
179 kfree(objp: kip);
180 return NULL;
181 }
182 INIT_LIST_HEAD(list: &kip->list);
183 memset(s: kip->slot_used, c: SLOT_CLEAN, n: slots_per_page(c));
184 kip->slot_used[0] = SLOT_USED;
185 kip->nused = 1;
186 kip->ngarbage = 0;
187 kip->cache = c;
188 list_add_rcu(new: &kip->list, head: &c->pages);
189
190 /* Record the perf ksymbol register event after adding the page */
191 perf_event_ksymbol(ksym_type: PERF_RECORD_KSYMBOL_TYPE_OOL, addr: (unsigned long)kip->insns,
192 PAGE_SIZE, unregister: false, sym: c->sym);
193
194 return kip->insns;
195}
196
197/* Return true if all garbages are collected, otherwise false. */
198static bool collect_one_slot(struct kprobe_insn_page *kip, int idx)
199{
200 kip->slot_used[idx] = SLOT_CLEAN;
201 kip->nused--;
202 if (kip->nused != 0)
203 return false;
204
205 /*
206 * Page is no longer in use. Free it unless
207 * it's the last one. We keep the last one
208 * so as not to have to set it up again the
209 * next time somebody inserts a probe.
210 */
211 if (!list_is_singular(head: &kip->list)) {
212 /*
213 * Record perf ksymbol unregister event before removing
214 * the page.
215 */
216 perf_event_ksymbol(ksym_type: PERF_RECORD_KSYMBOL_TYPE_OOL,
217 addr: (unsigned long)kip->insns, PAGE_SIZE, unregister: true,
218 sym: kip->cache->sym);
219 list_del_rcu(entry: &kip->list);
220 synchronize_rcu();
221 kip->cache->free(kip->insns);
222 kfree(objp: kip);
223 }
224 return true;
225}
226
227static int collect_garbage_slots(struct kprobe_insn_cache *c)
228{
229 struct kprobe_insn_page *kip, *next;
230
231 /* Ensure no-one is interrupted on the garbages */
232 synchronize_rcu();
233
234 list_for_each_entry_safe(kip, next, &c->pages, list) {
235 int i;
236
237 if (kip->ngarbage == 0)
238 continue;
239 kip->ngarbage = 0; /* we will collect all garbages */
240 for (i = 0; i < slots_per_page(c); i++) {
241 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, idx: i))
242 break;
243 }
244 }
245 c->nr_garbage = 0;
246 return 0;
247}
248
249static long __find_insn_page(struct kprobe_insn_cache *c,
250 kprobe_opcode_t *slot, struct kprobe_insn_page **pkip)
251{
252 struct kprobe_insn_page *kip = NULL;
253 long idx;
254
255 guard(rcu)();
256 list_for_each_entry_rcu(kip, &c->pages, list) {
257 idx = ((long)slot - (long)kip->insns) /
258 (c->insn_size * sizeof(kprobe_opcode_t));
259 if (idx >= 0 && idx < slots_per_page(c)) {
260 *pkip = kip;
261 return idx;
262 }
263 }
264 /* Could not find this slot. */
265 WARN_ON(1);
266 *pkip = NULL;
267 return -1;
268}
269
270void __free_insn_slot(struct kprobe_insn_cache *c,
271 kprobe_opcode_t *slot, int dirty)
272{
273 struct kprobe_insn_page *kip = NULL;
274 long idx;
275
276 guard(mutex)(T: &c->mutex);
277 idx = __find_insn_page(c, slot, pkip: &kip);
278 /* Mark and sweep: this may sleep */
279 if (kip) {
280 /* Check double free */
281 WARN_ON(kip->slot_used[idx] != SLOT_USED);
282 if (dirty) {
283 kip->slot_used[idx] = SLOT_DIRTY;
284 kip->ngarbage++;
285 if (++c->nr_garbage > slots_per_page(c))
286 collect_garbage_slots(c);
287 } else {
288 collect_one_slot(kip, idx);
289 }
290 }
291}
292
293/*
294 * Check given address is on the page of kprobe instruction slots.
295 * This will be used for checking whether the address on a stack
296 * is on a text area or not.
297 */
298bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
299{
300 struct kprobe_insn_page *kip;
301 bool ret = false;
302
303 rcu_read_lock();
304 list_for_each_entry_rcu(kip, &c->pages, list) {
305 if (addr >= (unsigned long)kip->insns &&
306 addr < (unsigned long)kip->insns + PAGE_SIZE) {
307 ret = true;
308 break;
309 }
310 }
311 rcu_read_unlock();
312
313 return ret;
314}
315
316int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,
317 unsigned long *value, char *type, char *sym)
318{
319 struct kprobe_insn_page *kip;
320 int ret = -ERANGE;
321
322 rcu_read_lock();
323 list_for_each_entry_rcu(kip, &c->pages, list) {
324 if ((*symnum)--)
325 continue;
326 strscpy(sym, c->sym, KSYM_NAME_LEN);
327 *type = 't';
328 *value = (unsigned long)kip->insns;
329 ret = 0;
330 break;
331 }
332 rcu_read_unlock();
333
334 return ret;
335}
336
337#ifdef CONFIG_OPTPROBES
338void __weak *alloc_optinsn_page(void)
339{
340 return alloc_insn_page();
341}
342
343void __weak free_optinsn_page(void *page)
344{
345 free_insn_page(page);
346}
347
348/* For optimized_kprobe buffer */
349struct kprobe_insn_cache kprobe_optinsn_slots = {
350 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
351 .alloc = alloc_optinsn_page,
352 .free = free_optinsn_page,
353 .sym = KPROBE_OPTINSN_PAGE_SYM,
354 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
355 /* .insn_size is initialized later */
356 .nr_garbage = 0,
357};
358#endif /* CONFIG_OPTPROBES */
359#endif /* __ARCH_WANT_KPROBES_INSN_SLOT */
360
361/* We have preemption disabled.. so it is safe to use __ versions */
362static inline void set_kprobe_instance(struct kprobe *kp)
363{
364 __this_cpu_write(kprobe_instance, kp);
365}
366
367static inline void reset_kprobe_instance(void)
368{
369 __this_cpu_write(kprobe_instance, NULL);
370}
371
372/*
373 * This routine is called either:
374 * - under the 'kprobe_mutex' - during kprobe_[un]register().
375 * OR
376 * - with preemption disabled - from architecture specific code.
377 */
378struct kprobe *get_kprobe(void *addr)
379{
380 struct hlist_head *head;
381 struct kprobe *p;
382
383 head = &kprobe_table[hash_ptr(ptr: addr, KPROBE_HASH_BITS)];
384 hlist_for_each_entry_rcu(p, head, hlist,
385 lockdep_is_held(&kprobe_mutex)) {
386 if (p->addr == addr)
387 return p;
388 }
389
390 return NULL;
391}
392NOKPROBE_SYMBOL(get_kprobe);
393
394static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
395
396/* Return true if 'p' is an aggregator */
397static inline bool kprobe_aggrprobe(struct kprobe *p)
398{
399 return p->pre_handler == aggr_pre_handler;
400}
401
402/* Return true if 'p' is unused */
403static inline bool kprobe_unused(struct kprobe *p)
404{
405 return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
406 list_empty(head: &p->list);
407}
408
409/* Keep all fields in the kprobe consistent. */
410static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
411{
412 memcpy(to: &p->opcode, from: &ap->opcode, len: sizeof(kprobe_opcode_t));
413 memcpy(to: &p->ainsn, from: &ap->ainsn, len: sizeof(struct arch_specific_insn));
414}
415
416#ifdef CONFIG_OPTPROBES
417/* NOTE: This is protected by 'kprobe_mutex'. */
418static bool kprobes_allow_optimization;
419
420/*
421 * Call all 'kprobe::pre_handler' on the list, but ignores its return value.
422 * This must be called from arch-dep optimized caller.
423 */
424void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
425{
426 struct kprobe *kp;
427
428 list_for_each_entry_rcu(kp, &p->list, list) {
429 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
430 set_kprobe_instance(kp);
431 kp->pre_handler(kp, regs);
432 }
433 reset_kprobe_instance();
434 }
435}
436NOKPROBE_SYMBOL(opt_pre_handler);
437
438/* Free optimized instructions and optimized_kprobe */
439static void free_aggr_kprobe(struct kprobe *p)
440{
441 struct optimized_kprobe *op;
442
443 op = container_of(p, struct optimized_kprobe, kp);
444 arch_remove_optimized_kprobe(op);
445 arch_remove_kprobe(p);
446 kfree(objp: op);
447}
448
449/* Return true if the kprobe is ready for optimization. */
450static inline int kprobe_optready(struct kprobe *p)
451{
452 struct optimized_kprobe *op;
453
454 if (kprobe_aggrprobe(p)) {
455 op = container_of(p, struct optimized_kprobe, kp);
456 return arch_prepared_optinsn(optinsn: &op->optinsn);
457 }
458
459 return 0;
460}
461
462/* Return true if the kprobe is disarmed. Note: p must be on hash list */
463bool kprobe_disarmed(struct kprobe *p)
464{
465 struct optimized_kprobe *op;
466
467 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
468 if (!kprobe_aggrprobe(p))
469 return kprobe_disabled(p);
470
471 op = container_of(p, struct optimized_kprobe, kp);
472
473 return kprobe_disabled(p) && list_empty(head: &op->list);
474}
475
476/* Return true if the probe is queued on (un)optimizing lists */
477static bool kprobe_queued(struct kprobe *p)
478{
479 struct optimized_kprobe *op;
480
481 if (kprobe_aggrprobe(p)) {
482 op = container_of(p, struct optimized_kprobe, kp);
483 if (!list_empty(head: &op->list))
484 return true;
485 }
486 return false;
487}
488
489/*
490 * Return an optimized kprobe whose optimizing code replaces
491 * instructions including 'addr' (exclude breakpoint).
492 */
493static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr)
494{
495 int i;
496 struct kprobe *p = NULL;
497 struct optimized_kprobe *op;
498
499 /* Don't check i == 0, since that is a breakpoint case. */
500 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++)
501 p = get_kprobe(addr: addr - i);
502
503 if (p && kprobe_optready(p)) {
504 op = container_of(p, struct optimized_kprobe, kp);
505 if (arch_within_optimized_kprobe(op, addr))
506 return p;
507 }
508
509 return NULL;
510}
511
512/* Optimization staging list, protected by 'kprobe_mutex' */
513static LIST_HEAD(optimizing_list);
514static LIST_HEAD(unoptimizing_list);
515static LIST_HEAD(freeing_list);
516
517static void kprobe_optimizer(struct work_struct *work);
518static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
519#define OPTIMIZE_DELAY 5
520
521/*
522 * Optimize (replace a breakpoint with a jump) kprobes listed on
523 * 'optimizing_list'.
524 */
525static void do_optimize_kprobes(void)
526{
527 lockdep_assert_held(&text_mutex);
528 /*
529 * The optimization/unoptimization refers 'online_cpus' via
530 * stop_machine() and cpu-hotplug modifies the 'online_cpus'.
531 * And same time, 'text_mutex' will be held in cpu-hotplug and here.
532 * This combination can cause a deadlock (cpu-hotplug tries to lock
533 * 'text_mutex' but stop_machine() can not be done because
534 * the 'online_cpus' has been changed)
535 * To avoid this deadlock, caller must have locked cpu-hotplug
536 * for preventing cpu-hotplug outside of 'text_mutex' locking.
537 */
538 lockdep_assert_cpus_held();
539
540 /* Optimization never be done when disarmed */
541 if (kprobes_all_disarmed || !kprobes_allow_optimization ||
542 list_empty(head: &optimizing_list))
543 return;
544
545 arch_optimize_kprobes(oplist: &optimizing_list);
546}
547
548/*
549 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
550 * if need) kprobes listed on 'unoptimizing_list'.
551 */
552static void do_unoptimize_kprobes(void)
553{
554 struct optimized_kprobe *op, *tmp;
555
556 lockdep_assert_held(&text_mutex);
557 /* See comment in do_optimize_kprobes() */
558 lockdep_assert_cpus_held();
559
560 if (!list_empty(head: &unoptimizing_list))
561 arch_unoptimize_kprobes(oplist: &unoptimizing_list, done_list: &freeing_list);
562
563 /* Loop on 'freeing_list' for disarming and removing from kprobe hash list */
564 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
565 /* Switching from detour code to origin */
566 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
567 /* Disarm probes if marked disabled and not gone */
568 if (kprobe_disabled(p: &op->kp) && !kprobe_gone(p: &op->kp))
569 arch_disarm_kprobe(p: &op->kp);
570 if (kprobe_unused(p: &op->kp)) {
571 /*
572 * Remove unused probes from hash list. After waiting
573 * for synchronization, these probes are reclaimed.
574 * (reclaiming is done by do_free_cleaned_kprobes().)
575 */
576 hlist_del_rcu(n: &op->kp.hlist);
577 } else
578 list_del_init(entry: &op->list);
579 }
580}
581
582/* Reclaim all kprobes on the 'freeing_list' */
583static void do_free_cleaned_kprobes(void)
584{
585 struct optimized_kprobe *op, *tmp;
586
587 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
588 list_del_init(entry: &op->list);
589 if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) {
590 /*
591 * This must not happen, but if there is a kprobe
592 * still in use, keep it on kprobes hash list.
593 */
594 continue;
595 }
596 free_aggr_kprobe(p: &op->kp);
597 }
598}
599
600/* Start optimizer after OPTIMIZE_DELAY passed */
601static void kick_kprobe_optimizer(void)
602{
603 schedule_delayed_work(dwork: &optimizing_work, OPTIMIZE_DELAY);
604}
605
606/* Kprobe jump optimizer */
607static void kprobe_optimizer(struct work_struct *work)
608{
609 guard(mutex)(T: &kprobe_mutex);
610
611 scoped_guard(cpus_read_lock) {
612 guard(mutex)(T: &text_mutex);
613
614 /*
615 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
616 * kprobes before waiting for quiesence period.
617 */
618 do_unoptimize_kprobes();
619
620 /*
621 * Step 2: Wait for quiesence period to ensure all potentially
622 * preempted tasks to have normally scheduled. Because optprobe
623 * may modify multiple instructions, there is a chance that Nth
624 * instruction is preempted. In that case, such tasks can return
625 * to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
626 * Note that on non-preemptive kernel, this is transparently converted
627 * to synchronoze_sched() to wait for all interrupts to have completed.
628 */
629 synchronize_rcu_tasks();
630
631 /* Step 3: Optimize kprobes after quiesence period */
632 do_optimize_kprobes();
633
634 /* Step 4: Free cleaned kprobes after quiesence period */
635 do_free_cleaned_kprobes();
636 }
637
638 /* Step 5: Kick optimizer again if needed */
639 if (!list_empty(head: &optimizing_list) || !list_empty(head: &unoptimizing_list))
640 kick_kprobe_optimizer();
641}
642
643static void wait_for_kprobe_optimizer_locked(void)
644{
645 lockdep_assert_held(&kprobe_mutex);
646
647 while (!list_empty(head: &optimizing_list) || !list_empty(head: &unoptimizing_list)) {
648 mutex_unlock(lock: &kprobe_mutex);
649
650 /* This will also make 'optimizing_work' execute immmediately */
651 flush_delayed_work(dwork: &optimizing_work);
652 /* 'optimizing_work' might not have been queued yet, relax */
653 cpu_relax();
654
655 mutex_lock(lock: &kprobe_mutex);
656 }
657}
658
659/* Wait for completing optimization and unoptimization */
660void wait_for_kprobe_optimizer(void)
661{
662 guard(mutex)(T: &kprobe_mutex);
663
664 wait_for_kprobe_optimizer_locked();
665}
666
667bool optprobe_queued_unopt(struct optimized_kprobe *op)
668{
669 struct optimized_kprobe *_op;
670
671 list_for_each_entry(_op, &unoptimizing_list, list) {
672 if (op == _op)
673 return true;
674 }
675
676 return false;
677}
678
679/* Optimize kprobe if p is ready to be optimized */
680static void optimize_kprobe(struct kprobe *p)
681{
682 struct optimized_kprobe *op;
683
684 /* Check if the kprobe is disabled or not ready for optimization. */
685 if (!kprobe_optready(p) || !kprobes_allow_optimization ||
686 (kprobe_disabled(p) || kprobes_all_disarmed))
687 return;
688
689 /* kprobes with 'post_handler' can not be optimized */
690 if (p->post_handler)
691 return;
692
693 op = container_of(p, struct optimized_kprobe, kp);
694
695 /* Check there is no other kprobes at the optimized instructions */
696 if (arch_check_optimized_kprobe(op) < 0)
697 return;
698
699 /* Check if it is already optimized. */
700 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) {
701 if (optprobe_queued_unopt(op)) {
702 /* This is under unoptimizing. Just dequeue the probe */
703 list_del_init(entry: &op->list);
704 }
705 return;
706 }
707 op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
708
709 /*
710 * On the 'unoptimizing_list' and 'optimizing_list',
711 * 'op' must have OPTIMIZED flag
712 */
713 if (WARN_ON_ONCE(!list_empty(&op->list)))
714 return;
715
716 list_add(new: &op->list, head: &optimizing_list);
717 kick_kprobe_optimizer();
718}
719
720/* Short cut to direct unoptimizing */
721static void force_unoptimize_kprobe(struct optimized_kprobe *op)
722{
723 lockdep_assert_cpus_held();
724 arch_unoptimize_kprobe(op);
725 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
726}
727
728/* Unoptimize a kprobe if p is optimized */
729static void unoptimize_kprobe(struct kprobe *p, bool force)
730{
731 struct optimized_kprobe *op;
732
733 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
734 return; /* This is not an optprobe nor optimized */
735
736 op = container_of(p, struct optimized_kprobe, kp);
737 if (!kprobe_optimized(p))
738 return;
739
740 if (!list_empty(head: &op->list)) {
741 if (optprobe_queued_unopt(op)) {
742 /* Queued in unoptimizing queue */
743 if (force) {
744 /*
745 * Forcibly unoptimize the kprobe here, and queue it
746 * in the freeing list for release afterwards.
747 */
748 force_unoptimize_kprobe(op);
749 list_move(list: &op->list, head: &freeing_list);
750 }
751 } else {
752 /* Dequeue from the optimizing queue */
753 list_del_init(entry: &op->list);
754 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
755 }
756 return;
757 }
758
759 /* Optimized kprobe case */
760 if (force) {
761 /* Forcibly update the code: this is a special case */
762 force_unoptimize_kprobe(op);
763 } else {
764 list_add(new: &op->list, head: &unoptimizing_list);
765 kick_kprobe_optimizer();
766 }
767}
768
769/* Cancel unoptimizing for reusing */
770static int reuse_unused_kprobe(struct kprobe *ap)
771{
772 struct optimized_kprobe *op;
773
774 /*
775 * Unused kprobe MUST be on the way of delayed unoptimizing (means
776 * there is still a relative jump) and disabled.
777 */
778 op = container_of(ap, struct optimized_kprobe, kp);
779 WARN_ON_ONCE(list_empty(&op->list));
780 /* Enable the probe again */
781 ap->flags &= ~KPROBE_FLAG_DISABLED;
782 /* Optimize it again. (remove from 'op->list') */
783 if (!kprobe_optready(p: ap))
784 return -EINVAL;
785
786 optimize_kprobe(p: ap);
787 return 0;
788}
789
790/* Remove optimized instructions */
791static void kill_optimized_kprobe(struct kprobe *p)
792{
793 struct optimized_kprobe *op;
794
795 op = container_of(p, struct optimized_kprobe, kp);
796 if (!list_empty(head: &op->list))
797 /* Dequeue from the (un)optimization queue */
798 list_del_init(entry: &op->list);
799 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
800
801 if (kprobe_unused(p)) {
802 /*
803 * Unused kprobe is on unoptimizing or freeing list. We move it
804 * to freeing_list and let the kprobe_optimizer() remove it from
805 * the kprobe hash list and free it.
806 */
807 if (optprobe_queued_unopt(op))
808 list_move(list: &op->list, head: &freeing_list);
809 }
810
811 /* Don't touch the code, because it is already freed. */
812 arch_remove_optimized_kprobe(op);
813}
814
815static inline
816void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
817{
818 if (!kprobe_ftrace(p))
819 arch_prepare_optimized_kprobe(op, orig: p);
820}
821
822/* Try to prepare optimized instructions */
823static void prepare_optimized_kprobe(struct kprobe *p)
824{
825 struct optimized_kprobe *op;
826
827 op = container_of(p, struct optimized_kprobe, kp);
828 __prepare_optimized_kprobe(op, p);
829}
830
831/* Allocate new optimized_kprobe and try to prepare optimized instructions. */
832static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
833{
834 struct optimized_kprobe *op;
835
836 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
837 if (!op)
838 return NULL;
839
840 INIT_LIST_HEAD(list: &op->list);
841 op->kp.addr = p->addr;
842 __prepare_optimized_kprobe(op, p);
843
844 return &op->kp;
845}
846
847static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
848
849/*
850 * Prepare an optimized_kprobe and optimize it.
851 * NOTE: 'p' must be a normal registered kprobe.
852 */
853static void try_to_optimize_kprobe(struct kprobe *p)
854{
855 struct kprobe *ap;
856 struct optimized_kprobe *op;
857
858 /* Impossible to optimize ftrace-based kprobe. */
859 if (kprobe_ftrace(p))
860 return;
861
862 /* For preparing optimization, jump_label_text_reserved() is called. */
863 guard(cpus_read_lock)();
864 guard(jump_label_lock)();
865 guard(mutex)(T: &text_mutex);
866
867 ap = alloc_aggr_kprobe(p);
868 if (!ap)
869 return;
870
871 op = container_of(ap, struct optimized_kprobe, kp);
872 if (!arch_prepared_optinsn(optinsn: &op->optinsn)) {
873 /* If failed to setup optimizing, fallback to kprobe. */
874 arch_remove_optimized_kprobe(op);
875 kfree(objp: op);
876 return;
877 }
878
879 init_aggr_kprobe(ap, p);
880 optimize_kprobe(p: ap); /* This just kicks optimizer thread. */
881}
882
883static void optimize_all_kprobes(void)
884{
885 struct hlist_head *head;
886 struct kprobe *p;
887 unsigned int i;
888
889 guard(mutex)(T: &kprobe_mutex);
890 /* If optimization is already allowed, just return. */
891 if (kprobes_allow_optimization)
892 return;
893
894 cpus_read_lock();
895 kprobes_allow_optimization = true;
896 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
897 head = &kprobe_table[i];
898 hlist_for_each_entry(p, head, hlist)
899 if (!kprobe_disabled(p))
900 optimize_kprobe(p);
901 }
902 cpus_read_unlock();
903 pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n");
904}
905
906#ifdef CONFIG_SYSCTL
907static void unoptimize_all_kprobes(void)
908{
909 struct hlist_head *head;
910 struct kprobe *p;
911 unsigned int i;
912
913 guard(mutex)(T: &kprobe_mutex);
914 /* If optimization is already prohibited, just return. */
915 if (!kprobes_allow_optimization)
916 return;
917
918 cpus_read_lock();
919 kprobes_allow_optimization = false;
920 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
921 head = &kprobe_table[i];
922 hlist_for_each_entry(p, head, hlist) {
923 if (!kprobe_disabled(p))
924 unoptimize_kprobe(p, force: false);
925 }
926 }
927 cpus_read_unlock();
928 /* Wait for unoptimizing completion. */
929 wait_for_kprobe_optimizer_locked();
930 pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n");
931}
932
933static DEFINE_MUTEX(kprobe_sysctl_mutex);
934static int sysctl_kprobes_optimization;
935static int proc_kprobes_optimization_handler(const struct ctl_table *table,
936 int write, void *buffer,
937 size_t *length, loff_t *ppos)
938{
939 int ret;
940
941 guard(mutex)(T: &kprobe_sysctl_mutex);
942 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
943 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
944
945 if (sysctl_kprobes_optimization)
946 optimize_all_kprobes();
947 else
948 unoptimize_all_kprobes();
949
950 return ret;
951}
952
953static const struct ctl_table kprobe_sysctls[] = {
954 {
955 .procname = "kprobes-optimization",
956 .data = &sysctl_kprobes_optimization,
957 .maxlen = sizeof(int),
958 .mode = 0644,
959 .proc_handler = proc_kprobes_optimization_handler,
960 .extra1 = SYSCTL_ZERO,
961 .extra2 = SYSCTL_ONE,
962 },
963};
964
965static void __init kprobe_sysctls_init(void)
966{
967 register_sysctl_init("debug", kprobe_sysctls);
968}
969#endif /* CONFIG_SYSCTL */
970
971/* Put a breakpoint for a probe. */
972static void __arm_kprobe(struct kprobe *p)
973{
974 struct kprobe *_p;
975
976 lockdep_assert_held(&text_mutex);
977
978 /* Find the overlapping optimized kprobes. */
979 _p = get_optimized_kprobe(addr: p->addr);
980 if (unlikely(_p))
981 /* Fallback to unoptimized kprobe */
982 unoptimize_kprobe(p: _p, force: true);
983
984 arch_arm_kprobe(p);
985 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
986}
987
988/* Remove the breakpoint of a probe. */
989static void __disarm_kprobe(struct kprobe *p, bool reopt)
990{
991 struct kprobe *_p;
992
993 lockdep_assert_held(&text_mutex);
994
995 /* Try to unoptimize */
996 unoptimize_kprobe(p, force: kprobes_all_disarmed);
997
998 if (!kprobe_queued(p)) {
999 arch_disarm_kprobe(p);
1000 /* If another kprobe was blocked, re-optimize it. */
1001 _p = get_optimized_kprobe(addr: p->addr);
1002 if (unlikely(_p) && reopt)
1003 optimize_kprobe(p: _p);
1004 }
1005 /*
1006 * TODO: Since unoptimization and real disarming will be done by
1007 * the worker thread, we can not check whether another probe are
1008 * unoptimized because of this probe here. It should be re-optimized
1009 * by the worker thread.
1010 */
1011}
1012
1013#else /* !CONFIG_OPTPROBES */
1014
1015#define optimize_kprobe(p) do {} while (0)
1016#define unoptimize_kprobe(p, f) do {} while (0)
1017#define kill_optimized_kprobe(p) do {} while (0)
1018#define prepare_optimized_kprobe(p) do {} while (0)
1019#define try_to_optimize_kprobe(p) do {} while (0)
1020#define __arm_kprobe(p) arch_arm_kprobe(p)
1021#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
1022#define kprobe_disarmed(p) kprobe_disabled(p)
1023#define wait_for_kprobe_optimizer_locked() \
1024 lockdep_assert_held(&kprobe_mutex)
1025
1026static int reuse_unused_kprobe(struct kprobe *ap)
1027{
1028 /*
1029 * If the optimized kprobe is NOT supported, the aggr kprobe is
1030 * released at the same time that the last aggregated kprobe is
1031 * unregistered.
1032 * Thus there should be no chance to reuse unused kprobe.
1033 */
1034 WARN_ON_ONCE(1);
1035 return -EINVAL;
1036}
1037
1038static void free_aggr_kprobe(struct kprobe *p)
1039{
1040 arch_remove_kprobe(p);
1041 kfree(p);
1042}
1043
1044static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
1045{
1046 return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
1047}
1048#endif /* CONFIG_OPTPROBES */
1049
1050#ifdef CONFIG_KPROBES_ON_FTRACE
1051static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
1052 .func = kprobe_ftrace_handler,
1053 .flags = FTRACE_OPS_FL_SAVE_REGS,
1054};
1055
1056static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
1057 .func = kprobe_ftrace_handler,
1058 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
1059};
1060
1061static int kprobe_ipmodify_enabled;
1062static int kprobe_ftrace_enabled;
1063bool kprobe_ftrace_disabled;
1064
1065static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1066 int *cnt)
1067{
1068 int ret;
1069
1070 lockdep_assert_held(&kprobe_mutex);
1071
1072 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
1073 if (WARN_ONCE(ret < 0, "Failed to arm kprobe-ftrace at %pS (error %d)\n", p->addr, ret))
1074 return ret;
1075
1076 if (*cnt == 0) {
1077 ret = register_ftrace_function(ops);
1078 if (WARN(ret < 0, "Failed to register kprobe-ftrace (error %d)\n", ret)) {
1079 /*
1080 * At this point, sinec ops is not registered, we should be sefe from
1081 * registering empty filter.
1082 */
1083 ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1084 return ret;
1085 }
1086 }
1087
1088 (*cnt)++;
1089 return ret;
1090}
1091
1092static int arm_kprobe_ftrace(struct kprobe *p)
1093{
1094 bool ipmodify = (p->post_handler != NULL);
1095
1096 return __arm_kprobe_ftrace(p,
1097 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1098 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1099}
1100
1101static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1102 int *cnt)
1103{
1104 int ret;
1105
1106 lockdep_assert_held(&kprobe_mutex);
1107
1108 if (*cnt == 1) {
1109 ret = unregister_ftrace_function(ops);
1110 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret))
1111 return ret;
1112 }
1113
1114 (*cnt)--;
1115
1116 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1117 WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n",
1118 p->addr, ret);
1119 return ret;
1120}
1121
1122static int disarm_kprobe_ftrace(struct kprobe *p)
1123{
1124 bool ipmodify = (p->post_handler != NULL);
1125
1126 return __disarm_kprobe_ftrace(p,
1127 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1128 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1129}
1130
1131void kprobe_ftrace_kill(void)
1132{
1133 kprobe_ftrace_disabled = true;
1134}
1135#else /* !CONFIG_KPROBES_ON_FTRACE */
1136static inline int arm_kprobe_ftrace(struct kprobe *p)
1137{
1138 return -ENODEV;
1139}
1140
1141static inline int disarm_kprobe_ftrace(struct kprobe *p)
1142{
1143 return -ENODEV;
1144}
1145#endif
1146
1147static int prepare_kprobe(struct kprobe *p)
1148{
1149 /* Must ensure p->addr is really on ftrace */
1150 if (kprobe_ftrace(p))
1151 return arch_prepare_kprobe_ftrace(p);
1152
1153 return arch_prepare_kprobe(p);
1154}
1155
1156static int arm_kprobe(struct kprobe *kp)
1157{
1158 if (unlikely(kprobe_ftrace(kp)))
1159 return arm_kprobe_ftrace(p: kp);
1160
1161 guard(cpus_read_lock)();
1162 guard(mutex)(T: &text_mutex);
1163 __arm_kprobe(p: kp);
1164 return 0;
1165}
1166
1167static int disarm_kprobe(struct kprobe *kp, bool reopt)
1168{
1169 if (unlikely(kprobe_ftrace(kp)))
1170 return disarm_kprobe_ftrace(p: kp);
1171
1172 guard(cpus_read_lock)();
1173 guard(mutex)(T: &text_mutex);
1174 __disarm_kprobe(p: kp, reopt);
1175 return 0;
1176}
1177
1178/*
1179 * Aggregate handlers for multiple kprobes support - these handlers
1180 * take care of invoking the individual kprobe handlers on p->list
1181 */
1182static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
1183{
1184 struct kprobe *kp;
1185
1186 list_for_each_entry_rcu(kp, &p->list, list) {
1187 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1188 set_kprobe_instance(kp);
1189 if (kp->pre_handler(kp, regs))
1190 return 1;
1191 }
1192 reset_kprobe_instance();
1193 }
1194 return 0;
1195}
1196NOKPROBE_SYMBOL(aggr_pre_handler);
1197
1198static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1199 unsigned long flags)
1200{
1201 struct kprobe *kp;
1202
1203 list_for_each_entry_rcu(kp, &p->list, list) {
1204 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1205 set_kprobe_instance(kp);
1206 kp->post_handler(kp, regs, flags);
1207 reset_kprobe_instance();
1208 }
1209 }
1210}
1211NOKPROBE_SYMBOL(aggr_post_handler);
1212
1213/* Walks the list and increments 'nmissed' if 'p' has child probes. */
1214void kprobes_inc_nmissed_count(struct kprobe *p)
1215{
1216 struct kprobe *kp;
1217
1218 if (!kprobe_aggrprobe(p)) {
1219 p->nmissed++;
1220 } else {
1221 list_for_each_entry_rcu(kp, &p->list, list)
1222 kp->nmissed++;
1223 }
1224}
1225NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1226
1227static struct kprobe kprobe_busy = {
1228 .addr = (void *) get_kprobe,
1229};
1230
1231void kprobe_busy_begin(void)
1232{
1233 struct kprobe_ctlblk *kcb;
1234
1235 preempt_disable();
1236 __this_cpu_write(current_kprobe, &kprobe_busy);
1237 kcb = get_kprobe_ctlblk();
1238 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1239}
1240
1241void kprobe_busy_end(void)
1242{
1243 __this_cpu_write(current_kprobe, NULL);
1244 preempt_enable();
1245}
1246
1247/* Add the new probe to 'ap->list'. */
1248static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1249{
1250 if (p->post_handler)
1251 unoptimize_kprobe(p: ap, force: true); /* Fall back to normal kprobe */
1252
1253 list_add_rcu(new: &p->list, head: &ap->list);
1254 if (p->post_handler && !ap->post_handler)
1255 ap->post_handler = aggr_post_handler;
1256
1257 return 0;
1258}
1259
1260/*
1261 * Fill in the required fields of the aggregator kprobe. Replace the
1262 * earlier kprobe in the hlist with the aggregator kprobe.
1263 */
1264static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1265{
1266 /* Copy the insn slot of 'p' to 'ap'. */
1267 copy_kprobe(ap: p, p: ap);
1268 flush_insn_slot(ap);
1269 ap->addr = p->addr;
1270 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1271 ap->pre_handler = aggr_pre_handler;
1272 /* We don't care the kprobe which has gone. */
1273 if (p->post_handler && !kprobe_gone(p))
1274 ap->post_handler = aggr_post_handler;
1275
1276 INIT_LIST_HEAD(list: &ap->list);
1277 INIT_HLIST_NODE(h: &ap->hlist);
1278
1279 list_add_rcu(new: &p->list, head: &ap->list);
1280 hlist_replace_rcu(old: &p->hlist, new: &ap->hlist);
1281}
1282
1283/*
1284 * This registers the second or subsequent kprobe at the same address.
1285 */
1286static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
1287{
1288 int ret = 0;
1289 struct kprobe *ap = orig_p;
1290
1291 scoped_guard(cpus_read_lock) {
1292 /* For preparing optimization, jump_label_text_reserved() is called */
1293 guard(jump_label_lock)();
1294 guard(mutex)(T: &text_mutex);
1295
1296 if (!kprobe_aggrprobe(p: orig_p)) {
1297 /* If 'orig_p' is not an 'aggr_kprobe', create new one. */
1298 ap = alloc_aggr_kprobe(p: orig_p);
1299 if (!ap)
1300 return -ENOMEM;
1301 init_aggr_kprobe(ap, p: orig_p);
1302 } else if (kprobe_unused(p: ap)) {
1303 /* This probe is going to die. Rescue it */
1304 ret = reuse_unused_kprobe(ap);
1305 if (ret)
1306 return ret;
1307 }
1308
1309 if (kprobe_gone(p: ap)) {
1310 /*
1311 * Attempting to insert new probe at the same location that
1312 * had a probe in the module vaddr area which already
1313 * freed. So, the instruction slot has already been
1314 * released. We need a new slot for the new probe.
1315 */
1316 ret = arch_prepare_kprobe(p: ap);
1317 if (ret)
1318 /*
1319 * Even if fail to allocate new slot, don't need to
1320 * free the 'ap'. It will be used next time, or
1321 * freed by unregister_kprobe().
1322 */
1323 return ret;
1324
1325 /* Prepare optimized instructions if possible. */
1326 prepare_optimized_kprobe(p: ap);
1327
1328 /*
1329 * Clear gone flag to prevent allocating new slot again, and
1330 * set disabled flag because it is not armed yet.
1331 */
1332 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1333 | KPROBE_FLAG_DISABLED;
1334 }
1335
1336 /* Copy the insn slot of 'p' to 'ap'. */
1337 copy_kprobe(ap, p);
1338 ret = add_new_kprobe(ap, p);
1339 }
1340
1341 if (ret == 0 && kprobe_disabled(p: ap) && !kprobe_disabled(p)) {
1342 ap->flags &= ~KPROBE_FLAG_DISABLED;
1343 if (!kprobes_all_disarmed) {
1344 /* Arm the breakpoint again. */
1345 ret = arm_kprobe(kp: ap);
1346 if (ret) {
1347 ap->flags |= KPROBE_FLAG_DISABLED;
1348 list_del_rcu(entry: &p->list);
1349 synchronize_rcu();
1350 }
1351 }
1352 }
1353 return ret;
1354}
1355
1356bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1357{
1358 /* The '__kprobes' functions and entry code must not be probed. */
1359 return addr >= (unsigned long)__kprobes_text_start &&
1360 addr < (unsigned long)__kprobes_text_end;
1361}
1362
1363static bool __within_kprobe_blacklist(unsigned long addr)
1364{
1365 struct kprobe_blacklist_entry *ent;
1366
1367 if (arch_within_kprobe_blacklist(addr))
1368 return true;
1369 /*
1370 * If 'kprobe_blacklist' is defined, check the address and
1371 * reject any probe registration in the prohibited area.
1372 */
1373 list_for_each_entry(ent, &kprobe_blacklist, list) {
1374 if (addr >= ent->start_addr && addr < ent->end_addr)
1375 return true;
1376 }
1377 return false;
1378}
1379
1380bool within_kprobe_blacklist(unsigned long addr)
1381{
1382 char symname[KSYM_NAME_LEN], *p;
1383
1384 if (__within_kprobe_blacklist(addr))
1385 return true;
1386
1387 /* Check if the address is on a suffixed-symbol */
1388 if (!lookup_symbol_name(addr, symname)) {
1389 p = strchr(symname, '.');
1390 if (!p)
1391 return false;
1392 *p = '\0';
1393 addr = (unsigned long)kprobe_lookup_name(name: symname, unused: 0);
1394 if (addr)
1395 return __within_kprobe_blacklist(addr);
1396 }
1397 return false;
1398}
1399
1400/*
1401 * arch_adjust_kprobe_addr - adjust the address
1402 * @addr: symbol base address
1403 * @offset: offset within the symbol
1404 * @on_func_entry: was this @addr+@offset on the function entry
1405 *
1406 * Typically returns @addr + @offset, except for special cases where the
1407 * function might be prefixed by a CFI landing pad, in that case any offset
1408 * inside the landing pad is mapped to the first 'real' instruction of the
1409 * symbol.
1410 *
1411 * Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C
1412 * instruction at +0.
1413 */
1414kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr,
1415 unsigned long offset,
1416 bool *on_func_entry)
1417{
1418 *on_func_entry = !offset;
1419 return (kprobe_opcode_t *)(addr + offset);
1420}
1421
1422/*
1423 * If 'symbol_name' is specified, look it up and add the 'offset'
1424 * to it. This way, we can specify a relative address to a symbol.
1425 * This returns encoded errors if it fails to look up symbol or invalid
1426 * combination of parameters.
1427 */
1428static kprobe_opcode_t *
1429_kprobe_addr(kprobe_opcode_t *addr, const char *symbol_name,
1430 unsigned long offset, bool *on_func_entry)
1431{
1432 if ((symbol_name && addr) || (!symbol_name && !addr))
1433 return ERR_PTR(error: -EINVAL);
1434
1435 if (symbol_name) {
1436 /*
1437 * Input: @sym + @offset
1438 * Output: @addr + @offset
1439 *
1440 * NOTE: kprobe_lookup_name() does *NOT* fold the offset
1441 * argument into it's output!
1442 */
1443 addr = kprobe_lookup_name(name: symbol_name, unused: offset);
1444 if (!addr)
1445 return ERR_PTR(error: -ENOENT);
1446 }
1447
1448 /*
1449 * So here we have @addr + @offset, displace it into a new
1450 * @addr' + @offset' where @addr' is the symbol start address.
1451 */
1452 addr = (void *)addr + offset;
1453 if (!kallsyms_lookup_size_offset(addr: (unsigned long)addr, NULL, offset: &offset))
1454 return ERR_PTR(error: -ENOENT);
1455 addr = (void *)addr - offset;
1456
1457 /*
1458 * Then ask the architecture to re-combine them, taking care of
1459 * magical function entry details while telling us if this was indeed
1460 * at the start of the function.
1461 */
1462 addr = arch_adjust_kprobe_addr(addr: (unsigned long)addr, offset, on_func_entry);
1463 if (!addr)
1464 return ERR_PTR(error: -EINVAL);
1465
1466 return addr;
1467}
1468
1469static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1470{
1471 bool on_func_entry;
1472
1473 return _kprobe_addr(addr: p->addr, symbol_name: p->symbol_name, offset: p->offset, on_func_entry: &on_func_entry);
1474}
1475
1476/*
1477 * Check the 'p' is valid and return the aggregator kprobe
1478 * at the same address.
1479 */
1480static struct kprobe *__get_valid_kprobe(struct kprobe *p)
1481{
1482 struct kprobe *ap, *list_p;
1483
1484 lockdep_assert_held(&kprobe_mutex);
1485
1486 ap = get_kprobe(addr: p->addr);
1487 if (unlikely(!ap))
1488 return NULL;
1489
1490 if (p == ap)
1491 return ap;
1492
1493 list_for_each_entry(list_p, &ap->list, list)
1494 if (list_p == p)
1495 /* kprobe p is a valid probe */
1496 return ap;
1497
1498 return NULL;
1499}
1500
1501/*
1502 * Warn and return error if the kprobe is being re-registered since
1503 * there must be a software bug.
1504 */
1505static inline int warn_kprobe_rereg(struct kprobe *p)
1506{
1507 guard(mutex)(T: &kprobe_mutex);
1508
1509 if (WARN_ON_ONCE(__get_valid_kprobe(p)))
1510 return -EINVAL;
1511
1512 return 0;
1513}
1514
1515static int check_ftrace_location(struct kprobe *p)
1516{
1517 unsigned long addr = (unsigned long)p->addr;
1518
1519 if (ftrace_location(ip: addr) == addr) {
1520#ifdef CONFIG_KPROBES_ON_FTRACE
1521 p->flags |= KPROBE_FLAG_FTRACE;
1522#else
1523 return -EINVAL;
1524#endif
1525 }
1526 return 0;
1527}
1528
1529static bool is_cfi_preamble_symbol(unsigned long addr)
1530{
1531 char symbuf[KSYM_NAME_LEN];
1532
1533 if (lookup_symbol_name(addr, symname: symbuf))
1534 return false;
1535
1536 return str_has_prefix(str: symbuf, prefix: "__cfi_") ||
1537 str_has_prefix(str: symbuf, prefix: "__pfx_");
1538}
1539
1540static int check_kprobe_address_safe(struct kprobe *p,
1541 struct module **probed_mod)
1542{
1543 int ret;
1544
1545 ret = check_ftrace_location(p);
1546 if (ret)
1547 return ret;
1548
1549 guard(jump_label_lock)();
1550
1551 /* Ensure the address is in a text area, and find a module if exists. */
1552 *probed_mod = NULL;
1553 if (!core_kernel_text(addr: (unsigned long) p->addr)) {
1554 guard(rcu)();
1555 *probed_mod = __module_text_address(addr: (unsigned long) p->addr);
1556 if (!(*probed_mod))
1557 return -EINVAL;
1558
1559 /*
1560 * We must hold a refcount of the probed module while updating
1561 * its code to prohibit unexpected unloading.
1562 */
1563 if (unlikely(!try_module_get(*probed_mod)))
1564 return -ENOENT;
1565 }
1566 /* Ensure it is not in reserved area. */
1567 if (in_gate_area_no_mm(addr: (unsigned long) p->addr) ||
1568 within_kprobe_blacklist(addr: (unsigned long) p->addr) ||
1569 jump_label_text_reserved(start: p->addr, end: p->addr) ||
1570 static_call_text_reserved(start: p->addr, end: p->addr) ||
1571 find_bug(bugaddr: (unsigned long)p->addr) ||
1572 is_cfi_preamble_symbol(addr: (unsigned long)p->addr)) {
1573 module_put(module: *probed_mod);
1574 return -EINVAL;
1575 }
1576
1577 /* Get module refcount and reject __init functions for loaded modules. */
1578 if (IS_ENABLED(CONFIG_MODULES) && *probed_mod) {
1579 /*
1580 * If the module freed '.init.text', we couldn't insert
1581 * kprobes in there.
1582 */
1583 if (within_module_init(addr: (unsigned long)p->addr, mod: *probed_mod) &&
1584 !module_is_coming(mod: *probed_mod)) {
1585 module_put(module: *probed_mod);
1586 return -ENOENT;
1587 }
1588 }
1589
1590 return 0;
1591}
1592
1593static int __register_kprobe(struct kprobe *p)
1594{
1595 int ret;
1596 struct kprobe *old_p;
1597
1598 guard(mutex)(T: &kprobe_mutex);
1599
1600 old_p = get_kprobe(addr: p->addr);
1601 if (old_p)
1602 /* Since this may unoptimize 'old_p', locking 'text_mutex'. */
1603 return register_aggr_kprobe(orig_p: old_p, p);
1604
1605 scoped_guard(cpus_read_lock) {
1606 /* Prevent text modification */
1607 guard(mutex)(T: &text_mutex);
1608 ret = prepare_kprobe(p);
1609 if (ret)
1610 return ret;
1611 }
1612
1613 INIT_HLIST_NODE(h: &p->hlist);
1614 hlist_add_head_rcu(n: &p->hlist,
1615 h: &kprobe_table[hash_ptr(ptr: p->addr, KPROBE_HASH_BITS)]);
1616
1617 if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1618 ret = arm_kprobe(kp: p);
1619 if (ret) {
1620 hlist_del_rcu(n: &p->hlist);
1621 synchronize_rcu();
1622 }
1623 }
1624
1625 /* Try to optimize kprobe */
1626 try_to_optimize_kprobe(p);
1627 return 0;
1628}
1629
1630int register_kprobe(struct kprobe *p)
1631{
1632 int ret;
1633 struct module *probed_mod;
1634 kprobe_opcode_t *addr;
1635 bool on_func_entry;
1636
1637 /* Canonicalize probe address from symbol */
1638 addr = _kprobe_addr(addr: p->addr, symbol_name: p->symbol_name, offset: p->offset, on_func_entry: &on_func_entry);
1639 if (IS_ERR(ptr: addr))
1640 return PTR_ERR(ptr: addr);
1641 p->addr = addr;
1642
1643 ret = warn_kprobe_rereg(p);
1644 if (ret)
1645 return ret;
1646
1647 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1648 p->flags &= KPROBE_FLAG_DISABLED;
1649 if (on_func_entry)
1650 p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY;
1651 p->nmissed = 0;
1652 INIT_LIST_HEAD(list: &p->list);
1653
1654 ret = check_kprobe_address_safe(p, probed_mod: &probed_mod);
1655 if (ret)
1656 return ret;
1657
1658 ret = __register_kprobe(p);
1659
1660 if (probed_mod)
1661 module_put(module: probed_mod);
1662
1663 return ret;
1664}
1665EXPORT_SYMBOL_GPL(register_kprobe);
1666
1667/* Check if all probes on the 'ap' are disabled. */
1668static bool aggr_kprobe_disabled(struct kprobe *ap)
1669{
1670 struct kprobe *kp;
1671
1672 lockdep_assert_held(&kprobe_mutex);
1673
1674 list_for_each_entry(kp, &ap->list, list)
1675 if (!kprobe_disabled(p: kp))
1676 /*
1677 * Since there is an active probe on the list,
1678 * we can't disable this 'ap'.
1679 */
1680 return false;
1681
1682 return true;
1683}
1684
1685static struct kprobe *__disable_kprobe(struct kprobe *p)
1686{
1687 struct kprobe *orig_p;
1688 int ret;
1689
1690 lockdep_assert_held(&kprobe_mutex);
1691
1692 /* Get an original kprobe for return */
1693 orig_p = __get_valid_kprobe(p);
1694 if (unlikely(orig_p == NULL))
1695 return ERR_PTR(error: -EINVAL);
1696
1697 if (kprobe_disabled(p))
1698 return orig_p;
1699
1700 /* Disable probe if it is a child probe */
1701 if (p != orig_p)
1702 p->flags |= KPROBE_FLAG_DISABLED;
1703
1704 /* Try to disarm and disable this/parent probe */
1705 if (p == orig_p || aggr_kprobe_disabled(ap: orig_p)) {
1706 /*
1707 * Don't be lazy here. Even if 'kprobes_all_disarmed'
1708 * is false, 'orig_p' might not have been armed yet.
1709 * Note arm_all_kprobes() __tries__ to arm all kprobes
1710 * on the best effort basis.
1711 */
1712 if (!kprobes_all_disarmed && !kprobe_disabled(p: orig_p)) {
1713 ret = disarm_kprobe(kp: orig_p, reopt: true);
1714 if (ret) {
1715 p->flags &= ~KPROBE_FLAG_DISABLED;
1716 return ERR_PTR(error: ret);
1717 }
1718 }
1719 orig_p->flags |= KPROBE_FLAG_DISABLED;
1720 }
1721
1722 return orig_p;
1723}
1724
1725/*
1726 * Unregister a kprobe without a scheduler synchronization.
1727 */
1728static int __unregister_kprobe_top(struct kprobe *p)
1729{
1730 struct kprobe *ap, *list_p;
1731
1732 /* Disable kprobe. This will disarm it if needed. */
1733 ap = __disable_kprobe(p);
1734 if (IS_ERR(ptr: ap))
1735 return PTR_ERR(ptr: ap);
1736
1737 WARN_ON(ap != p && !kprobe_aggrprobe(ap));
1738
1739 /*
1740 * If the probe is an independent(and non-optimized) kprobe
1741 * (not an aggrprobe), the last kprobe on the aggrprobe, or
1742 * kprobe is already disarmed, just remove from the hash list.
1743 */
1744 if (ap == p ||
1745 (list_is_singular(head: &ap->list) && kprobe_disarmed(p: ap))) {
1746 /*
1747 * !disarmed could be happen if the probe is under delayed
1748 * unoptimizing.
1749 */
1750 hlist_del_rcu(n: &ap->hlist);
1751 return 0;
1752 }
1753
1754 /* If disabling probe has special handlers, update aggrprobe */
1755 if (p->post_handler && !kprobe_gone(p)) {
1756 list_for_each_entry(list_p, &ap->list, list) {
1757 if ((list_p != p) && (list_p->post_handler))
1758 break;
1759 }
1760 /* No other probe has post_handler */
1761 if (list_entry_is_head(list_p, &ap->list, list)) {
1762 /*
1763 * For the kprobe-on-ftrace case, we keep the
1764 * post_handler setting to identify this aggrprobe
1765 * armed with kprobe_ipmodify_ops.
1766 */
1767 if (!kprobe_ftrace(p: ap))
1768 ap->post_handler = NULL;
1769 }
1770 }
1771
1772 /*
1773 * Remove from the aggrprobe: this path will do nothing in
1774 * __unregister_kprobe_bottom().
1775 */
1776 list_del_rcu(entry: &p->list);
1777 if (!kprobe_disabled(p: ap) && !kprobes_all_disarmed)
1778 /*
1779 * Try to optimize this probe again, because post
1780 * handler may have been changed.
1781 */
1782 optimize_kprobe(p: ap);
1783 return 0;
1784
1785}
1786
1787static void __unregister_kprobe_bottom(struct kprobe *p)
1788{
1789 struct kprobe *ap;
1790
1791 if (list_empty(head: &p->list))
1792 /* This is an independent kprobe */
1793 arch_remove_kprobe(p);
1794 else if (list_is_singular(head: &p->list)) {
1795 /* This is the last child of an aggrprobe */
1796 ap = list_entry(p->list.next, struct kprobe, list);
1797 list_del(entry: &p->list);
1798 free_aggr_kprobe(p: ap);
1799 }
1800 /* Otherwise, do nothing. */
1801}
1802
1803int register_kprobes(struct kprobe **kps, int num)
1804{
1805 int i, ret = 0;
1806
1807 if (num <= 0)
1808 return -EINVAL;
1809 for (i = 0; i < num; i++) {
1810 ret = register_kprobe(kps[i]);
1811 if (ret < 0) {
1812 if (i > 0)
1813 unregister_kprobes(kps, num: i);
1814 break;
1815 }
1816 }
1817 return ret;
1818}
1819EXPORT_SYMBOL_GPL(register_kprobes);
1820
1821void unregister_kprobe(struct kprobe *p)
1822{
1823 unregister_kprobes(kps: &p, num: 1);
1824}
1825EXPORT_SYMBOL_GPL(unregister_kprobe);
1826
1827void unregister_kprobes(struct kprobe **kps, int num)
1828{
1829 int i;
1830
1831 if (num <= 0)
1832 return;
1833 scoped_guard(mutex, &kprobe_mutex) {
1834 for (i = 0; i < num; i++)
1835 if (__unregister_kprobe_top(p: kps[i]) < 0)
1836 kps[i]->addr = NULL;
1837 }
1838 synchronize_rcu();
1839 for (i = 0; i < num; i++)
1840 if (kps[i]->addr)
1841 __unregister_kprobe_bottom(p: kps[i]);
1842}
1843EXPORT_SYMBOL_GPL(unregister_kprobes);
1844
1845int __weak kprobe_exceptions_notify(struct notifier_block *self,
1846 unsigned long val, void *data)
1847{
1848 return NOTIFY_DONE;
1849}
1850NOKPROBE_SYMBOL(kprobe_exceptions_notify);
1851
1852static struct notifier_block kprobe_exceptions_nb = {
1853 .notifier_call = kprobe_exceptions_notify,
1854 .priority = 0x7fffffff /* we need to be notified first */
1855};
1856
1857#ifdef CONFIG_KRETPROBES
1858
1859#if !defined(CONFIG_KRETPROBE_ON_RETHOOK)
1860
1861/* callbacks for objpool of kretprobe instances */
1862static int kretprobe_init_inst(void *nod, void *context)
1863{
1864 struct kretprobe_instance *ri = nod;
1865
1866 ri->rph = context;
1867 return 0;
1868}
1869static int kretprobe_fini_pool(struct objpool_head *head, void *context)
1870{
1871 kfree(context);
1872 return 0;
1873}
1874
1875static void free_rp_inst_rcu(struct rcu_head *head)
1876{
1877 struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu);
1878 struct kretprobe_holder *rph = ri->rph;
1879
1880 objpool_drop(ri, &rph->pool);
1881}
1882NOKPROBE_SYMBOL(free_rp_inst_rcu);
1883
1884static void recycle_rp_inst(struct kretprobe_instance *ri)
1885{
1886 struct kretprobe *rp = get_kretprobe(ri);
1887
1888 if (likely(rp))
1889 objpool_push(ri, &rp->rph->pool);
1890 else
1891 call_rcu(&ri->rcu, free_rp_inst_rcu);
1892}
1893NOKPROBE_SYMBOL(recycle_rp_inst);
1894
1895/*
1896 * This function is called from delayed_put_task_struct() when a task is
1897 * dead and cleaned up to recycle any kretprobe instances associated with
1898 * this task. These left over instances represent probed functions that
1899 * have been called but will never return.
1900 */
1901void kprobe_flush_task(struct task_struct *tk)
1902{
1903 struct kretprobe_instance *ri;
1904 struct llist_node *node;
1905
1906 /* Early boot, not yet initialized. */
1907 if (unlikely(!kprobes_initialized))
1908 return;
1909
1910 kprobe_busy_begin();
1911
1912 node = __llist_del_all(&tk->kretprobe_instances);
1913 while (node) {
1914 ri = container_of(node, struct kretprobe_instance, llist);
1915 node = node->next;
1916
1917 recycle_rp_inst(ri);
1918 }
1919
1920 kprobe_busy_end();
1921}
1922NOKPROBE_SYMBOL(kprobe_flush_task);
1923
1924static inline void free_rp_inst(struct kretprobe *rp)
1925{
1926 struct kretprobe_holder *rph = rp->rph;
1927
1928 if (!rph)
1929 return;
1930 rp->rph = NULL;
1931 objpool_fini(&rph->pool);
1932}
1933
1934/* This assumes the 'tsk' is the current task or the is not running. */
1935static kprobe_opcode_t *__kretprobe_find_ret_addr(struct task_struct *tsk,
1936 struct llist_node **cur)
1937{
1938 struct kretprobe_instance *ri = NULL;
1939 struct llist_node *node = *cur;
1940
1941 if (!node)
1942 node = tsk->kretprobe_instances.first;
1943 else
1944 node = node->next;
1945
1946 while (node) {
1947 ri = container_of(node, struct kretprobe_instance, llist);
1948 if (ri->ret_addr != kretprobe_trampoline_addr()) {
1949 *cur = node;
1950 return ri->ret_addr;
1951 }
1952 node = node->next;
1953 }
1954 return NULL;
1955}
1956NOKPROBE_SYMBOL(__kretprobe_find_ret_addr);
1957
1958/**
1959 * kretprobe_find_ret_addr -- Find correct return address modified by kretprobe
1960 * @tsk: Target task
1961 * @fp: A frame pointer
1962 * @cur: a storage of the loop cursor llist_node pointer for next call
1963 *
1964 * Find the correct return address modified by a kretprobe on @tsk in unsigned
1965 * long type. If it finds the return address, this returns that address value,
1966 * or this returns 0.
1967 * The @tsk must be 'current' or a task which is not running. @fp is a hint
1968 * to get the currect return address - which is compared with the
1969 * kretprobe_instance::fp field. The @cur is a loop cursor for searching the
1970 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
1971 * first call, but '@cur' itself must NOT NULL.
1972 */
1973unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
1974 struct llist_node **cur)
1975{
1976 struct kretprobe_instance *ri;
1977 kprobe_opcode_t *ret;
1978
1979 if (WARN_ON_ONCE(!cur))
1980 return 0;
1981
1982 do {
1983 ret = __kretprobe_find_ret_addr(tsk, cur);
1984 if (!ret)
1985 break;
1986 ri = container_of(*cur, struct kretprobe_instance, llist);
1987 } while (ri->fp != fp);
1988
1989 return (unsigned long)ret;
1990}
1991NOKPROBE_SYMBOL(kretprobe_find_ret_addr);
1992
1993void __weak arch_kretprobe_fixup_return(struct pt_regs *regs,
1994 kprobe_opcode_t *correct_ret_addr)
1995{
1996 /*
1997 * Do nothing by default. Please fill this to update the fake return
1998 * address on the stack with the correct one on each arch if possible.
1999 */
2000}
2001
2002unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
2003 void *frame_pointer)
2004{
2005 struct kretprobe_instance *ri = NULL;
2006 struct llist_node *first, *node = NULL;
2007 kprobe_opcode_t *correct_ret_addr;
2008 struct kretprobe *rp;
2009
2010 /* Find correct address and all nodes for this frame. */
2011 correct_ret_addr = __kretprobe_find_ret_addr(current, &node);
2012 if (!correct_ret_addr) {
2013 pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n");
2014 BUG_ON(1);
2015 }
2016
2017 /*
2018 * Set the return address as the instruction pointer, because if the
2019 * user handler calls stack_trace_save_regs() with this 'regs',
2020 * the stack trace will start from the instruction pointer.
2021 */
2022 instruction_pointer_set(regs, (unsigned long)correct_ret_addr);
2023
2024 /* Run the user handler of the nodes. */
2025 first = current->kretprobe_instances.first;
2026 while (first) {
2027 ri = container_of(first, struct kretprobe_instance, llist);
2028
2029 if (WARN_ON_ONCE(ri->fp != frame_pointer))
2030 break;
2031
2032 rp = get_kretprobe(ri);
2033 if (rp && rp->handler) {
2034 struct kprobe *prev = kprobe_running();
2035
2036 __this_cpu_write(current_kprobe, &rp->kp);
2037 ri->ret_addr = correct_ret_addr;
2038 rp->handler(ri, regs);
2039 __this_cpu_write(current_kprobe, prev);
2040 }
2041 if (first == node)
2042 break;
2043
2044 first = first->next;
2045 }
2046
2047 arch_kretprobe_fixup_return(regs, correct_ret_addr);
2048
2049 /* Unlink all nodes for this frame. */
2050 first = current->kretprobe_instances.first;
2051 current->kretprobe_instances.first = node->next;
2052 node->next = NULL;
2053
2054 /* Recycle free instances. */
2055 while (first) {
2056 ri = container_of(first, struct kretprobe_instance, llist);
2057 first = first->next;
2058
2059 recycle_rp_inst(ri);
2060 }
2061
2062 return (unsigned long)correct_ret_addr;
2063}
2064NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
2065
2066/*
2067 * This kprobe pre_handler is registered with every kretprobe. When probe
2068 * hits it will set up the return probe.
2069 */
2070static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2071{
2072 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2073 struct kretprobe_holder *rph = rp->rph;
2074 struct kretprobe_instance *ri;
2075
2076 ri = objpool_pop(&rph->pool);
2077 if (!ri) {
2078 rp->nmissed++;
2079 return 0;
2080 }
2081
2082 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
2083 objpool_push(ri, &rph->pool);
2084 return 0;
2085 }
2086
2087 arch_prepare_kretprobe(ri, regs);
2088
2089 __llist_add(&ri->llist, &current->kretprobe_instances);
2090
2091 return 0;
2092}
2093NOKPROBE_SYMBOL(pre_handler_kretprobe);
2094#else /* CONFIG_KRETPROBE_ON_RETHOOK */
2095/*
2096 * This kprobe pre_handler is registered with every kretprobe. When probe
2097 * hits it will set up the return probe.
2098 */
2099static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2100{
2101 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2102 struct kretprobe_instance *ri;
2103 struct rethook_node *rhn;
2104
2105 rhn = rethook_try_get(rh: rp->rh);
2106 if (!rhn) {
2107 rp->nmissed++;
2108 return 0;
2109 }
2110
2111 ri = container_of(rhn, struct kretprobe_instance, node);
2112
2113 if (rp->entry_handler && rp->entry_handler(ri, regs))
2114 rethook_recycle(node: rhn);
2115 else
2116 rethook_hook(node: rhn, regs, mcount: kprobe_ftrace(p));
2117
2118 return 0;
2119}
2120NOKPROBE_SYMBOL(pre_handler_kretprobe);
2121
2122static void kretprobe_rethook_handler(struct rethook_node *rh, void *data,
2123 unsigned long ret_addr,
2124 struct pt_regs *regs)
2125{
2126 struct kretprobe *rp = (struct kretprobe *)data;
2127 struct kretprobe_instance *ri;
2128 struct kprobe_ctlblk *kcb;
2129
2130 /* The data must NOT be null. This means rethook data structure is broken. */
2131 if (WARN_ON_ONCE(!data) || !rp->handler)
2132 return;
2133
2134 __this_cpu_write(current_kprobe, &rp->kp);
2135 kcb = get_kprobe_ctlblk();
2136 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
2137
2138 ri = container_of(rh, struct kretprobe_instance, node);
2139 rp->handler(ri, regs);
2140
2141 __this_cpu_write(current_kprobe, NULL);
2142}
2143NOKPROBE_SYMBOL(kretprobe_rethook_handler);
2144
2145#endif /* !CONFIG_KRETPROBE_ON_RETHOOK */
2146
2147/**
2148 * kprobe_on_func_entry() -- check whether given address is function entry
2149 * @addr: Target address
2150 * @sym: Target symbol name
2151 * @offset: The offset from the symbol or the address
2152 *
2153 * This checks whether the given @addr+@offset or @sym+@offset is on the
2154 * function entry address or not.
2155 * This returns 0 if it is the function entry, or -EINVAL if it is not.
2156 * And also it returns -ENOENT if it fails the symbol or address lookup.
2157 * Caller must pass @addr or @sym (either one must be NULL), or this
2158 * returns -EINVAL.
2159 */
2160int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
2161{
2162 bool on_func_entry;
2163 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, symbol_name: sym, offset, on_func_entry: &on_func_entry);
2164
2165 if (IS_ERR(ptr: kp_addr))
2166 return PTR_ERR(ptr: kp_addr);
2167
2168 if (!on_func_entry)
2169 return -EINVAL;
2170
2171 return 0;
2172}
2173
2174int register_kretprobe(struct kretprobe *rp)
2175{
2176 int ret;
2177 int i;
2178 void *addr;
2179
2180 ret = kprobe_on_func_entry(addr: rp->kp.addr, sym: rp->kp.symbol_name, offset: rp->kp.offset);
2181 if (ret)
2182 return ret;
2183
2184 /* If only 'rp->kp.addr' is specified, check reregistering kprobes */
2185 if (rp->kp.addr && warn_kprobe_rereg(p: &rp->kp))
2186 return -EINVAL;
2187
2188 if (kretprobe_blacklist_size) {
2189 addr = kprobe_addr(p: &rp->kp);
2190 if (IS_ERR(ptr: addr))
2191 return PTR_ERR(ptr: addr);
2192
2193 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2194 if (kretprobe_blacklist[i].addr == addr)
2195 return -EINVAL;
2196 }
2197 }
2198
2199 if (rp->data_size > KRETPROBE_MAX_DATA_SIZE)
2200 return -E2BIG;
2201
2202 rp->kp.pre_handler = pre_handler_kretprobe;
2203 rp->kp.post_handler = NULL;
2204
2205 /* Pre-allocate memory for max kretprobe instances */
2206 if (rp->maxactive <= 0)
2207 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
2208
2209#ifdef CONFIG_KRETPROBE_ON_RETHOOK
2210 rp->rh = rethook_alloc(data: (void *)rp, handler: kretprobe_rethook_handler,
2211 size: sizeof(struct kretprobe_instance) +
2212 rp->data_size, num: rp->maxactive);
2213 if (IS_ERR(ptr: rp->rh))
2214 return PTR_ERR(ptr: rp->rh);
2215
2216 rp->nmissed = 0;
2217 /* Establish function entry probe point */
2218 ret = register_kprobe(&rp->kp);
2219 if (ret != 0) {
2220 rethook_free(rh: rp->rh);
2221 rp->rh = NULL;
2222 }
2223#else /* !CONFIG_KRETPROBE_ON_RETHOOK */
2224 rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL);
2225 if (!rp->rph)
2226 return -ENOMEM;
2227
2228 if (objpool_init(&rp->rph->pool, rp->maxactive, rp->data_size +
2229 sizeof(struct kretprobe_instance), GFP_KERNEL,
2230 rp->rph, kretprobe_init_inst, kretprobe_fini_pool)) {
2231 kfree(rp->rph);
2232 rp->rph = NULL;
2233 return -ENOMEM;
2234 }
2235 rcu_assign_pointer(rp->rph->rp, rp);
2236 rp->nmissed = 0;
2237 /* Establish function entry probe point */
2238 ret = register_kprobe(&rp->kp);
2239 if (ret != 0)
2240 free_rp_inst(rp);
2241#endif
2242 return ret;
2243}
2244EXPORT_SYMBOL_GPL(register_kretprobe);
2245
2246int register_kretprobes(struct kretprobe **rps, int num)
2247{
2248 int ret = 0, i;
2249
2250 if (num <= 0)
2251 return -EINVAL;
2252 for (i = 0; i < num; i++) {
2253 ret = register_kretprobe(rps[i]);
2254 if (ret < 0) {
2255 if (i > 0)
2256 unregister_kretprobes(rps, num: i);
2257 break;
2258 }
2259 }
2260 return ret;
2261}
2262EXPORT_SYMBOL_GPL(register_kretprobes);
2263
2264void unregister_kretprobe(struct kretprobe *rp)
2265{
2266 unregister_kretprobes(rps: &rp, num: 1);
2267}
2268EXPORT_SYMBOL_GPL(unregister_kretprobe);
2269
2270void unregister_kretprobes(struct kretprobe **rps, int num)
2271{
2272 int i;
2273
2274 if (num <= 0)
2275 return;
2276 for (i = 0; i < num; i++) {
2277 guard(mutex)(T: &kprobe_mutex);
2278
2279 if (__unregister_kprobe_top(p: &rps[i]->kp) < 0)
2280 rps[i]->kp.addr = NULL;
2281#ifdef CONFIG_KRETPROBE_ON_RETHOOK
2282 rethook_free(rh: rps[i]->rh);
2283#else
2284 rcu_assign_pointer(rps[i]->rph->rp, NULL);
2285#endif
2286 }
2287
2288 synchronize_rcu();
2289 for (i = 0; i < num; i++) {
2290 if (rps[i]->kp.addr) {
2291 __unregister_kprobe_bottom(p: &rps[i]->kp);
2292#ifndef CONFIG_KRETPROBE_ON_RETHOOK
2293 free_rp_inst(rps[i]);
2294#endif
2295 }
2296 }
2297}
2298EXPORT_SYMBOL_GPL(unregister_kretprobes);
2299
2300#else /* CONFIG_KRETPROBES */
2301int register_kretprobe(struct kretprobe *rp)
2302{
2303 return -EOPNOTSUPP;
2304}
2305EXPORT_SYMBOL_GPL(register_kretprobe);
2306
2307int register_kretprobes(struct kretprobe **rps, int num)
2308{
2309 return -EOPNOTSUPP;
2310}
2311EXPORT_SYMBOL_GPL(register_kretprobes);
2312
2313void unregister_kretprobe(struct kretprobe *rp)
2314{
2315}
2316EXPORT_SYMBOL_GPL(unregister_kretprobe);
2317
2318void unregister_kretprobes(struct kretprobe **rps, int num)
2319{
2320}
2321EXPORT_SYMBOL_GPL(unregister_kretprobes);
2322
2323static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2324{
2325 return 0;
2326}
2327NOKPROBE_SYMBOL(pre_handler_kretprobe);
2328
2329#endif /* CONFIG_KRETPROBES */
2330
2331/* Set the kprobe gone and remove its instruction buffer. */
2332static void kill_kprobe(struct kprobe *p)
2333{
2334 struct kprobe *kp;
2335
2336 lockdep_assert_held(&kprobe_mutex);
2337
2338 /*
2339 * The module is going away. We should disarm the kprobe which
2340 * is using ftrace, because ftrace framework is still available at
2341 * 'MODULE_STATE_GOING' notification.
2342 */
2343 if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed)
2344 disarm_kprobe_ftrace(p);
2345
2346 p->flags |= KPROBE_FLAG_GONE;
2347 if (kprobe_aggrprobe(p)) {
2348 /*
2349 * If this is an aggr_kprobe, we have to list all the
2350 * chained probes and mark them GONE.
2351 */
2352 list_for_each_entry(kp, &p->list, list)
2353 kp->flags |= KPROBE_FLAG_GONE;
2354 p->post_handler = NULL;
2355 kill_optimized_kprobe(p);
2356 }
2357 /*
2358 * Here, we can remove insn_slot safely, because no thread calls
2359 * the original probed function (which will be freed soon) any more.
2360 */
2361 arch_remove_kprobe(p);
2362}
2363
2364/* Disable one kprobe */
2365int disable_kprobe(struct kprobe *kp)
2366{
2367 struct kprobe *p;
2368
2369 guard(mutex)(T: &kprobe_mutex);
2370
2371 /* Disable this kprobe */
2372 p = __disable_kprobe(p: kp);
2373
2374 return IS_ERR(ptr: p) ? PTR_ERR(ptr: p) : 0;
2375}
2376EXPORT_SYMBOL_GPL(disable_kprobe);
2377
2378/* Enable one kprobe */
2379int enable_kprobe(struct kprobe *kp)
2380{
2381 int ret = 0;
2382 struct kprobe *p;
2383
2384 guard(mutex)(T: &kprobe_mutex);
2385
2386 /* Check whether specified probe is valid. */
2387 p = __get_valid_kprobe(p: kp);
2388 if (unlikely(p == NULL))
2389 return -EINVAL;
2390
2391 if (kprobe_gone(p: kp))
2392 /* This kprobe has gone, we couldn't enable it. */
2393 return -EINVAL;
2394
2395 if (p != kp)
2396 kp->flags &= ~KPROBE_FLAG_DISABLED;
2397
2398 if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2399 p->flags &= ~KPROBE_FLAG_DISABLED;
2400 ret = arm_kprobe(kp: p);
2401 if (ret) {
2402 p->flags |= KPROBE_FLAG_DISABLED;
2403 if (p != kp)
2404 kp->flags |= KPROBE_FLAG_DISABLED;
2405 }
2406 }
2407 return ret;
2408}
2409EXPORT_SYMBOL_GPL(enable_kprobe);
2410
2411/* Caller must NOT call this in usual path. This is only for critical case */
2412void dump_kprobe(struct kprobe *kp)
2413{
2414 pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n",
2415 kp->symbol_name, kp->offset, kp->addr);
2416}
2417NOKPROBE_SYMBOL(dump_kprobe);
2418
2419int kprobe_add_ksym_blacklist(unsigned long entry)
2420{
2421 struct kprobe_blacklist_entry *ent;
2422 unsigned long offset = 0, size = 0;
2423
2424 if (!kernel_text_address(addr: entry) ||
2425 !kallsyms_lookup_size_offset(addr: entry, symbolsize: &size, offset: &offset))
2426 return -EINVAL;
2427
2428 ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2429 if (!ent)
2430 return -ENOMEM;
2431 ent->start_addr = entry;
2432 ent->end_addr = entry + size;
2433 INIT_LIST_HEAD(list: &ent->list);
2434 list_add_tail(new: &ent->list, head: &kprobe_blacklist);
2435
2436 return (int)size;
2437}
2438
2439/* Add all symbols in given area into kprobe blacklist */
2440int kprobe_add_area_blacklist(unsigned long start, unsigned long end)
2441{
2442 unsigned long entry;
2443 int ret = 0;
2444
2445 for (entry = start; entry < end; entry += ret) {
2446 ret = kprobe_add_ksym_blacklist(entry);
2447 if (ret < 0)
2448 return ret;
2449 if (ret == 0) /* In case of alias symbol */
2450 ret = 1;
2451 }
2452 return 0;
2453}
2454
2455int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value,
2456 char *type, char *sym)
2457{
2458 return -ERANGE;
2459}
2460
2461int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
2462 char *sym)
2463{
2464#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
2465 if (!kprobe_cache_get_kallsym(c: &kprobe_insn_slots, symnum: &symnum, value, type, sym))
2466 return 0;
2467#ifdef CONFIG_OPTPROBES
2468 if (!kprobe_cache_get_kallsym(c: &kprobe_optinsn_slots, symnum: &symnum, value, type, sym))
2469 return 0;
2470#endif
2471#endif
2472 if (!arch_kprobe_get_kallsym(symnum: &symnum, value, type, sym))
2473 return 0;
2474 return -ERANGE;
2475}
2476
2477int __init __weak arch_populate_kprobe_blacklist(void)
2478{
2479 return 0;
2480}
2481
2482/*
2483 * Lookup and populate the kprobe_blacklist.
2484 *
2485 * Unlike the kretprobe blacklist, we'll need to determine
2486 * the range of addresses that belong to the said functions,
2487 * since a kprobe need not necessarily be at the beginning
2488 * of a function.
2489 */
2490static int __init populate_kprobe_blacklist(unsigned long *start,
2491 unsigned long *end)
2492{
2493 unsigned long entry;
2494 unsigned long *iter;
2495 int ret;
2496
2497 for (iter = start; iter < end; iter++) {
2498 entry = (unsigned long)dereference_symbol_descriptor(ptr: (void *)*iter);
2499 ret = kprobe_add_ksym_blacklist(entry);
2500 if (ret == -EINVAL)
2501 continue;
2502 if (ret < 0)
2503 return ret;
2504 }
2505
2506 /* Symbols in '__kprobes_text' are blacklisted */
2507 ret = kprobe_add_area_blacklist(start: (unsigned long)__kprobes_text_start,
2508 end: (unsigned long)__kprobes_text_end);
2509 if (ret)
2510 return ret;
2511
2512 /* Symbols in 'noinstr' section are blacklisted */
2513 ret = kprobe_add_area_blacklist(start: (unsigned long)__noinstr_text_start,
2514 end: (unsigned long)__noinstr_text_end);
2515
2516 return ret ? : arch_populate_kprobe_blacklist();
2517}
2518
2519#ifdef CONFIG_MODULES
2520/* Remove all symbols in given area from kprobe blacklist */
2521static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
2522{
2523 struct kprobe_blacklist_entry *ent, *n;
2524
2525 list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
2526 if (ent->start_addr < start || ent->start_addr >= end)
2527 continue;
2528 list_del(entry: &ent->list);
2529 kfree(objp: ent);
2530 }
2531}
2532
2533static void kprobe_remove_ksym_blacklist(unsigned long entry)
2534{
2535 kprobe_remove_area_blacklist(start: entry, end: entry + 1);
2536}
2537
2538static void add_module_kprobe_blacklist(struct module *mod)
2539{
2540 unsigned long start, end;
2541 int i;
2542
2543 if (mod->kprobe_blacklist) {
2544 for (i = 0; i < mod->num_kprobe_blacklist; i++)
2545 kprobe_add_ksym_blacklist(entry: mod->kprobe_blacklist[i]);
2546 }
2547
2548 start = (unsigned long)mod->kprobes_text_start;
2549 if (start) {
2550 end = start + mod->kprobes_text_size;
2551 kprobe_add_area_blacklist(start, end);
2552 }
2553
2554 start = (unsigned long)mod->noinstr_text_start;
2555 if (start) {
2556 end = start + mod->noinstr_text_size;
2557 kprobe_add_area_blacklist(start, end);
2558 }
2559}
2560
2561static void remove_module_kprobe_blacklist(struct module *mod)
2562{
2563 unsigned long start, end;
2564 int i;
2565
2566 if (mod->kprobe_blacklist) {
2567 for (i = 0; i < mod->num_kprobe_blacklist; i++)
2568 kprobe_remove_ksym_blacklist(entry: mod->kprobe_blacklist[i]);
2569 }
2570
2571 start = (unsigned long)mod->kprobes_text_start;
2572 if (start) {
2573 end = start + mod->kprobes_text_size;
2574 kprobe_remove_area_blacklist(start, end);
2575 }
2576
2577 start = (unsigned long)mod->noinstr_text_start;
2578 if (start) {
2579 end = start + mod->noinstr_text_size;
2580 kprobe_remove_area_blacklist(start, end);
2581 }
2582}
2583
2584/* Module notifier call back, checking kprobes on the module */
2585static int kprobes_module_callback(struct notifier_block *nb,
2586 unsigned long val, void *data)
2587{
2588 struct module *mod = data;
2589 struct hlist_head *head;
2590 struct kprobe *p;
2591 unsigned int i;
2592 int checkcore = (val == MODULE_STATE_GOING);
2593
2594 guard(mutex)(T: &kprobe_mutex);
2595
2596 if (val == MODULE_STATE_COMING)
2597 add_module_kprobe_blacklist(mod);
2598
2599 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2600 return NOTIFY_DONE;
2601
2602 /*
2603 * When 'MODULE_STATE_GOING' was notified, both of module '.text' and
2604 * '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was
2605 * notified, only '.init.text' section would be freed. We need to
2606 * disable kprobes which have been inserted in the sections.
2607 */
2608 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2609 head = &kprobe_table[i];
2610 hlist_for_each_entry(p, head, hlist)
2611 if (within_module_init(addr: (unsigned long)p->addr, mod) ||
2612 (checkcore &&
2613 within_module_core(addr: (unsigned long)p->addr, mod))) {
2614 /*
2615 * The vaddr this probe is installed will soon
2616 * be vfreed buy not synced to disk. Hence,
2617 * disarming the breakpoint isn't needed.
2618 *
2619 * Note, this will also move any optimized probes
2620 * that are pending to be removed from their
2621 * corresponding lists to the 'freeing_list' and
2622 * will not be touched by the delayed
2623 * kprobe_optimizer() work handler.
2624 */
2625 kill_kprobe(p);
2626 }
2627 }
2628 if (val == MODULE_STATE_GOING)
2629 remove_module_kprobe_blacklist(mod);
2630 return NOTIFY_DONE;
2631}
2632
2633static struct notifier_block kprobe_module_nb = {
2634 .notifier_call = kprobes_module_callback,
2635 .priority = 0
2636};
2637
2638static int kprobe_register_module_notifier(void)
2639{
2640 return register_module_notifier(nb: &kprobe_module_nb);
2641}
2642#else
2643static int kprobe_register_module_notifier(void)
2644{
2645 return 0;
2646}
2647#endif /* CONFIG_MODULES */
2648
2649void kprobe_free_init_mem(void)
2650{
2651 void *start = (void *)(&__init_begin);
2652 void *end = (void *)(&__init_end);
2653 struct hlist_head *head;
2654 struct kprobe *p;
2655 int i;
2656
2657 guard(mutex)(T: &kprobe_mutex);
2658
2659 /* Kill all kprobes on initmem because the target code has been freed. */
2660 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2661 head = &kprobe_table[i];
2662 hlist_for_each_entry(p, head, hlist) {
2663 if (start <= (void *)p->addr && (void *)p->addr < end)
2664 kill_kprobe(p);
2665 }
2666 }
2667}
2668
2669static int __init init_kprobes(void)
2670{
2671 int i, err;
2672
2673 /* FIXME allocate the probe table, currently defined statically */
2674 /* initialize all list heads */
2675 for (i = 0; i < KPROBE_TABLE_SIZE; i++)
2676 INIT_HLIST_HEAD(&kprobe_table[i]);
2677
2678 err = populate_kprobe_blacklist(start: __start_kprobe_blacklist,
2679 end: __stop_kprobe_blacklist);
2680 if (err)
2681 pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err);
2682
2683 if (kretprobe_blacklist_size) {
2684 /* lookup the function address from its name */
2685 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2686 kretprobe_blacklist[i].addr =
2687 kprobe_lookup_name(name: kretprobe_blacklist[i].name, unused: 0);
2688 if (!kretprobe_blacklist[i].addr)
2689 pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n",
2690 kretprobe_blacklist[i].name);
2691 }
2692 }
2693
2694 /* By default, kprobes are armed */
2695 kprobes_all_disarmed = false;
2696
2697#if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT)
2698 /* Init 'kprobe_optinsn_slots' for allocation */
2699 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2700#endif
2701
2702 err = arch_init_kprobes();
2703 if (!err)
2704 err = register_die_notifier(nb: &kprobe_exceptions_nb);
2705 if (!err)
2706 err = kprobe_register_module_notifier();
2707
2708 kprobes_initialized = (err == 0);
2709 kprobe_sysctls_init();
2710 return err;
2711}
2712early_initcall(init_kprobes);
2713
2714#if defined(CONFIG_OPTPROBES)
2715static int __init init_optprobes(void)
2716{
2717 /*
2718 * Enable kprobe optimization - this kicks the optimizer which
2719 * depends on synchronize_rcu_tasks() and ksoftirqd, that is
2720 * not spawned in early initcall. So delay the optimization.
2721 */
2722 optimize_all_kprobes();
2723
2724 return 0;
2725}
2726subsys_initcall(init_optprobes);
2727#endif
2728
2729#ifdef CONFIG_DEBUG_FS
2730static void report_probe(struct seq_file *pi, struct kprobe *p,
2731 const char *sym, int offset, char *modname, struct kprobe *pp)
2732{
2733 char *kprobe_type;
2734 void *addr = p->addr;
2735
2736 if (p->pre_handler == pre_handler_kretprobe)
2737 kprobe_type = "r";
2738 else
2739 kprobe_type = "k";
2740
2741 if (!kallsyms_show_value(cred: pi->file->f_cred))
2742 addr = NULL;
2743
2744 if (sym)
2745 seq_printf(m: pi, fmt: "%px %s %s+0x%x %s ",
2746 addr, kprobe_type, sym, offset,
2747 (modname ? modname : " "));
2748 else /* try to use %pS */
2749 seq_printf(m: pi, fmt: "%px %s %pS ",
2750 addr, kprobe_type, p->addr);
2751
2752 if (!pp)
2753 pp = p;
2754 seq_printf(m: pi, fmt: "%s%s%s%s\n",
2755 (kprobe_gone(p) ? "[GONE]" : ""),
2756 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
2757 (kprobe_optimized(p: pp) ? "[OPTIMIZED]" : ""),
2758 (kprobe_ftrace(p: pp) ? "[FTRACE]" : ""));
2759}
2760
2761static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
2762{
2763 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2764}
2765
2766static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
2767{
2768 (*pos)++;
2769 if (*pos >= KPROBE_TABLE_SIZE)
2770 return NULL;
2771 return pos;
2772}
2773
2774static void kprobe_seq_stop(struct seq_file *f, void *v)
2775{
2776 /* Nothing to do */
2777}
2778
2779static int show_kprobe_addr(struct seq_file *pi, void *v)
2780{
2781 struct hlist_head *head;
2782 struct kprobe *p, *kp;
2783 const char *sym;
2784 unsigned int i = *(loff_t *) v;
2785 unsigned long offset = 0;
2786 char *modname, namebuf[KSYM_NAME_LEN];
2787
2788 head = &kprobe_table[i];
2789 preempt_disable();
2790 hlist_for_each_entry_rcu(p, head, hlist) {
2791 sym = kallsyms_lookup(addr: (unsigned long)p->addr, NULL,
2792 offset: &offset, modname: &modname, namebuf);
2793 if (kprobe_aggrprobe(p)) {
2794 list_for_each_entry_rcu(kp, &p->list, list)
2795 report_probe(pi, p: kp, sym, offset, modname, pp: p);
2796 } else
2797 report_probe(pi, p, sym, offset, modname, NULL);
2798 }
2799 preempt_enable();
2800 return 0;
2801}
2802
2803static const struct seq_operations kprobes_sops = {
2804 .start = kprobe_seq_start,
2805 .next = kprobe_seq_next,
2806 .stop = kprobe_seq_stop,
2807 .show = show_kprobe_addr
2808};
2809
2810DEFINE_SEQ_ATTRIBUTE(kprobes);
2811
2812/* kprobes/blacklist -- shows which functions can not be probed */
2813static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2814{
2815 mutex_lock(lock: &kprobe_mutex);
2816 return seq_list_start(head: &kprobe_blacklist, pos: *pos);
2817}
2818
2819static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2820{
2821 return seq_list_next(v, head: &kprobe_blacklist, ppos: pos);
2822}
2823
2824static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2825{
2826 struct kprobe_blacklist_entry *ent =
2827 list_entry(v, struct kprobe_blacklist_entry, list);
2828
2829 /*
2830 * If '/proc/kallsyms' is not showing kernel address, we won't
2831 * show them here either.
2832 */
2833 if (!kallsyms_show_value(cred: m->file->f_cred))
2834 seq_printf(m, fmt: "0x%px-0x%px\t%ps\n", NULL, NULL,
2835 (void *)ent->start_addr);
2836 else
2837 seq_printf(m, fmt: "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2838 (void *)ent->end_addr, (void *)ent->start_addr);
2839 return 0;
2840}
2841
2842static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v)
2843{
2844 mutex_unlock(lock: &kprobe_mutex);
2845}
2846
2847static const struct seq_operations kprobe_blacklist_sops = {
2848 .start = kprobe_blacklist_seq_start,
2849 .next = kprobe_blacklist_seq_next,
2850 .stop = kprobe_blacklist_seq_stop,
2851 .show = kprobe_blacklist_seq_show,
2852};
2853DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist);
2854
2855static int arm_all_kprobes(void)
2856{
2857 struct hlist_head *head;
2858 struct kprobe *p;
2859 unsigned int i, total = 0, errors = 0;
2860 int err, ret = 0;
2861
2862 guard(mutex)(T: &kprobe_mutex);
2863
2864 /* If kprobes are armed, just return */
2865 if (!kprobes_all_disarmed)
2866 return 0;
2867
2868 /*
2869 * optimize_kprobe() called by arm_kprobe() checks
2870 * kprobes_all_disarmed, so set kprobes_all_disarmed before
2871 * arm_kprobe.
2872 */
2873 kprobes_all_disarmed = false;
2874 /* Arming kprobes doesn't optimize kprobe itself */
2875 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2876 head = &kprobe_table[i];
2877 /* Arm all kprobes on a best-effort basis */
2878 hlist_for_each_entry(p, head, hlist) {
2879 if (!kprobe_disabled(p)) {
2880 err = arm_kprobe(kp: p);
2881 if (err) {
2882 errors++;
2883 ret = err;
2884 }
2885 total++;
2886 }
2887 }
2888 }
2889
2890 if (errors)
2891 pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n",
2892 errors, total);
2893 else
2894 pr_info("Kprobes globally enabled\n");
2895
2896 return ret;
2897}
2898
2899static int disarm_all_kprobes(void)
2900{
2901 struct hlist_head *head;
2902 struct kprobe *p;
2903 unsigned int i, total = 0, errors = 0;
2904 int err, ret = 0;
2905
2906 guard(mutex)(T: &kprobe_mutex);
2907
2908 /* If kprobes are already disarmed, just return */
2909 if (kprobes_all_disarmed)
2910 return 0;
2911
2912 kprobes_all_disarmed = true;
2913
2914 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2915 head = &kprobe_table[i];
2916 /* Disarm all kprobes on a best-effort basis */
2917 hlist_for_each_entry(p, head, hlist) {
2918 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
2919 err = disarm_kprobe(kp: p, reopt: false);
2920 if (err) {
2921 errors++;
2922 ret = err;
2923 }
2924 total++;
2925 }
2926 }
2927 }
2928
2929 if (errors)
2930 pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n",
2931 errors, total);
2932 else
2933 pr_info("Kprobes globally disabled\n");
2934
2935 /* Wait for disarming all kprobes by optimizer */
2936 wait_for_kprobe_optimizer_locked();
2937 return ret;
2938}
2939
2940/*
2941 * XXX: The debugfs bool file interface doesn't allow for callbacks
2942 * when the bool state is switched. We can reuse that facility when
2943 * available
2944 */
2945static ssize_t read_enabled_file_bool(struct file *file,
2946 char __user *user_buf, size_t count, loff_t *ppos)
2947{
2948 char buf[3];
2949
2950 if (!kprobes_all_disarmed)
2951 buf[0] = '1';
2952 else
2953 buf[0] = '0';
2954 buf[1] = '\n';
2955 buf[2] = 0x00;
2956 return simple_read_from_buffer(to: user_buf, count, ppos, from: buf, available: 2);
2957}
2958
2959static ssize_t write_enabled_file_bool(struct file *file,
2960 const char __user *user_buf, size_t count, loff_t *ppos)
2961{
2962 bool enable;
2963 int ret;
2964
2965 ret = kstrtobool_from_user(s: user_buf, count, res: &enable);
2966 if (ret)
2967 return ret;
2968
2969 ret = enable ? arm_all_kprobes() : disarm_all_kprobes();
2970 if (ret)
2971 return ret;
2972
2973 return count;
2974}
2975
2976static const struct file_operations fops_kp = {
2977 .read = read_enabled_file_bool,
2978 .write = write_enabled_file_bool,
2979 .llseek = default_llseek,
2980};
2981
2982static int __init debugfs_kprobe_init(void)
2983{
2984 struct dentry *dir;
2985
2986 dir = debugfs_create_dir(name: "kprobes", NULL);
2987
2988 debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
2989
2990 debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
2991
2992 debugfs_create_file("blacklist", 0400, dir, NULL,
2993 &kprobe_blacklist_fops);
2994
2995 return 0;
2996}
2997
2998late_initcall(debugfs_kprobe_init);
2999#endif /* CONFIG_DEBUG_FS */
3000