1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_UNWIND_H
3#define _ASM_X86_UNWIND_H
4
5#include <linux/sched.h>
6#include <linux/ftrace.h>
7#include <linux/rethook.h>
8#include <asm/ptrace.h>
9#include <asm/stacktrace.h>
10
11#define IRET_FRAME_OFFSET (offsetof(struct pt_regs, ip))
12#define IRET_FRAME_SIZE (sizeof(struct pt_regs) - IRET_FRAME_OFFSET)
13
14struct unwind_state {
15 struct stack_info stack_info;
16 unsigned long stack_mask;
17 struct task_struct *task;
18 int graph_idx;
19#if defined(CONFIG_RETHOOK)
20 struct llist_node *kr_cur;
21#endif
22 bool error;
23#if defined(CONFIG_UNWINDER_ORC)
24 bool signal, full_regs;
25 unsigned long sp, bp, ip;
26 struct pt_regs *regs, *prev_regs;
27#elif defined(CONFIG_UNWINDER_FRAME_POINTER)
28 bool got_irq;
29 unsigned long *bp, *orig_sp, ip;
30 /*
31 * If non-NULL: The current frame is incomplete and doesn't contain a
32 * valid BP. When looking for the next frame, use this instead of the
33 * non-existent saved BP.
34 */
35 unsigned long *next_bp;
36 struct pt_regs *regs;
37#else
38 unsigned long *sp;
39#endif
40};
41
42void __unwind_start(struct unwind_state *state, struct task_struct *task,
43 struct pt_regs *regs, unsigned long *first_frame);
44bool unwind_next_frame(struct unwind_state *state);
45unsigned long unwind_get_return_address(struct unwind_state *state);
46unsigned long *unwind_get_return_address_ptr(struct unwind_state *state);
47
48static inline bool unwind_done(struct unwind_state *state)
49{
50 return state->stack_info.type == STACK_TYPE_UNKNOWN;
51}
52
53static inline bool unwind_error(struct unwind_state *state)
54{
55 return state->error;
56}
57
58static inline
59void unwind_start(struct unwind_state *state, struct task_struct *task,
60 struct pt_regs *regs, unsigned long *first_frame)
61{
62 first_frame = first_frame ? : get_stack_pointer(task, regs);
63
64 __unwind_start(state, task, regs, first_frame);
65}
66
67#if defined(CONFIG_UNWINDER_ORC) || defined(CONFIG_UNWINDER_FRAME_POINTER)
68/*
69 * If 'partial' returns true, only the iret frame registers are valid.
70 */
71static inline struct pt_regs *unwind_get_entry_regs(struct unwind_state *state,
72 bool *partial)
73{
74 if (unwind_done(state))
75 return NULL;
76
77 if (partial) {
78#ifdef CONFIG_UNWINDER_ORC
79 *partial = !state->full_regs;
80#else
81 *partial = false;
82#endif
83 }
84
85 return state->regs;
86}
87#else
88static inline struct pt_regs *unwind_get_entry_regs(struct unwind_state *state,
89 bool *partial)
90{
91 return NULL;
92}
93#endif
94
95#ifdef CONFIG_UNWINDER_ORC
96void unwind_init(void);
97void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size,
98 void *orc, size_t orc_size);
99#else
100static inline void unwind_init(void) {}
101static inline
102void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size,
103 void *orc, size_t orc_size) {}
104#endif
105
106static inline
107unsigned long unwind_recover_rethook(struct unwind_state *state,
108 unsigned long addr, unsigned long *addr_p)
109{
110#ifdef CONFIG_RETHOOK
111 if (is_rethook_trampoline(addr))
112 return rethook_find_ret_addr(tsk: state->task, frame: (unsigned long)addr_p,
113 cur: &state->kr_cur);
114#endif
115 return addr;
116}
117
118/* Recover the return address modified by rethook and ftrace_graph. */
119static inline
120unsigned long unwind_recover_ret_addr(struct unwind_state *state,
121 unsigned long addr, unsigned long *addr_p)
122{
123 unsigned long ret;
124
125 ret = ftrace_graph_ret_addr(task: state->task, idx: &state->graph_idx,
126 ret: addr, retp: addr_p);
127 return unwind_recover_rethook(state, addr: ret, addr_p);
128}
129
130/*
131 * This disables KASAN checking when reading a value from another task's stack,
132 * since the other task could be running on another CPU and could have poisoned
133 * the stack in the meantime.
134 */
135#define READ_ONCE_TASK_STACK(task, x) \
136({ \
137 unsigned long val; \
138 if (task == current) \
139 val = READ_ONCE(x); \
140 else \
141 val = READ_ONCE_NOCHECK(x); \
142 val; \
143})
144
145static inline bool task_on_another_cpu(struct task_struct *task)
146{
147#ifdef CONFIG_SMP
148 return task != current && task->on_cpu;
149#else
150 return false;
151#endif
152}
153
154#endif /* _ASM_X86_UNWIND_H */
155