1// SPDX-License-Identifier: GPL-2.0
2#include <linux/perf_event.h>
3#include <asm/msr.h>
4#include <asm/perf_event.h>
5
6#include "../perf_event.h"
7
8/* LBR Branch Select valid bits */
9#define LBR_SELECT_MASK 0x1ff
10
11/*
12 * LBR Branch Select filter bits which when set, ensures that the
13 * corresponding type of branches are not recorded
14 */
15#define LBR_SELECT_KERNEL 0 /* Branches ending in CPL = 0 */
16#define LBR_SELECT_USER 1 /* Branches ending in CPL > 0 */
17#define LBR_SELECT_JCC 2 /* Conditional branches */
18#define LBR_SELECT_CALL_NEAR_REL 3 /* Near relative calls */
19#define LBR_SELECT_CALL_NEAR_IND 4 /* Indirect relative calls */
20#define LBR_SELECT_RET_NEAR 5 /* Near returns */
21#define LBR_SELECT_JMP_NEAR_IND 6 /* Near indirect jumps (excl. calls and returns) */
22#define LBR_SELECT_JMP_NEAR_REL 7 /* Near relative jumps (excl. calls) */
23#define LBR_SELECT_FAR_BRANCH 8 /* Far branches */
24
25#define LBR_KERNEL BIT(LBR_SELECT_KERNEL)
26#define LBR_USER BIT(LBR_SELECT_USER)
27#define LBR_JCC BIT(LBR_SELECT_JCC)
28#define LBR_REL_CALL BIT(LBR_SELECT_CALL_NEAR_REL)
29#define LBR_IND_CALL BIT(LBR_SELECT_CALL_NEAR_IND)
30#define LBR_RETURN BIT(LBR_SELECT_RET_NEAR)
31#define LBR_REL_JMP BIT(LBR_SELECT_JMP_NEAR_REL)
32#define LBR_IND_JMP BIT(LBR_SELECT_JMP_NEAR_IND)
33#define LBR_FAR BIT(LBR_SELECT_FAR_BRANCH)
34#define LBR_NOT_SUPP -1 /* unsupported filter */
35#define LBR_IGNORE 0
36
37#define LBR_ANY \
38 (LBR_JCC | LBR_REL_CALL | LBR_IND_CALL | LBR_RETURN | \
39 LBR_REL_JMP | LBR_IND_JMP | LBR_FAR)
40
41struct branch_entry {
42 union {
43 struct {
44 u64 ip:58;
45 u64 ip_sign_ext:5;
46 u64 mispredict:1;
47 } split;
48 u64 full;
49 } from;
50
51 union {
52 struct {
53 u64 ip:58;
54 u64 ip_sign_ext:3;
55 u64 reserved:1;
56 u64 spec:1;
57 u64 valid:1;
58 } split;
59 u64 full;
60 } to;
61};
62
63static __always_inline void amd_pmu_lbr_set_from(unsigned int idx, u64 val)
64{
65 wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val);
66}
67
68static __always_inline void amd_pmu_lbr_set_to(unsigned int idx, u64 val)
69{
70 wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val);
71}
72
73static __always_inline u64 amd_pmu_lbr_get_from(unsigned int idx)
74{
75 u64 val;
76
77 rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val);
78
79 return val;
80}
81
82static __always_inline u64 amd_pmu_lbr_get_to(unsigned int idx)
83{
84 u64 val;
85
86 rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val);
87
88 return val;
89}
90
91static __always_inline u64 sign_ext_branch_ip(u64 ip)
92{
93 u32 shift = 64 - boot_cpu_data.x86_virt_bits;
94
95 return (u64)(((s64)ip << shift) >> shift);
96}
97
98static void amd_pmu_lbr_filter(void)
99{
100 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
101 int br_sel = cpuc->br_sel, offset, type, i, j;
102 bool compress = false;
103 bool fused_only = false;
104 u64 from, to;
105
106 /* If sampling all branches, there is nothing to filter */
107 if (((br_sel & X86_BR_ALL) == X86_BR_ALL) &&
108 ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE))
109 fused_only = true;
110
111 for (i = 0; i < cpuc->lbr_stack.nr; i++) {
112 from = cpuc->lbr_entries[i].from;
113 to = cpuc->lbr_entries[i].to;
114 type = branch_type_fused(from, to, abort: 0, offset: &offset);
115
116 /*
117 * Adjust the branch from address in case of instruction
118 * fusion where it points to an instruction preceding the
119 * actual branch
120 */
121 if (offset) {
122 cpuc->lbr_entries[i].from += offset;
123 if (fused_only)
124 continue;
125 }
126
127 /* If type does not correspond, then discard */
128 if (type == X86_BR_NONE || (br_sel & type) != type) {
129 cpuc->lbr_entries[i].from = 0; /* mark invalid */
130 compress = true;
131 }
132
133 if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE)
134 cpuc->lbr_entries[i].type = common_branch_type(type);
135 }
136
137 if (!compress)
138 return;
139
140 /* Remove all invalid entries */
141 for (i = 0; i < cpuc->lbr_stack.nr; ) {
142 if (!cpuc->lbr_entries[i].from) {
143 j = i;
144 while (++j < cpuc->lbr_stack.nr)
145 cpuc->lbr_entries[j - 1] = cpuc->lbr_entries[j];
146 cpuc->lbr_stack.nr--;
147 if (!cpuc->lbr_entries[i].from)
148 continue;
149 }
150 i++;
151 }
152}
153
154static const int lbr_spec_map[PERF_BR_SPEC_MAX] = {
155 PERF_BR_SPEC_NA,
156 PERF_BR_SPEC_WRONG_PATH,
157 PERF_BR_NON_SPEC_CORRECT_PATH,
158 PERF_BR_SPEC_CORRECT_PATH,
159};
160
161void amd_pmu_lbr_read(void)
162{
163 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
164 struct perf_branch_entry *br = cpuc->lbr_entries;
165 struct branch_entry entry;
166 int out = 0, idx, i;
167
168 if (!cpuc->lbr_users)
169 return;
170
171 for (i = 0; i < x86_pmu.lbr_nr; i++) {
172 entry.from.full = amd_pmu_lbr_get_from(idx: i);
173 entry.to.full = amd_pmu_lbr_get_to(idx: i);
174
175 /*
176 * Check if a branch has been logged; if valid = 0, spec = 0
177 * then no branch was recorded; if reserved = 1 then an
178 * erroneous branch was recorded (see Erratum 1452)
179 */
180 if ((!entry.to.split.valid && !entry.to.split.spec) ||
181 entry.to.split.reserved)
182 continue;
183
184 perf_clear_branch_entry_bitfields(br: br + out);
185
186 br[out].from = sign_ext_branch_ip(ip: entry.from.split.ip);
187 br[out].to = sign_ext_branch_ip(ip: entry.to.split.ip);
188 br[out].mispred = entry.from.split.mispredict;
189 br[out].predicted = !br[out].mispred;
190
191 /*
192 * Set branch speculation information using the status of
193 * the valid and spec bits.
194 *
195 * When valid = 0, spec = 0, no branch was recorded and the
196 * entry is discarded as seen above.
197 *
198 * When valid = 0, spec = 1, the recorded branch was
199 * speculative but took the wrong path.
200 *
201 * When valid = 1, spec = 0, the recorded branch was
202 * non-speculative but took the correct path.
203 *
204 * When valid = 1, spec = 1, the recorded branch was
205 * speculative and took the correct path
206 */
207 idx = (entry.to.split.valid << 1) | entry.to.split.spec;
208 br[out].spec = lbr_spec_map[idx];
209 out++;
210 }
211
212 cpuc->lbr_stack.nr = out;
213
214 /*
215 * Internal register renaming always ensures that LBR From[0] and
216 * LBR To[0] always represent the TOS
217 */
218 cpuc->lbr_stack.hw_idx = 0;
219
220 /* Perform further software filtering */
221 amd_pmu_lbr_filter();
222}
223
224static const int lbr_select_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = {
225 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER,
226 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL,
227 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGNORE,
228
229 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY,
230 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL | LBR_FAR,
231 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR,
232 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL,
233 [PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT] = LBR_NOT_SUPP,
234 [PERF_SAMPLE_BRANCH_IN_TX_SHIFT] = LBR_NOT_SUPP,
235 [PERF_SAMPLE_BRANCH_NO_TX_SHIFT] = LBR_NOT_SUPP,
236 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC,
237
238 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_NOT_SUPP,
239 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP,
240 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL,
241
242 [PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT] = LBR_NOT_SUPP,
243 [PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT] = LBR_NOT_SUPP,
244};
245
246static int amd_pmu_lbr_setup_filter(struct perf_event *event)
247{
248 struct hw_perf_event_extra *reg = &event->hw.branch_reg;
249 u64 br_type = event->attr.branch_sample_type;
250 u64 mask = 0, v;
251 int i;
252
253 /* No LBR support */
254 if (!x86_pmu.lbr_nr)
255 return -EOPNOTSUPP;
256
257 if (br_type & PERF_SAMPLE_BRANCH_USER)
258 mask |= X86_BR_USER;
259
260 if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
261 mask |= X86_BR_KERNEL;
262
263 /* Ignore BRANCH_HV here */
264
265 if (br_type & PERF_SAMPLE_BRANCH_ANY)
266 mask |= X86_BR_ANY;
267
268 if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
269 mask |= X86_BR_ANY_CALL;
270
271 if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
272 mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET;
273
274 if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
275 mask |= X86_BR_IND_CALL;
276
277 if (br_type & PERF_SAMPLE_BRANCH_COND)
278 mask |= X86_BR_JCC;
279
280 if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP)
281 mask |= X86_BR_IND_JMP;
282
283 if (br_type & PERF_SAMPLE_BRANCH_CALL)
284 mask |= X86_BR_CALL | X86_BR_ZERO_CALL;
285
286 if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE)
287 mask |= X86_BR_TYPE_SAVE;
288
289 reg->reg = mask;
290 mask = 0;
291
292 for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) {
293 if (!(br_type & BIT_ULL(i)))
294 continue;
295
296 v = lbr_select_map[i];
297 if (v == LBR_NOT_SUPP)
298 return -EOPNOTSUPP;
299
300 if (v != LBR_IGNORE)
301 mask |= v;
302 }
303
304 /* Filter bits operate in suppress mode */
305 reg->config = mask ^ LBR_SELECT_MASK;
306
307 return 0;
308}
309
310int amd_pmu_lbr_hw_config(struct perf_event *event)
311{
312 int ret = 0;
313
314 ret = amd_pmu_lbr_setup_filter(event);
315 if (!ret)
316 event->attach_state |= PERF_ATTACH_SCHED_CB;
317
318 return ret;
319}
320
321void amd_pmu_lbr_reset(void)
322{
323 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
324 int i;
325
326 if (!x86_pmu.lbr_nr)
327 return;
328
329 /* Reset all branch records individually */
330 for (i = 0; i < x86_pmu.lbr_nr; i++) {
331 amd_pmu_lbr_set_from(idx: i, val: 0);
332 amd_pmu_lbr_set_to(idx: i, val: 0);
333 }
334
335 cpuc->last_task_ctx = NULL;
336 cpuc->last_log_id = 0;
337 wrmsrq(MSR_AMD64_LBR_SELECT, val: 0);
338}
339
340void amd_pmu_lbr_add(struct perf_event *event)
341{
342 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
343 struct hw_perf_event_extra *reg = &event->hw.branch_reg;
344
345 if (!x86_pmu.lbr_nr)
346 return;
347
348 if (has_branch_stack(event)) {
349 cpuc->lbr_select = 1;
350 cpuc->lbr_sel->config = reg->config;
351 cpuc->br_sel = reg->reg;
352 }
353
354 perf_sched_cb_inc(pmu: event->pmu);
355
356 if (!cpuc->lbr_users++ && !event->total_time_running)
357 amd_pmu_lbr_reset();
358}
359
360void amd_pmu_lbr_del(struct perf_event *event)
361{
362 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
363
364 if (!x86_pmu.lbr_nr)
365 return;
366
367 if (has_branch_stack(event))
368 cpuc->lbr_select = 0;
369
370 cpuc->lbr_users--;
371 WARN_ON_ONCE(cpuc->lbr_users < 0);
372 perf_sched_cb_dec(pmu: event->pmu);
373}
374
375void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
376 struct task_struct *task, bool sched_in)
377{
378 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
379
380 /*
381 * A context switch can flip the address space and LBR entries are
382 * not tagged with an identifier. Hence, branches cannot be resolved
383 * from the old address space and the LBR records should be wiped.
384 */
385 if (cpuc->lbr_users && sched_in)
386 amd_pmu_lbr_reset();
387}
388
389void amd_pmu_lbr_enable_all(void)
390{
391 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
392 u64 lbr_select, dbg_ctl, dbg_extn_cfg;
393
394 if (!cpuc->lbr_users || !x86_pmu.lbr_nr)
395 return;
396
397 /* Set hardware branch filter */
398 if (cpuc->lbr_select) {
399 lbr_select = cpuc->lbr_sel->config & LBR_SELECT_MASK;
400 wrmsrq(MSR_AMD64_LBR_SELECT, val: lbr_select);
401 }
402
403 if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
404 rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
405 wrmsrq(MSR_IA32_DEBUGCTLMSR, val: dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
406 }
407
408 rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
409 wrmsrq(MSR_AMD_DBG_EXTN_CFG, val: dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN);
410}
411
412void amd_pmu_lbr_disable_all(void)
413{
414 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
415
416 if (!cpuc->lbr_users || !x86_pmu.lbr_nr)
417 return;
418
419 __amd_pmu_lbr_disable();
420}
421
422__init int amd_pmu_lbr_init(void)
423{
424 union cpuid_0x80000022_ebx ebx;
425
426 if (x86_pmu.version < 2 || !boot_cpu_has(X86_FEATURE_AMD_LBR_V2))
427 return -EOPNOTSUPP;
428
429 /* Set number of entries */
430 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
431 x86_pmu.lbr_nr = ebx.split.lbr_v2_stack_sz;
432
433 pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr);
434
435 return 0;
436}
437