1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Zhaoxin PMU; like Intel Architectural PerfMon-v2
4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/stddef.h>
9#include <linux/types.h>
10#include <linux/init.h>
11#include <linux/slab.h>
12#include <linux/export.h>
13#include <linux/nmi.h>
14
15#include <asm/cpufeature.h>
16#include <asm/hardirq.h>
17#include <asm/apic.h>
18#include <asm/msr.h>
19
20#include "../perf_event.h"
21
22/*
23 * Zhaoxin PerfMon, used on zxc and later.
24 */
25static u64 zx_pmon_event_map[PERF_COUNT_HW_MAX] __read_mostly = {
26
27 [PERF_COUNT_HW_CPU_CYCLES] = 0x0082,
28 [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0,
29 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0515,
30 [PERF_COUNT_HW_CACHE_MISSES] = 0x051a,
31 [PERF_COUNT_HW_BUS_CYCLES] = 0x0083,
32};
33
34static struct event_constraint zxc_event_constraints[] __read_mostly = {
35
36 FIXED_EVENT_CONSTRAINT(0x0082, 1), /* unhalted core clock cycles */
37 EVENT_CONSTRAINT_END
38};
39
40static struct event_constraint zxd_event_constraints[] __read_mostly = {
41
42 FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* retired instructions */
43 FIXED_EVENT_CONSTRAINT(0x0082, 1), /* unhalted core clock cycles */
44 FIXED_EVENT_CONSTRAINT(0x0083, 2), /* unhalted bus clock cycles */
45 EVENT_CONSTRAINT_END
46};
47
48static __initconst const u64 zxd_hw_cache_event_ids
49 [PERF_COUNT_HW_CACHE_MAX]
50 [PERF_COUNT_HW_CACHE_OP_MAX]
51 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
52[C(L1D)] = {
53 [C(OP_READ)] = {
54 [C(RESULT_ACCESS)] = 0x0042,
55 [C(RESULT_MISS)] = 0x0538,
56 },
57 [C(OP_WRITE)] = {
58 [C(RESULT_ACCESS)] = 0x0043,
59 [C(RESULT_MISS)] = 0x0562,
60 },
61 [C(OP_PREFETCH)] = {
62 [C(RESULT_ACCESS)] = -1,
63 [C(RESULT_MISS)] = -1,
64 },
65},
66[C(L1I)] = {
67 [C(OP_READ)] = {
68 [C(RESULT_ACCESS)] = 0x0300,
69 [C(RESULT_MISS)] = 0x0301,
70 },
71 [C(OP_WRITE)] = {
72 [C(RESULT_ACCESS)] = -1,
73 [C(RESULT_MISS)] = -1,
74 },
75 [C(OP_PREFETCH)] = {
76 [C(RESULT_ACCESS)] = 0x030a,
77 [C(RESULT_MISS)] = 0x030b,
78 },
79},
80[C(LL)] = {
81 [C(OP_READ)] = {
82 [C(RESULT_ACCESS)] = -1,
83 [C(RESULT_MISS)] = -1,
84 },
85 [C(OP_WRITE)] = {
86 [C(RESULT_ACCESS)] = -1,
87 [C(RESULT_MISS)] = -1,
88 },
89 [C(OP_PREFETCH)] = {
90 [C(RESULT_ACCESS)] = -1,
91 [C(RESULT_MISS)] = -1,
92 },
93},
94[C(DTLB)] = {
95 [C(OP_READ)] = {
96 [C(RESULT_ACCESS)] = 0x0042,
97 [C(RESULT_MISS)] = 0x052c,
98 },
99 [C(OP_WRITE)] = {
100 [C(RESULT_ACCESS)] = 0x0043,
101 [C(RESULT_MISS)] = 0x0530,
102 },
103 [C(OP_PREFETCH)] = {
104 [C(RESULT_ACCESS)] = 0x0564,
105 [C(RESULT_MISS)] = 0x0565,
106 },
107},
108[C(ITLB)] = {
109 [C(OP_READ)] = {
110 [C(RESULT_ACCESS)] = 0x00c0,
111 [C(RESULT_MISS)] = 0x0534,
112 },
113 [C(OP_WRITE)] = {
114 [C(RESULT_ACCESS)] = -1,
115 [C(RESULT_MISS)] = -1,
116 },
117 [C(OP_PREFETCH)] = {
118 [C(RESULT_ACCESS)] = -1,
119 [C(RESULT_MISS)] = -1,
120 },
121},
122[C(BPU)] = {
123 [C(OP_READ)] = {
124 [C(RESULT_ACCESS)] = 0x0700,
125 [C(RESULT_MISS)] = 0x0709,
126 },
127 [C(OP_WRITE)] = {
128 [C(RESULT_ACCESS)] = -1,
129 [C(RESULT_MISS)] = -1,
130 },
131 [C(OP_PREFETCH)] = {
132 [C(RESULT_ACCESS)] = -1,
133 [C(RESULT_MISS)] = -1,
134 },
135},
136[C(NODE)] = {
137 [C(OP_READ)] = {
138 [C(RESULT_ACCESS)] = -1,
139 [C(RESULT_MISS)] = -1,
140 },
141 [C(OP_WRITE)] = {
142 [C(RESULT_ACCESS)] = -1,
143 [C(RESULT_MISS)] = -1,
144 },
145 [C(OP_PREFETCH)] = {
146 [C(RESULT_ACCESS)] = -1,
147 [C(RESULT_MISS)] = -1,
148 },
149},
150};
151
152static __initconst const u64 zxe_hw_cache_event_ids
153 [PERF_COUNT_HW_CACHE_MAX]
154 [PERF_COUNT_HW_CACHE_OP_MAX]
155 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
156[C(L1D)] = {
157 [C(OP_READ)] = {
158 [C(RESULT_ACCESS)] = 0x0568,
159 [C(RESULT_MISS)] = 0x054b,
160 },
161 [C(OP_WRITE)] = {
162 [C(RESULT_ACCESS)] = 0x0669,
163 [C(RESULT_MISS)] = 0x0562,
164 },
165 [C(OP_PREFETCH)] = {
166 [C(RESULT_ACCESS)] = -1,
167 [C(RESULT_MISS)] = -1,
168 },
169},
170[C(L1I)] = {
171 [C(OP_READ)] = {
172 [C(RESULT_ACCESS)] = 0x0300,
173 [C(RESULT_MISS)] = 0x0301,
174 },
175 [C(OP_WRITE)] = {
176 [C(RESULT_ACCESS)] = -1,
177 [C(RESULT_MISS)] = -1,
178 },
179 [C(OP_PREFETCH)] = {
180 [C(RESULT_ACCESS)] = 0x030a,
181 [C(RESULT_MISS)] = 0x030b,
182 },
183},
184[C(LL)] = {
185 [C(OP_READ)] = {
186 [C(RESULT_ACCESS)] = 0x0,
187 [C(RESULT_MISS)] = 0x0,
188 },
189 [C(OP_WRITE)] = {
190 [C(RESULT_ACCESS)] = 0x0,
191 [C(RESULT_MISS)] = 0x0,
192 },
193 [C(OP_PREFETCH)] = {
194 [C(RESULT_ACCESS)] = 0x0,
195 [C(RESULT_MISS)] = 0x0,
196 },
197},
198[C(DTLB)] = {
199 [C(OP_READ)] = {
200 [C(RESULT_ACCESS)] = 0x0568,
201 [C(RESULT_MISS)] = 0x052c,
202 },
203 [C(OP_WRITE)] = {
204 [C(RESULT_ACCESS)] = 0x0669,
205 [C(RESULT_MISS)] = 0x0530,
206 },
207 [C(OP_PREFETCH)] = {
208 [C(RESULT_ACCESS)] = 0x0564,
209 [C(RESULT_MISS)] = 0x0565,
210 },
211},
212[C(ITLB)] = {
213 [C(OP_READ)] = {
214 [C(RESULT_ACCESS)] = 0x00c0,
215 [C(RESULT_MISS)] = 0x0534,
216 },
217 [C(OP_WRITE)] = {
218 [C(RESULT_ACCESS)] = -1,
219 [C(RESULT_MISS)] = -1,
220 },
221 [C(OP_PREFETCH)] = {
222 [C(RESULT_ACCESS)] = -1,
223 [C(RESULT_MISS)] = -1,
224 },
225},
226[C(BPU)] = {
227 [C(OP_READ)] = {
228 [C(RESULT_ACCESS)] = 0x0028,
229 [C(RESULT_MISS)] = 0x0029,
230 },
231 [C(OP_WRITE)] = {
232 [C(RESULT_ACCESS)] = -1,
233 [C(RESULT_MISS)] = -1,
234 },
235 [C(OP_PREFETCH)] = {
236 [C(RESULT_ACCESS)] = -1,
237 [C(RESULT_MISS)] = -1,
238 },
239},
240[C(NODE)] = {
241 [C(OP_READ)] = {
242 [C(RESULT_ACCESS)] = -1,
243 [C(RESULT_MISS)] = -1,
244 },
245 [C(OP_WRITE)] = {
246 [C(RESULT_ACCESS)] = -1,
247 [C(RESULT_MISS)] = -1,
248 },
249 [C(OP_PREFETCH)] = {
250 [C(RESULT_ACCESS)] = -1,
251 [C(RESULT_MISS)] = -1,
252 },
253},
254};
255
256static void zhaoxin_pmu_disable_all(void)
257{
258 wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, val: 0);
259}
260
261static void zhaoxin_pmu_enable_all(int added)
262{
263 wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, val: x86_pmu.intel_ctrl);
264}
265
266static inline u64 zhaoxin_pmu_get_status(void)
267{
268 u64 status;
269
270 rdmsrq(MSR_CORE_PERF_GLOBAL_STATUS, status);
271
272 return status;
273}
274
275static inline void zhaoxin_pmu_ack_status(u64 ack)
276{
277 wrmsrq(MSR_CORE_PERF_GLOBAL_OVF_CTRL, val: ack);
278}
279
280static inline void zxc_pmu_ack_status(u64 ack)
281{
282 /*
283 * ZXC needs global control enabled in order to clear status bits.
284 */
285 zhaoxin_pmu_enable_all(added: 0);
286 zhaoxin_pmu_ack_status(ack);
287 zhaoxin_pmu_disable_all();
288}
289
290static void zhaoxin_pmu_disable_fixed(struct hw_perf_event *hwc)
291{
292 int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
293 u64 ctrl_val, mask;
294
295 mask = 0xfULL << (idx * 4);
296
297 rdmsrq(hwc->config_base, ctrl_val);
298 ctrl_val &= ~mask;
299 wrmsrq(msr: hwc->config_base, val: ctrl_val);
300}
301
302static void zhaoxin_pmu_disable_event(struct perf_event *event)
303{
304 struct hw_perf_event *hwc = &event->hw;
305
306 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
307 zhaoxin_pmu_disable_fixed(hwc);
308 return;
309 }
310
311 x86_pmu_disable_event(event);
312}
313
314static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc)
315{
316 int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
317 u64 ctrl_val, bits, mask;
318
319 /*
320 * Enable IRQ generation (0x8),
321 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
322 * if requested:
323 */
324 bits = 0x8ULL;
325 if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
326 bits |= 0x2;
327 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
328 bits |= 0x1;
329
330 bits <<= (idx * 4);
331 mask = 0xfULL << (idx * 4);
332
333 rdmsrq(hwc->config_base, ctrl_val);
334 ctrl_val &= ~mask;
335 ctrl_val |= bits;
336 wrmsrq(msr: hwc->config_base, val: ctrl_val);
337}
338
339static void zhaoxin_pmu_enable_event(struct perf_event *event)
340{
341 struct hw_perf_event *hwc = &event->hw;
342
343 if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
344 zhaoxin_pmu_enable_fixed(hwc);
345 return;
346 }
347
348 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
349}
350
351/*
352 * This handler is triggered by the local APIC, so the APIC IRQ handling
353 * rules apply:
354 */
355static int zhaoxin_pmu_handle_irq(struct pt_regs *regs)
356{
357 struct perf_sample_data data;
358 struct cpu_hw_events *cpuc;
359 int handled = 0;
360 u64 status;
361 int bit;
362
363 cpuc = this_cpu_ptr(&cpu_hw_events);
364 apic_write(APIC_LVTPC, APIC_DM_NMI);
365 zhaoxin_pmu_disable_all();
366 status = zhaoxin_pmu_get_status();
367 if (!status)
368 goto done;
369
370again:
371 if (x86_pmu.enabled_ack)
372 zxc_pmu_ack_status(ack: status);
373 else
374 zhaoxin_pmu_ack_status(ack: status);
375
376 inc_irq_stat(apic_perf_irqs);
377
378 /*
379 * CondChgd bit 63 doesn't mean any overflow status. Ignore
380 * and clear the bit.
381 */
382 if (__test_and_clear_bit(63, (unsigned long *)&status)) {
383 if (!status)
384 goto done;
385 }
386
387 for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
388 struct perf_event *event = cpuc->events[bit];
389
390 handled++;
391
392 if (!test_bit(bit, cpuc->active_mask))
393 continue;
394
395 x86_perf_event_update(event);
396 perf_sample_data_init(data: &data, addr: 0, period: event->hw.last_period);
397
398 if (!x86_perf_event_set_period(event))
399 continue;
400
401 perf_event_overflow(event, data: &data, regs);
402 }
403
404 /*
405 * Repeat if there is more work to be done:
406 */
407 status = zhaoxin_pmu_get_status();
408 if (status)
409 goto again;
410
411done:
412 zhaoxin_pmu_enable_all(added: 0);
413 return handled;
414}
415
416static u64 zhaoxin_pmu_event_map(int hw_event)
417{
418 return zx_pmon_event_map[hw_event];
419}
420
421static struct event_constraint *
422zhaoxin_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
423 struct perf_event *event)
424{
425 struct event_constraint *c;
426
427 if (x86_pmu.event_constraints) {
428 for_each_event_constraint(c, x86_pmu.event_constraints) {
429 if ((event->hw.config & c->cmask) == c->code)
430 return c;
431 }
432 }
433
434 return &unconstrained;
435}
436
437PMU_FORMAT_ATTR(event, "config:0-7");
438PMU_FORMAT_ATTR(umask, "config:8-15");
439PMU_FORMAT_ATTR(edge, "config:18");
440PMU_FORMAT_ATTR(inv, "config:23");
441PMU_FORMAT_ATTR(cmask, "config:24-31");
442
443static struct attribute *zx_arch_formats_attr[] = {
444 &format_attr_event.attr,
445 &format_attr_umask.attr,
446 &format_attr_edge.attr,
447 &format_attr_inv.attr,
448 &format_attr_cmask.attr,
449 NULL,
450};
451
452static ssize_t zhaoxin_event_sysfs_show(char *page, u64 config)
453{
454 u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);
455
456 return x86_event_sysfs_show(page, config, event);
457}
458
459static const struct x86_pmu zhaoxin_pmu __initconst = {
460 .name = "zhaoxin",
461 .handle_irq = zhaoxin_pmu_handle_irq,
462 .disable_all = zhaoxin_pmu_disable_all,
463 .enable_all = zhaoxin_pmu_enable_all,
464 .enable = zhaoxin_pmu_enable_event,
465 .disable = zhaoxin_pmu_disable_event,
466 .hw_config = x86_pmu_hw_config,
467 .schedule_events = x86_schedule_events,
468 .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
469 .perfctr = MSR_ARCH_PERFMON_PERFCTR0,
470 .event_map = zhaoxin_pmu_event_map,
471 .max_events = ARRAY_SIZE(zx_pmon_event_map),
472 .apic = 1,
473 /*
474 * For zxd/zxe, read/write operation for PMCx MSR is 48 bits.
475 */
476 .max_period = (1ULL << 47) - 1,
477 .get_event_constraints = zhaoxin_get_event_constraints,
478
479 .format_attrs = zx_arch_formats_attr,
480 .events_sysfs_show = zhaoxin_event_sysfs_show,
481};
482
483static const struct { int id; char *name; } zx_arch_events_map[] __initconst = {
484 { PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" },
485 { .id: PERF_COUNT_HW_INSTRUCTIONS, .name: "instructions" },
486 { .id: PERF_COUNT_HW_BUS_CYCLES, .name: "bus cycles" },
487 { .id: PERF_COUNT_HW_CACHE_REFERENCES, .name: "cache references" },
488 { .id: PERF_COUNT_HW_CACHE_MISSES, .name: "cache misses" },
489 { .id: PERF_COUNT_HW_BRANCH_INSTRUCTIONS, .name: "branch instructions" },
490 { .id: PERF_COUNT_HW_BRANCH_MISSES, .name: "branch misses" },
491};
492
493static __init void zhaoxin_arch_events_quirk(void)
494{
495 int bit;
496
497 /* disable event that reported as not present by cpuid */
498 for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(zx_arch_events_map)) {
499 zx_pmon_event_map[zx_arch_events_map[bit].id] = 0;
500 pr_warn("CPUID marked event: \'%s\' unavailable\n",
501 zx_arch_events_map[bit].name);
502 }
503}
504
505__init int zhaoxin_pmu_init(void)
506{
507 union cpuid10_edx edx;
508 union cpuid10_eax eax;
509 union cpuid10_ebx ebx;
510 struct event_constraint *c;
511 unsigned int unused;
512 int version;
513
514 pr_info("Welcome to zhaoxin pmu!\n");
515
516 /*
517 * Check whether the Architectural PerfMon supports
518 * hw_event or not.
519 */
520 cpuid(op: 10, eax: &eax.full, ebx: &ebx.full, ecx: &unused, edx: &edx.full);
521
522 if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT - 1)
523 return -ENODEV;
524
525 version = eax.split.version_id;
526 if (version != 2)
527 return -ENODEV;
528
529 x86_pmu = zhaoxin_pmu;
530 pr_info("Version check pass!\n");
531
532 x86_pmu.version = version;
533 x86_pmu.cntr_mask64 = GENMASK_ULL(eax.split.num_counters - 1, 0);
534 x86_pmu.cntval_bits = eax.split.bit_width;
535 x86_pmu.cntval_mask = (1ULL << eax.split.bit_width) - 1;
536 x86_pmu.events_maskl = ebx.full;
537 x86_pmu.events_mask_len = eax.split.mask_length;
538
539 x86_pmu.fixed_cntr_mask64 = GENMASK_ULL(edx.split.num_counters_fixed - 1, 0);
540 x86_add_quirk(zhaoxin_arch_events_quirk);
541
542 switch (boot_cpu_data.x86) {
543 case 0x06:
544 /*
545 * Support Zhaoxin CPU from ZXC series, exclude Nano series through FMS.
546 * Nano FMS: Family=6, Model=F, Stepping=[0-A][C-D]
547 * ZXC FMS: Family=6, Model=F, Stepping=E-F OR Family=6, Model=0x19, Stepping=0-3
548 */
549 if ((boot_cpu_data.x86_model == 0x0f && boot_cpu_data.x86_stepping >= 0x0e) ||
550 boot_cpu_data.x86_model == 0x19) {
551
552 x86_pmu.max_period = x86_pmu.cntval_mask >> 1;
553
554 /* Clearing status works only if the global control is enable on zxc. */
555 x86_pmu.enabled_ack = 1;
556
557 x86_pmu.event_constraints = zxc_event_constraints;
558 zx_pmon_event_map[PERF_COUNT_HW_INSTRUCTIONS] = 0;
559 zx_pmon_event_map[PERF_COUNT_HW_CACHE_REFERENCES] = 0;
560 zx_pmon_event_map[PERF_COUNT_HW_CACHE_MISSES] = 0;
561 zx_pmon_event_map[PERF_COUNT_HW_BUS_CYCLES] = 0;
562
563 pr_cont("ZXC events, ");
564 break;
565 }
566 return -ENODEV;
567
568 case 0x07:
569 zx_pmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
570 X86_CONFIG(.event = 0x01, .umask = 0x01, .inv = 0x01, .cmask = 0x01);
571
572 zx_pmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
573 X86_CONFIG(.event = 0x0f, .umask = 0x04, .inv = 0, .cmask = 0);
574
575 switch (boot_cpu_data.x86_model) {
576 case 0x1b:
577 memcpy(to: hw_cache_event_ids, from: zxd_hw_cache_event_ids,
578 len: sizeof(hw_cache_event_ids));
579
580 x86_pmu.event_constraints = zxd_event_constraints;
581
582 zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0700;
583 zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x0709;
584
585 pr_cont("ZXD events, ");
586 break;
587 case 0x3b:
588 memcpy(to: hw_cache_event_ids, from: zxe_hw_cache_event_ids,
589 len: sizeof(hw_cache_event_ids));
590
591 x86_pmu.event_constraints = zxd_event_constraints;
592
593 zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0028;
594 zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x0029;
595
596 pr_cont("ZXE events, ");
597 break;
598 default:
599 return -ENODEV;
600 }
601 break;
602
603 default:
604 return -ENODEV;
605 }
606
607 x86_pmu.intel_ctrl = x86_pmu.cntr_mask64;
608 x86_pmu.intel_ctrl |= x86_pmu.fixed_cntr_mask64 << INTEL_PMC_IDX_FIXED;
609
610 if (x86_pmu.event_constraints) {
611 for_each_event_constraint(c, x86_pmu.event_constraints) {
612 c->idxmsk64 |= x86_pmu.cntr_mask64;
613 c->weight += x86_pmu_num_counters(NULL);
614 }
615 }
616
617 return 0;
618}
619
620