1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2013 Advanced Micro Devices, Inc.
4 *
5 * Author: Jacob Shin <jacob.shin@amd.com>
6 */
7
8#include <linux/perf_event.h>
9#include <linux/percpu.h>
10#include <linux/types.h>
11#include <linux/slab.h>
12#include <linux/init.h>
13#include <linux/cpu.h>
14#include <linux/cpumask.h>
15#include <linux/cpufeature.h>
16#include <linux/smp.h>
17
18#include <asm/perf_event.h>
19#include <asm/msr.h>
20
21#define NUM_COUNTERS_NB 4
22#define NUM_COUNTERS_L2 4
23#define NUM_COUNTERS_L3 6
24#define NUM_COUNTERS_MAX 64
25
26#define RDPMC_BASE_NB 6
27#define RDPMC_BASE_LLC 10
28
29#define COUNTER_SHIFT 16
30#define UNCORE_NAME_LEN 16
31#define UNCORE_GROUP_MAX 256
32
33#undef pr_fmt
34#define pr_fmt(fmt) "amd_uncore: " fmt
35
36static int pmu_version;
37
38struct amd_uncore_ctx {
39 int refcnt;
40 int cpu;
41 struct perf_event **events;
42 unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)];
43 int nr_active;
44 struct hrtimer hrtimer;
45 u64 hrtimer_duration;
46};
47
48struct amd_uncore_pmu {
49 char name[UNCORE_NAME_LEN];
50 int num_counters;
51 int rdpmc_base;
52 u32 msr_base;
53 int group;
54 cpumask_t active_mask;
55 struct pmu pmu;
56 struct amd_uncore_ctx * __percpu *ctx;
57};
58
59enum {
60 UNCORE_TYPE_DF,
61 UNCORE_TYPE_L3,
62 UNCORE_TYPE_UMC,
63
64 UNCORE_TYPE_MAX
65};
66
67union amd_uncore_info {
68 struct {
69 u64 aux_data:32; /* auxiliary data */
70 u64 num_pmcs:8; /* number of counters */
71 u64 gid:8; /* group id */
72 u64 cid:8; /* context id */
73 } split;
74 u64 full;
75};
76
77struct amd_uncore {
78 union amd_uncore_info __percpu *info;
79 struct amd_uncore_pmu *pmus;
80 unsigned int num_pmus;
81 bool init_done;
82 void (*scan)(struct amd_uncore *uncore, unsigned int cpu);
83 int (*init)(struct amd_uncore *uncore, unsigned int cpu);
84 void (*move)(struct amd_uncore *uncore, unsigned int cpu);
85 void (*free)(struct amd_uncore *uncore, unsigned int cpu);
86};
87
88static struct amd_uncore uncores[UNCORE_TYPE_MAX];
89
90/* Interval for hrtimer, defaults to 60000 milliseconds */
91static unsigned int update_interval = 60 * MSEC_PER_SEC;
92module_param(update_interval, uint, 0444);
93
94static struct amd_uncore_pmu *event_to_amd_uncore_pmu(struct perf_event *event)
95{
96 return container_of(event->pmu, struct amd_uncore_pmu, pmu);
97}
98
99static enum hrtimer_restart amd_uncore_hrtimer(struct hrtimer *hrtimer)
100{
101 struct amd_uncore_ctx *ctx;
102 struct perf_event *event;
103 int bit;
104
105 ctx = container_of(hrtimer, struct amd_uncore_ctx, hrtimer);
106
107 if (!ctx->nr_active || ctx->cpu != smp_processor_id())
108 return HRTIMER_NORESTART;
109
110 for_each_set_bit(bit, ctx->active_mask, NUM_COUNTERS_MAX) {
111 event = ctx->events[bit];
112 event->pmu->read(event);
113 }
114
115 hrtimer_forward_now(timer: hrtimer, interval: ns_to_ktime(ns: ctx->hrtimer_duration));
116 return HRTIMER_RESTART;
117}
118
119static void amd_uncore_start_hrtimer(struct amd_uncore_ctx *ctx)
120{
121 hrtimer_start(timer: &ctx->hrtimer, tim: ns_to_ktime(ns: ctx->hrtimer_duration),
122 mode: HRTIMER_MODE_REL_PINNED_HARD);
123}
124
125static void amd_uncore_cancel_hrtimer(struct amd_uncore_ctx *ctx)
126{
127 hrtimer_cancel(timer: &ctx->hrtimer);
128}
129
130static void amd_uncore_init_hrtimer(struct amd_uncore_ctx *ctx)
131{
132 hrtimer_setup(timer: &ctx->hrtimer, function: amd_uncore_hrtimer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL_HARD);
133}
134
135static void amd_uncore_read(struct perf_event *event)
136{
137 struct hw_perf_event *hwc = &event->hw;
138 u64 prev, new;
139 s64 delta;
140
141 /*
142 * since we do not enable counter overflow interrupts,
143 * we do not have to worry about prev_count changing on us
144 */
145
146 prev = local64_read(&hwc->prev_count);
147
148 /*
149 * Some uncore PMUs do not have RDPMC assignments. In such cases,
150 * read counts directly from the corresponding PERF_CTR.
151 */
152 if (hwc->event_base_rdpmc < 0)
153 rdmsrq(hwc->event_base, new);
154 else
155 new = rdpmc(counter: hwc->event_base_rdpmc);
156
157 local64_set(&hwc->prev_count, new);
158 delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
159 delta >>= COUNTER_SHIFT;
160 local64_add(delta, &event->count);
161}
162
163static void amd_uncore_start(struct perf_event *event, int flags)
164{
165 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
166 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
167 struct hw_perf_event *hwc = &event->hw;
168
169 if (!ctx->nr_active++)
170 amd_uncore_start_hrtimer(ctx);
171
172 if (flags & PERF_EF_RELOAD)
173 wrmsrq(msr: hwc->event_base, val: (u64)local64_read(&hwc->prev_count));
174
175 hwc->state = 0;
176 __set_bit(hwc->idx, ctx->active_mask);
177 wrmsrq(msr: hwc->config_base, val: (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
178 perf_event_update_userpage(event);
179}
180
181static void amd_uncore_stop(struct perf_event *event, int flags)
182{
183 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
184 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
185 struct hw_perf_event *hwc = &event->hw;
186
187 wrmsrq(msr: hwc->config_base, val: hwc->config);
188 hwc->state |= PERF_HES_STOPPED;
189
190 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
191 event->pmu->read(event);
192 hwc->state |= PERF_HES_UPTODATE;
193 }
194
195 if (!--ctx->nr_active)
196 amd_uncore_cancel_hrtimer(ctx);
197
198 __clear_bit(hwc->idx, ctx->active_mask);
199}
200
201static int amd_uncore_add(struct perf_event *event, int flags)
202{
203 int i;
204 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
205 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
206 struct hw_perf_event *hwc = &event->hw;
207
208 /* are we already assigned? */
209 if (hwc->idx != -1 && ctx->events[hwc->idx] == event)
210 goto out;
211
212 for (i = 0; i < pmu->num_counters; i++) {
213 if (ctx->events[i] == event) {
214 hwc->idx = i;
215 goto out;
216 }
217 }
218
219 /* if not, take the first available counter */
220 hwc->idx = -1;
221 for (i = 0; i < pmu->num_counters; i++) {
222 struct perf_event *tmp = NULL;
223
224 if (try_cmpxchg(&ctx->events[i], &tmp, event)) {
225 hwc->idx = i;
226 break;
227 }
228 }
229
230out:
231 if (hwc->idx == -1)
232 return -EBUSY;
233
234 hwc->config_base = pmu->msr_base + (2 * hwc->idx);
235 hwc->event_base = pmu->msr_base + 1 + (2 * hwc->idx);
236 hwc->event_base_rdpmc = pmu->rdpmc_base + hwc->idx;
237 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
238
239 if (pmu->rdpmc_base < 0)
240 hwc->event_base_rdpmc = -1;
241
242 if (flags & PERF_EF_START)
243 event->pmu->start(event, PERF_EF_RELOAD);
244
245 return 0;
246}
247
248static void amd_uncore_del(struct perf_event *event, int flags)
249{
250 int i;
251 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
252 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
253 struct hw_perf_event *hwc = &event->hw;
254
255 event->pmu->stop(event, PERF_EF_UPDATE);
256
257 for (i = 0; i < pmu->num_counters; i++) {
258 struct perf_event *tmp = event;
259
260 if (try_cmpxchg(&ctx->events[i], &tmp, NULL))
261 break;
262 }
263
264 hwc->idx = -1;
265}
266
267static int amd_uncore_event_init(struct perf_event *event)
268{
269 struct amd_uncore_pmu *pmu;
270 struct amd_uncore_ctx *ctx;
271 struct hw_perf_event *hwc = &event->hw;
272
273 if (event->attr.type != event->pmu->type)
274 return -ENOENT;
275
276 if (event->cpu < 0)
277 return -EINVAL;
278
279 pmu = event_to_amd_uncore_pmu(event);
280 ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
281 if (!ctx)
282 return -ENODEV;
283
284 /*
285 * NB and Last level cache counters (MSRs) are shared across all cores
286 * that share the same NB / Last level cache. On family 16h and below,
287 * Interrupts can be directed to a single target core, however, event
288 * counts generated by processes running on other cores cannot be masked
289 * out. So we do not support sampling and per-thread events via
290 * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts:
291 */
292 hwc->config = event->attr.config;
293 hwc->idx = -1;
294
295 /*
296 * since request can come in to any of the shared cores, we will remap
297 * to a single common cpu.
298 */
299 event->cpu = ctx->cpu;
300
301 return 0;
302}
303
304static umode_t
305amd_f17h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i)
306{
307 return boot_cpu_data.x86 >= 0x17 && boot_cpu_data.x86 < 0x19 ?
308 attr->mode : 0;
309}
310
311static umode_t
312amd_f19h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i)
313{
314 return boot_cpu_data.x86 >= 0x19 ? attr->mode : 0;
315}
316
317static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
318 struct device_attribute *attr,
319 char *buf)
320{
321 struct pmu *ptr = dev_get_drvdata(dev);
322 struct amd_uncore_pmu *pmu = container_of(ptr, struct amd_uncore_pmu, pmu);
323
324 return cpumap_print_to_pagebuf(list: true, buf, mask: &pmu->active_mask);
325}
326static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
327
328static struct attribute *amd_uncore_attrs[] = {
329 &dev_attr_cpumask.attr,
330 NULL,
331};
332
333static struct attribute_group amd_uncore_attr_group = {
334 .attrs = amd_uncore_attrs,
335};
336
337#define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \
338static ssize_t __uncore_##_var##_show(struct device *dev, \
339 struct device_attribute *attr, \
340 char *page) \
341{ \
342 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
343 return sprintf(page, _format "\n"); \
344} \
345static struct device_attribute format_attr_##_var = \
346 __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
347
348DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35");
349DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */
350DEFINE_UNCORE_FORMAT_ATTR(event14v2, event, "config:0-7,32-37"); /* PerfMonV2 DF */
351DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3, PerfMonV2 UMC */
352DEFINE_UNCORE_FORMAT_ATTR(umask8, umask, "config:8-15");
353DEFINE_UNCORE_FORMAT_ATTR(umask12, umask, "config:8-15,24-27"); /* PerfMonV2 DF */
354DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */
355DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */
356DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */
357DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L3 */
358DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */
359DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */
360DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */
361DEFINE_UNCORE_FORMAT_ATTR(rdwrmask, rdwrmask, "config:8-9"); /* PerfMonV2 UMC */
362
363/* Common DF and NB attributes */
364static struct attribute *amd_uncore_df_format_attr[] = {
365 &format_attr_event12.attr, /* event */
366 &format_attr_umask8.attr, /* umask */
367 NULL,
368};
369
370/* Common L2 and L3 attributes */
371static struct attribute *amd_uncore_l3_format_attr[] = {
372 &format_attr_event12.attr, /* event */
373 &format_attr_umask8.attr, /* umask */
374 NULL, /* threadmask */
375 NULL,
376};
377
378/* Common UMC attributes */
379static struct attribute *amd_uncore_umc_format_attr[] = {
380 &format_attr_event8.attr, /* event */
381 &format_attr_rdwrmask.attr, /* rdwrmask */
382 NULL,
383};
384
385/* F17h unique L3 attributes */
386static struct attribute *amd_f17h_uncore_l3_format_attr[] = {
387 &format_attr_slicemask.attr, /* slicemask */
388 NULL,
389};
390
391/* F19h unique L3 attributes */
392static struct attribute *amd_f19h_uncore_l3_format_attr[] = {
393 &format_attr_coreid.attr, /* coreid */
394 &format_attr_enallslices.attr, /* enallslices */
395 &format_attr_enallcores.attr, /* enallcores */
396 &format_attr_sliceid.attr, /* sliceid */
397 NULL,
398};
399
400static struct attribute_group amd_uncore_df_format_group = {
401 .name = "format",
402 .attrs = amd_uncore_df_format_attr,
403};
404
405static struct attribute_group amd_uncore_l3_format_group = {
406 .name = "format",
407 .attrs = amd_uncore_l3_format_attr,
408};
409
410static struct attribute_group amd_f17h_uncore_l3_format_group = {
411 .name = "format",
412 .attrs = amd_f17h_uncore_l3_format_attr,
413 .is_visible = amd_f17h_uncore_is_visible,
414};
415
416static struct attribute_group amd_f19h_uncore_l3_format_group = {
417 .name = "format",
418 .attrs = amd_f19h_uncore_l3_format_attr,
419 .is_visible = amd_f19h_uncore_is_visible,
420};
421
422static struct attribute_group amd_uncore_umc_format_group = {
423 .name = "format",
424 .attrs = amd_uncore_umc_format_attr,
425};
426
427static const struct attribute_group *amd_uncore_df_attr_groups[] = {
428 &amd_uncore_attr_group,
429 &amd_uncore_df_format_group,
430 NULL,
431};
432
433static const struct attribute_group *amd_uncore_l3_attr_groups[] = {
434 &amd_uncore_attr_group,
435 &amd_uncore_l3_format_group,
436 NULL,
437};
438
439static const struct attribute_group *amd_uncore_l3_attr_update[] = {
440 &amd_f17h_uncore_l3_format_group,
441 &amd_f19h_uncore_l3_format_group,
442 NULL,
443};
444
445static const struct attribute_group *amd_uncore_umc_attr_groups[] = {
446 &amd_uncore_attr_group,
447 &amd_uncore_umc_format_group,
448 NULL,
449};
450
451static __always_inline
452int amd_uncore_ctx_cid(struct amd_uncore *uncore, unsigned int cpu)
453{
454 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
455 return info->split.cid;
456}
457
458static __always_inline
459int amd_uncore_ctx_gid(struct amd_uncore *uncore, unsigned int cpu)
460{
461 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
462 return info->split.gid;
463}
464
465static __always_inline
466int amd_uncore_ctx_num_pmcs(struct amd_uncore *uncore, unsigned int cpu)
467{
468 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
469 return info->split.num_pmcs;
470}
471
472static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu)
473{
474 struct amd_uncore_pmu *pmu;
475 struct amd_uncore_ctx *ctx;
476 int i;
477
478 if (!uncore->init_done)
479 return;
480
481 for (i = 0; i < uncore->num_pmus; i++) {
482 pmu = &uncore->pmus[i];
483 ctx = *per_cpu_ptr(pmu->ctx, cpu);
484 if (!ctx)
485 continue;
486
487 if (cpu == ctx->cpu)
488 cpumask_clear_cpu(cpu, dstp: &pmu->active_mask);
489
490 if (!--ctx->refcnt) {
491 kfree(objp: ctx->events);
492 kfree(objp: ctx);
493 }
494
495 *per_cpu_ptr(pmu->ctx, cpu) = NULL;
496 }
497}
498
499static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
500{
501 struct amd_uncore_ctx *curr, *prev;
502 struct amd_uncore_pmu *pmu;
503 int node, cid, gid, i, j;
504
505 if (!uncore->init_done || !uncore->num_pmus)
506 return 0;
507
508 cid = amd_uncore_ctx_cid(uncore, cpu);
509 gid = amd_uncore_ctx_gid(uncore, cpu);
510
511 for (i = 0; i < uncore->num_pmus; i++) {
512 pmu = &uncore->pmus[i];
513 *per_cpu_ptr(pmu->ctx, cpu) = NULL;
514 curr = NULL;
515
516 /* Check for group exclusivity */
517 if (gid != pmu->group)
518 continue;
519
520 /* Find a sibling context */
521 for_each_online_cpu(j) {
522 if (cpu == j)
523 continue;
524
525 prev = *per_cpu_ptr(pmu->ctx, j);
526 if (!prev)
527 continue;
528
529 if (cid == amd_uncore_ctx_cid(uncore, cpu: j)) {
530 curr = prev;
531 break;
532 }
533 }
534
535 /* Allocate context if sibling does not exist */
536 if (!curr) {
537 node = cpu_to_node(cpu);
538 curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node);
539 if (!curr)
540 goto fail;
541
542 curr->cpu = cpu;
543 curr->events = kzalloc_node(sizeof(*curr->events) *
544 pmu->num_counters,
545 GFP_KERNEL, node);
546 if (!curr->events) {
547 kfree(objp: curr);
548 goto fail;
549 }
550
551 amd_uncore_init_hrtimer(ctx: curr);
552 curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC;
553
554 cpumask_set_cpu(cpu, dstp: &pmu->active_mask);
555 }
556
557 curr->refcnt++;
558 *per_cpu_ptr(pmu->ctx, cpu) = curr;
559 }
560
561 return 0;
562
563fail:
564 amd_uncore_ctx_free(uncore, cpu);
565
566 return -ENOMEM;
567}
568
569static void amd_uncore_ctx_move(struct amd_uncore *uncore, unsigned int cpu)
570{
571 struct amd_uncore_ctx *curr, *next;
572 struct amd_uncore_pmu *pmu;
573 int i, j;
574
575 if (!uncore->init_done)
576 return;
577
578 for (i = 0; i < uncore->num_pmus; i++) {
579 pmu = &uncore->pmus[i];
580 curr = *per_cpu_ptr(pmu->ctx, cpu);
581 if (!curr)
582 continue;
583
584 /* Migrate to a shared sibling if possible */
585 for_each_online_cpu(j) {
586 next = *per_cpu_ptr(pmu->ctx, j);
587 if (!next || cpu == j)
588 continue;
589
590 if (curr == next) {
591 perf_pmu_migrate_context(pmu: &pmu->pmu, src_cpu: cpu, dst_cpu: j);
592 cpumask_clear_cpu(cpu, dstp: &pmu->active_mask);
593 cpumask_set_cpu(cpu: j, dstp: &pmu->active_mask);
594 next->cpu = j;
595 break;
596 }
597 }
598 }
599}
600
601static int amd_uncore_cpu_starting(unsigned int cpu)
602{
603 struct amd_uncore *uncore;
604 int i;
605
606 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
607 uncore = &uncores[i];
608 uncore->scan(uncore, cpu);
609 }
610
611 return 0;
612}
613
614static int amd_uncore_cpu_online(unsigned int cpu)
615{
616 struct amd_uncore *uncore;
617 int i;
618
619 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
620 uncore = &uncores[i];
621 if (uncore->init(uncore, cpu))
622 break;
623 }
624
625 return 0;
626}
627
628static int amd_uncore_cpu_down_prepare(unsigned int cpu)
629{
630 struct amd_uncore *uncore;
631 int i;
632
633 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
634 uncore = &uncores[i];
635 uncore->move(uncore, cpu);
636 }
637
638 return 0;
639}
640
641static int amd_uncore_cpu_dead(unsigned int cpu)
642{
643 struct amd_uncore *uncore;
644 int i;
645
646 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
647 uncore = &uncores[i];
648 uncore->free(uncore, cpu);
649 }
650
651 return 0;
652}
653
654static int amd_uncore_df_event_init(struct perf_event *event)
655{
656 struct hw_perf_event *hwc = &event->hw;
657 int ret = amd_uncore_event_init(event);
658
659 if (ret || pmu_version < 2)
660 return ret;
661
662 hwc->config = event->attr.config &
663 (pmu_version >= 2 ? AMD64_PERFMON_V2_RAW_EVENT_MASK_NB :
664 AMD64_RAW_EVENT_MASK_NB);
665
666 return 0;
667}
668
669static int amd_uncore_df_add(struct perf_event *event, int flags)
670{
671 int ret = amd_uncore_add(event, flags: flags & ~PERF_EF_START);
672 struct hw_perf_event *hwc = &event->hw;
673
674 if (ret)
675 return ret;
676
677 /*
678 * The first four DF counters are accessible via RDPMC index 6 to 9
679 * followed by the L3 counters from index 10 to 15. For processors
680 * with more than four DF counters, the DF RDPMC assignments become
681 * discontiguous as the additional counters are accessible starting
682 * from index 16.
683 */
684 if (hwc->idx >= NUM_COUNTERS_NB)
685 hwc->event_base_rdpmc += NUM_COUNTERS_L3;
686
687 /* Delayed start after rdpmc base update */
688 if (flags & PERF_EF_START)
689 amd_uncore_start(event, PERF_EF_RELOAD);
690
691 return 0;
692}
693
694static
695void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
696{
697 union cpuid_0x80000022_ebx ebx;
698 union amd_uncore_info info;
699
700 if (!boot_cpu_has(X86_FEATURE_PERFCTR_NB))
701 return;
702
703 info.split.aux_data = 0;
704 info.split.num_pmcs = NUM_COUNTERS_NB;
705 info.split.gid = 0;
706 info.split.cid = topology_logical_package_id(cpu);
707
708 if (pmu_version >= 2) {
709 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
710 info.split.num_pmcs = ebx.split.num_df_pmc;
711 }
712
713 *per_cpu_ptr(uncore->info, cpu) = info;
714}
715
716static
717int amd_uncore_df_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
718{
719 struct attribute **df_attr = amd_uncore_df_format_attr;
720 struct amd_uncore_pmu *pmu;
721 int num_counters;
722
723 /* Run just once */
724 if (uncore->init_done)
725 return amd_uncore_ctx_init(uncore, cpu);
726
727 num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);
728 if (!num_counters)
729 goto done;
730
731 /* No grouping, single instance for a system */
732 uncore->pmus = kzalloc(sizeof(*uncore->pmus), GFP_KERNEL);
733 if (!uncore->pmus)
734 goto done;
735
736 /*
737 * For Family 17h and above, the Northbridge counters are repurposed
738 * as Data Fabric counters. The PMUs are exported based on family as
739 * either NB or DF.
740 */
741 pmu = &uncore->pmus[0];
742 strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_df" : "amd_nb",
743 sizeof(pmu->name));
744 pmu->num_counters = num_counters;
745 pmu->msr_base = MSR_F15H_NB_PERF_CTL;
746 pmu->rdpmc_base = RDPMC_BASE_NB;
747 pmu->group = amd_uncore_ctx_gid(uncore, cpu);
748
749 if (pmu_version >= 2) {
750 *df_attr++ = &format_attr_event14v2.attr;
751 *df_attr++ = &format_attr_umask12.attr;
752 } else if (boot_cpu_data.x86 >= 0x17) {
753 *df_attr = &format_attr_event14.attr;
754 }
755
756 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
757 if (!pmu->ctx)
758 goto done;
759
760 pmu->pmu = (struct pmu) {
761 .task_ctx_nr = perf_invalid_context,
762 .attr_groups = amd_uncore_df_attr_groups,
763 .name = pmu->name,
764 .event_init = amd_uncore_df_event_init,
765 .add = amd_uncore_df_add,
766 .del = amd_uncore_del,
767 .start = amd_uncore_start,
768 .stop = amd_uncore_stop,
769 .read = amd_uncore_read,
770 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
771 .module = THIS_MODULE,
772 };
773
774 if (perf_pmu_register(pmu: &pmu->pmu, name: pmu->pmu.name, type: -1)) {
775 free_percpu(pdata: pmu->ctx);
776 pmu->ctx = NULL;
777 goto done;
778 }
779
780 pr_info("%d %s%s counters detected\n", pmu->num_counters,
781 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "",
782 pmu->pmu.name);
783
784 uncore->num_pmus = 1;
785
786done:
787 uncore->init_done = true;
788
789 return amd_uncore_ctx_init(uncore, cpu);
790}
791
792static int amd_uncore_l3_event_init(struct perf_event *event)
793{
794 int ret = amd_uncore_event_init(event);
795 struct hw_perf_event *hwc = &event->hw;
796 u64 config = event->attr.config;
797 u64 mask;
798
799 hwc->config = config & AMD64_RAW_EVENT_MASK_NB;
800
801 /*
802 * SliceMask and ThreadMask need to be set for certain L3 events.
803 * For other events, the two fields do not affect the count.
804 */
805 if (ret || boot_cpu_data.x86 < 0x17)
806 return ret;
807
808 mask = config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
809 AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
810 AMD64_L3_COREID_MASK);
811
812 if (boot_cpu_data.x86 <= 0x18)
813 mask = ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
814 ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
815
816 /*
817 * If the user doesn't specify a ThreadMask, they're not trying to
818 * count core 0, so we enable all cores & threads.
819 * We'll also assume that they want to count slice 0 if they specify
820 * a ThreadMask and leave SliceId and EnAllSlices unpopulated.
821 */
822 else if (!(config & AMD64_L3_F19H_THREAD_MASK))
823 mask = AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
824 AMD64_L3_EN_ALL_CORES;
825
826 hwc->config |= mask;
827
828 return 0;
829}
830
831static
832void amd_uncore_l3_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
833{
834 union amd_uncore_info info;
835
836 if (!boot_cpu_has(X86_FEATURE_PERFCTR_LLC))
837 return;
838
839 info.split.aux_data = 0;
840 info.split.num_pmcs = NUM_COUNTERS_L2;
841 info.split.gid = 0;
842 info.split.cid = per_cpu_llc_id(cpu);
843
844 if (boot_cpu_data.x86 >= 0x17)
845 info.split.num_pmcs = NUM_COUNTERS_L3;
846
847 *per_cpu_ptr(uncore->info, cpu) = info;
848}
849
850static
851int amd_uncore_l3_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
852{
853 struct attribute **l3_attr = amd_uncore_l3_format_attr;
854 struct amd_uncore_pmu *pmu;
855 int num_counters;
856
857 /* Run just once */
858 if (uncore->init_done)
859 return amd_uncore_ctx_init(uncore, cpu);
860
861 num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);
862 if (!num_counters)
863 goto done;
864
865 /* No grouping, single instance for a system */
866 uncore->pmus = kzalloc(sizeof(*uncore->pmus), GFP_KERNEL);
867 if (!uncore->pmus)
868 goto done;
869
870 /*
871 * For Family 17h and above, L3 cache counters are available instead
872 * of L2 cache counters. The PMUs are exported based on family as
873 * either L2 or L3.
874 */
875 pmu = &uncore->pmus[0];
876 strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_l3" : "amd_l2",
877 sizeof(pmu->name));
878 pmu->num_counters = num_counters;
879 pmu->msr_base = MSR_F16H_L2I_PERF_CTL;
880 pmu->rdpmc_base = RDPMC_BASE_LLC;
881 pmu->group = amd_uncore_ctx_gid(uncore, cpu);
882
883 if (boot_cpu_data.x86 >= 0x17) {
884 *l3_attr++ = &format_attr_event8.attr;
885 *l3_attr++ = &format_attr_umask8.attr;
886 *l3_attr++ = boot_cpu_data.x86 >= 0x19 ?
887 &format_attr_threadmask2.attr :
888 &format_attr_threadmask8.attr;
889 }
890
891 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
892 if (!pmu->ctx)
893 goto done;
894
895 pmu->pmu = (struct pmu) {
896 .task_ctx_nr = perf_invalid_context,
897 .attr_groups = amd_uncore_l3_attr_groups,
898 .attr_update = amd_uncore_l3_attr_update,
899 .name = pmu->name,
900 .event_init = amd_uncore_l3_event_init,
901 .add = amd_uncore_add,
902 .del = amd_uncore_del,
903 .start = amd_uncore_start,
904 .stop = amd_uncore_stop,
905 .read = amd_uncore_read,
906 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
907 .module = THIS_MODULE,
908 };
909
910 if (perf_pmu_register(pmu: &pmu->pmu, name: pmu->pmu.name, type: -1)) {
911 free_percpu(pdata: pmu->ctx);
912 pmu->ctx = NULL;
913 goto done;
914 }
915
916 pr_info("%d %s%s counters detected\n", pmu->num_counters,
917 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "",
918 pmu->pmu.name);
919
920 uncore->num_pmus = 1;
921
922done:
923 uncore->init_done = true;
924
925 return amd_uncore_ctx_init(uncore, cpu);
926}
927
928static int amd_uncore_umc_event_init(struct perf_event *event)
929{
930 struct hw_perf_event *hwc = &event->hw;
931 int ret = amd_uncore_event_init(event);
932
933 if (ret)
934 return ret;
935
936 hwc->config = event->attr.config & AMD64_PERFMON_V2_RAW_EVENT_MASK_UMC;
937
938 return 0;
939}
940
941static void amd_uncore_umc_start(struct perf_event *event, int flags)
942{
943 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
944 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
945 struct hw_perf_event *hwc = &event->hw;
946
947 if (!ctx->nr_active++)
948 amd_uncore_start_hrtimer(ctx);
949
950 if (flags & PERF_EF_RELOAD)
951 wrmsrq(msr: hwc->event_base, val: (u64)local64_read(&hwc->prev_count));
952
953 hwc->state = 0;
954 __set_bit(hwc->idx, ctx->active_mask);
955 wrmsrq(msr: hwc->config_base, val: (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC));
956 perf_event_update_userpage(event);
957}
958
959static void amd_uncore_umc_read(struct perf_event *event)
960{
961 struct hw_perf_event *hwc = &event->hw;
962 u64 prev, new, shift;
963 s64 delta;
964
965 shift = COUNTER_SHIFT + 1;
966 prev = local64_read(&hwc->prev_count);
967
968 /*
969 * UMC counters do not have RDPMC assignments. Read counts directly
970 * from the corresponding PERF_CTR.
971 */
972 rdmsrl(hwc->event_base, new);
973
974 /*
975 * Unlike the other uncore counters, UMC counters saturate and set the
976 * Overflow bit (bit 48) on overflow. Since they do not roll over,
977 * proactively reset the corresponding PERF_CTR when bit 47 is set so
978 * that the counter never gets a chance to saturate.
979 */
980 if (new & BIT_ULL(63 - COUNTER_SHIFT)) {
981 wrmsrl(hwc->event_base, 0);
982 local64_set(&hwc->prev_count, 0);
983 } else {
984 local64_set(&hwc->prev_count, new);
985 }
986
987 delta = (new << shift) - (prev << shift);
988 delta >>= shift;
989 local64_add(delta, &event->count);
990}
991
992static
993void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
994{
995 union cpuid_0x80000022_ebx ebx;
996 union amd_uncore_info info;
997 unsigned int eax, ecx, edx;
998
999 if (pmu_version < 2)
1000 return;
1001
1002 cpuid(EXT_PERFMON_DEBUG_FEATURES, eax: &eax, ebx: &ebx.full, ecx: &ecx, edx: &edx);
1003 info.split.aux_data = ecx; /* stash active mask */
1004 info.split.num_pmcs = ebx.split.num_umc_pmc;
1005 info.split.gid = topology_logical_package_id(cpu);
1006 info.split.cid = topology_logical_package_id(cpu);
1007 *per_cpu_ptr(uncore->info, cpu) = info;
1008}
1009
1010static
1011int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
1012{
1013 DECLARE_BITMAP(gmask, UNCORE_GROUP_MAX) = { 0 };
1014 u8 group_num_pmus[UNCORE_GROUP_MAX] = { 0 };
1015 u8 group_num_pmcs[UNCORE_GROUP_MAX] = { 0 };
1016 union amd_uncore_info info;
1017 struct amd_uncore_pmu *pmu;
1018 int gid, i;
1019 u16 index = 0;
1020
1021 if (pmu_version < 2)
1022 return 0;
1023
1024 /* Run just once */
1025 if (uncore->init_done)
1026 return amd_uncore_ctx_init(uncore, cpu);
1027
1028 /* Find unique groups */
1029 for_each_online_cpu(i) {
1030 info = *per_cpu_ptr(uncore->info, i);
1031 gid = info.split.gid;
1032 if (test_bit(gid, gmask))
1033 continue;
1034
1035 __set_bit(gid, gmask);
1036 group_num_pmus[gid] = hweight32(info.split.aux_data);
1037 group_num_pmcs[gid] = info.split.num_pmcs;
1038 uncore->num_pmus += group_num_pmus[gid];
1039 }
1040
1041 uncore->pmus = kzalloc(sizeof(*uncore->pmus) * uncore->num_pmus,
1042 GFP_KERNEL);
1043 if (!uncore->pmus) {
1044 uncore->num_pmus = 0;
1045 goto done;
1046 }
1047
1048 for_each_set_bit(gid, gmask, UNCORE_GROUP_MAX) {
1049 for (i = 0; i < group_num_pmus[gid]; i++) {
1050 pmu = &uncore->pmus[index];
1051 snprintf(buf: pmu->name, size: sizeof(pmu->name), fmt: "amd_umc_%hu", index);
1052 pmu->num_counters = group_num_pmcs[gid] / group_num_pmus[gid];
1053 pmu->msr_base = MSR_F19H_UMC_PERF_CTL + i * pmu->num_counters * 2;
1054 pmu->rdpmc_base = -1;
1055 pmu->group = gid;
1056
1057 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
1058 if (!pmu->ctx)
1059 goto done;
1060
1061 pmu->pmu = (struct pmu) {
1062 .task_ctx_nr = perf_invalid_context,
1063 .attr_groups = amd_uncore_umc_attr_groups,
1064 .name = pmu->name,
1065 .event_init = amd_uncore_umc_event_init,
1066 .add = amd_uncore_add,
1067 .del = amd_uncore_del,
1068 .start = amd_uncore_umc_start,
1069 .stop = amd_uncore_stop,
1070 .read = amd_uncore_umc_read,
1071 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
1072 .module = THIS_MODULE,
1073 };
1074
1075 if (perf_pmu_register(pmu: &pmu->pmu, name: pmu->pmu.name, type: -1)) {
1076 free_percpu(pdata: pmu->ctx);
1077 pmu->ctx = NULL;
1078 goto done;
1079 }
1080
1081 pr_info("%d %s counters detected\n", pmu->num_counters,
1082 pmu->pmu.name);
1083
1084 index++;
1085 }
1086 }
1087
1088done:
1089 uncore->num_pmus = index;
1090 uncore->init_done = true;
1091
1092 return amd_uncore_ctx_init(uncore, cpu);
1093}
1094
1095static struct amd_uncore uncores[UNCORE_TYPE_MAX] = {
1096 /* UNCORE_TYPE_DF */
1097 {
1098 .scan = amd_uncore_df_ctx_scan,
1099 .init = amd_uncore_df_ctx_init,
1100 .move = amd_uncore_ctx_move,
1101 .free = amd_uncore_ctx_free,
1102 },
1103 /* UNCORE_TYPE_L3 */
1104 {
1105 .scan = amd_uncore_l3_ctx_scan,
1106 .init = amd_uncore_l3_ctx_init,
1107 .move = amd_uncore_ctx_move,
1108 .free = amd_uncore_ctx_free,
1109 },
1110 /* UNCORE_TYPE_UMC */
1111 {
1112 .scan = amd_uncore_umc_ctx_scan,
1113 .init = amd_uncore_umc_ctx_init,
1114 .move = amd_uncore_ctx_move,
1115 .free = amd_uncore_ctx_free,
1116 },
1117};
1118
1119static int __init amd_uncore_init(void)
1120{
1121 struct amd_uncore *uncore;
1122 int ret = -ENODEV;
1123 int i;
1124
1125 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
1126 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
1127 return -ENODEV;
1128
1129 if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
1130 return -ENODEV;
1131
1132 if (boot_cpu_has(X86_FEATURE_PERFMON_V2))
1133 pmu_version = 2;
1134
1135 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1136 uncore = &uncores[i];
1137
1138 BUG_ON(!uncore->scan);
1139 BUG_ON(!uncore->init);
1140 BUG_ON(!uncore->move);
1141 BUG_ON(!uncore->free);
1142
1143 uncore->info = alloc_percpu(union amd_uncore_info);
1144 if (!uncore->info) {
1145 ret = -ENOMEM;
1146 goto fail;
1147 }
1148 };
1149
1150 /*
1151 * Install callbacks. Core will call them for each online cpu.
1152 */
1153 ret = cpuhp_setup_state(state: CPUHP_PERF_X86_AMD_UNCORE_PREP,
1154 name: "perf/x86/amd/uncore:prepare",
1155 NULL, teardown: amd_uncore_cpu_dead);
1156 if (ret)
1157 goto fail;
1158
1159 ret = cpuhp_setup_state(state: CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
1160 name: "perf/x86/amd/uncore:starting",
1161 startup: amd_uncore_cpu_starting, NULL);
1162 if (ret)
1163 goto fail_prep;
1164
1165 ret = cpuhp_setup_state(state: CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
1166 name: "perf/x86/amd/uncore:online",
1167 startup: amd_uncore_cpu_online,
1168 teardown: amd_uncore_cpu_down_prepare);
1169 if (ret)
1170 goto fail_start;
1171
1172 return 0;
1173
1174fail_start:
1175 cpuhp_remove_state(state: CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
1176fail_prep:
1177 cpuhp_remove_state(state: CPUHP_PERF_X86_AMD_UNCORE_PREP);
1178fail:
1179 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1180 uncore = &uncores[i];
1181 if (uncore->info) {
1182 free_percpu(pdata: uncore->info);
1183 uncore->info = NULL;
1184 }
1185 }
1186
1187 return ret;
1188}
1189
1190static void __exit amd_uncore_exit(void)
1191{
1192 struct amd_uncore *uncore;
1193 struct amd_uncore_pmu *pmu;
1194 int i, j;
1195
1196 cpuhp_remove_state(state: CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE);
1197 cpuhp_remove_state(state: CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
1198 cpuhp_remove_state(state: CPUHP_PERF_X86_AMD_UNCORE_PREP);
1199
1200 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1201 uncore = &uncores[i];
1202 if (!uncore->info)
1203 continue;
1204
1205 free_percpu(pdata: uncore->info);
1206 uncore->info = NULL;
1207
1208 for (j = 0; j < uncore->num_pmus; j++) {
1209 pmu = &uncore->pmus[j];
1210 if (!pmu->ctx)
1211 continue;
1212
1213 perf_pmu_unregister(pmu: &pmu->pmu);
1214 free_percpu(pdata: pmu->ctx);
1215 pmu->ctx = NULL;
1216 }
1217
1218 kfree(objp: uncore->pmus);
1219 uncore->pmus = NULL;
1220 }
1221}
1222
1223module_init(amd_uncore_init);
1224module_exit(amd_uncore_exit);
1225
1226MODULE_DESCRIPTION("AMD Uncore Driver");
1227MODULE_LICENSE("GPL v2");
1228