1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2013 Advanced Micro Devices, Inc.
4 *
5 * Author: Steven Kinney <Steven.Kinney@amd.com>
6 * Author: Suravee Suthikulpanit <Suraveee.Suthikulpanit@amd.com>
7 *
8 * Perf: amd_iommu - AMD IOMMU Performance Counter PMU implementation
9 */
10
11#define pr_fmt(fmt) "perf/amd_iommu: " fmt
12
13#include <linux/perf_event.h>
14#include <linux/init.h>
15#include <linux/cpumask.h>
16#include <linux/slab.h>
17#include <linux/amd-iommu.h>
18
19#include <asm/msr.h>
20
21#include "../perf_event.h"
22#include "iommu.h"
23
24/* iommu pmu conf masks */
25#define GET_CSOURCE(x) ((x)->conf & 0xFFULL)
26#define GET_DEVID(x) (((x)->conf >> 8) & 0xFFFFULL)
27#define GET_DOMID(x) (((x)->conf >> 24) & 0xFFFFULL)
28#define GET_PASID(x) (((x)->conf >> 40) & 0xFFFFFULL)
29
30/* iommu pmu conf1 masks */
31#define GET_DEVID_MASK(x) ((x)->conf1 & 0xFFFFULL)
32#define GET_DOMID_MASK(x) (((x)->conf1 >> 16) & 0xFFFFULL)
33#define GET_PASID_MASK(x) (((x)->conf1 >> 32) & 0xFFFFFULL)
34
35#define IOMMU_NAME_SIZE 24
36
37struct perf_amd_iommu {
38 struct list_head list;
39 struct pmu pmu;
40 struct amd_iommu *iommu;
41 char name[IOMMU_NAME_SIZE];
42 u8 max_banks;
43 u8 max_counters;
44 u64 cntr_assign_mask;
45 raw_spinlock_t lock;
46};
47
48static LIST_HEAD(perf_amd_iommu_list);
49
50/*---------------------------------------------
51 * sysfs format attributes
52 *---------------------------------------------*/
53PMU_FORMAT_ATTR(csource, "config:0-7");
54PMU_FORMAT_ATTR(devid, "config:8-23");
55PMU_FORMAT_ATTR(domid, "config:24-39");
56PMU_FORMAT_ATTR(pasid, "config:40-59");
57PMU_FORMAT_ATTR(devid_mask, "config1:0-15");
58PMU_FORMAT_ATTR(domid_mask, "config1:16-31");
59PMU_FORMAT_ATTR(pasid_mask, "config1:32-51");
60
61static struct attribute *iommu_format_attrs[] = {
62 &format_attr_csource.attr,
63 &format_attr_devid.attr,
64 &format_attr_pasid.attr,
65 &format_attr_domid.attr,
66 &format_attr_devid_mask.attr,
67 &format_attr_pasid_mask.attr,
68 &format_attr_domid_mask.attr,
69 NULL,
70};
71
72static struct attribute_group amd_iommu_format_group = {
73 .name = "format",
74 .attrs = iommu_format_attrs,
75};
76
77/*---------------------------------------------
78 * sysfs events attributes
79 *---------------------------------------------*/
80static struct attribute_group amd_iommu_events_group = {
81 .name = "events",
82};
83
84struct amd_iommu_event_desc {
85 struct device_attribute attr;
86 const char *event;
87};
88
89static ssize_t _iommu_event_show(struct device *dev,
90 struct device_attribute *attr, char *buf)
91{
92 struct amd_iommu_event_desc *event =
93 container_of(attr, struct amd_iommu_event_desc, attr);
94 return sprintf(buf, fmt: "%s\n", event->event);
95}
96
97#define AMD_IOMMU_EVENT_DESC(_name, _event) \
98{ \
99 .attr = __ATTR(_name, 0444, _iommu_event_show, NULL), \
100 .event = _event, \
101}
102
103static struct amd_iommu_event_desc amd_iommu_v2_event_descs[] = {
104 AMD_IOMMU_EVENT_DESC(mem_pass_untrans, "csource=0x01"),
105 AMD_IOMMU_EVENT_DESC(mem_pass_pretrans, "csource=0x02"),
106 AMD_IOMMU_EVENT_DESC(mem_pass_excl, "csource=0x03"),
107 AMD_IOMMU_EVENT_DESC(mem_target_abort, "csource=0x04"),
108 AMD_IOMMU_EVENT_DESC(mem_trans_total, "csource=0x05"),
109 AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pte_hit, "csource=0x06"),
110 AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pte_mis, "csource=0x07"),
111 AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pde_hit, "csource=0x08"),
112 AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pde_mis, "csource=0x09"),
113 AMD_IOMMU_EVENT_DESC(mem_dte_hit, "csource=0x0a"),
114 AMD_IOMMU_EVENT_DESC(mem_dte_mis, "csource=0x0b"),
115 AMD_IOMMU_EVENT_DESC(page_tbl_read_tot, "csource=0x0c"),
116 AMD_IOMMU_EVENT_DESC(page_tbl_read_nst, "csource=0x0d"),
117 AMD_IOMMU_EVENT_DESC(page_tbl_read_gst, "csource=0x0e"),
118 AMD_IOMMU_EVENT_DESC(int_dte_hit, "csource=0x0f"),
119 AMD_IOMMU_EVENT_DESC(int_dte_mis, "csource=0x10"),
120 AMD_IOMMU_EVENT_DESC(cmd_processed, "csource=0x11"),
121 AMD_IOMMU_EVENT_DESC(cmd_processed_inv, "csource=0x12"),
122 AMD_IOMMU_EVENT_DESC(tlb_inv, "csource=0x13"),
123 AMD_IOMMU_EVENT_DESC(ign_rd_wr_mmio_1ff8h, "csource=0x14"),
124 AMD_IOMMU_EVENT_DESC(vapic_int_non_guest, "csource=0x15"),
125 AMD_IOMMU_EVENT_DESC(vapic_int_guest, "csource=0x16"),
126 AMD_IOMMU_EVENT_DESC(smi_recv, "csource=0x17"),
127 AMD_IOMMU_EVENT_DESC(smi_blk, "csource=0x18"),
128 { /* end: all zeroes */ },
129};
130
131/*---------------------------------------------
132 * sysfs cpumask attributes
133 *---------------------------------------------*/
134static cpumask_t iommu_cpumask;
135
136static ssize_t _iommu_cpumask_show(struct device *dev,
137 struct device_attribute *attr,
138 char *buf)
139{
140 return cpumap_print_to_pagebuf(list: true, buf, mask: &iommu_cpumask);
141}
142static DEVICE_ATTR(cpumask, S_IRUGO, _iommu_cpumask_show, NULL);
143
144static struct attribute *iommu_cpumask_attrs[] = {
145 &dev_attr_cpumask.attr,
146 NULL,
147};
148
149static struct attribute_group amd_iommu_cpumask_group = {
150 .attrs = iommu_cpumask_attrs,
151};
152
153/*---------------------------------------------*/
154
155static int get_next_avail_iommu_bnk_cntr(struct perf_event *event)
156{
157 struct perf_amd_iommu *piommu = container_of(event->pmu, struct perf_amd_iommu, pmu);
158 int max_cntrs = piommu->max_counters;
159 int max_banks = piommu->max_banks;
160 u32 shift, bank, cntr;
161 unsigned long flags;
162 int retval;
163
164 raw_spin_lock_irqsave(&piommu->lock, flags);
165
166 for (bank = 0; bank < max_banks; bank++) {
167 for (cntr = 0; cntr < max_cntrs; cntr++) {
168 shift = bank + (bank*3) + cntr;
169 if (piommu->cntr_assign_mask & BIT_ULL(shift)) {
170 continue;
171 } else {
172 piommu->cntr_assign_mask |= BIT_ULL(shift);
173 event->hw.iommu_bank = bank;
174 event->hw.iommu_cntr = cntr;
175 retval = 0;
176 goto out;
177 }
178 }
179 }
180 retval = -ENOSPC;
181out:
182 raw_spin_unlock_irqrestore(&piommu->lock, flags);
183 return retval;
184}
185
186static int clear_avail_iommu_bnk_cntr(struct perf_amd_iommu *perf_iommu,
187 u8 bank, u8 cntr)
188{
189 unsigned long flags;
190 int max_banks, max_cntrs;
191 int shift = 0;
192
193 max_banks = perf_iommu->max_banks;
194 max_cntrs = perf_iommu->max_counters;
195
196 if ((bank > max_banks) || (cntr > max_cntrs))
197 return -EINVAL;
198
199 shift = bank + cntr + (bank*3);
200
201 raw_spin_lock_irqsave(&perf_iommu->lock, flags);
202 perf_iommu->cntr_assign_mask &= ~(1ULL<<shift);
203 raw_spin_unlock_irqrestore(&perf_iommu->lock, flags);
204
205 return 0;
206}
207
208static int perf_iommu_event_init(struct perf_event *event)
209{
210 struct hw_perf_event *hwc = &event->hw;
211
212 /* test the event attr type check for PMU enumeration */
213 if (event->attr.type != event->pmu->type)
214 return -ENOENT;
215
216 /*
217 * IOMMU counters are shared across all cores.
218 * Therefore, it does not support per-process mode.
219 * Also, it does not support event sampling mode.
220 */
221 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
222 return -EINVAL;
223
224 if (event->cpu < 0)
225 return -EINVAL;
226
227 /* update the hw_perf_event struct with the iommu config data */
228 hwc->conf = event->attr.config;
229 hwc->conf1 = event->attr.config1;
230
231 return 0;
232}
233
234static inline struct amd_iommu *perf_event_2_iommu(struct perf_event *ev)
235{
236 return (container_of(ev->pmu, struct perf_amd_iommu, pmu))->iommu;
237}
238
239static void perf_iommu_enable_event(struct perf_event *ev)
240{
241 struct amd_iommu *iommu = perf_event_2_iommu(ev);
242 struct hw_perf_event *hwc = &ev->hw;
243 u8 bank = hwc->iommu_bank;
244 u8 cntr = hwc->iommu_cntr;
245 u64 reg = 0ULL;
246
247 reg = GET_CSOURCE(hwc);
248 amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_COUNTER_SRC_REG, value: &reg);
249
250 reg = GET_DEVID_MASK(hwc);
251 reg = GET_DEVID(hwc) | (reg << 32);
252 if (reg)
253 reg |= BIT(31);
254 amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DEVID_MATCH_REG, value: &reg);
255
256 reg = GET_PASID_MASK(hwc);
257 reg = GET_PASID(hwc) | (reg << 32);
258 if (reg)
259 reg |= BIT(31);
260 amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_PASID_MATCH_REG, value: &reg);
261
262 reg = GET_DOMID_MASK(hwc);
263 reg = GET_DOMID(hwc) | (reg << 32);
264 if (reg)
265 reg |= BIT(31);
266 amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DOMID_MATCH_REG, value: &reg);
267}
268
269static void perf_iommu_disable_event(struct perf_event *event)
270{
271 struct amd_iommu *iommu = perf_event_2_iommu(ev: event);
272 struct hw_perf_event *hwc = &event->hw;
273 u64 reg = 0ULL;
274
275 amd_iommu_pc_set_reg(iommu, bank: hwc->iommu_bank, cntr: hwc->iommu_cntr,
276 IOMMU_PC_COUNTER_SRC_REG, value: &reg);
277}
278
279static void perf_iommu_start(struct perf_event *event, int flags)
280{
281 struct hw_perf_event *hwc = &event->hw;
282
283 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
284 return;
285
286 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
287 hwc->state = 0;
288
289 /*
290 * To account for power-gating, which prevents write to
291 * the counter, we need to enable the counter
292 * before setting up counter register.
293 */
294 perf_iommu_enable_event(ev: event);
295
296 if (flags & PERF_EF_RELOAD) {
297 u64 count = 0;
298 struct amd_iommu *iommu = perf_event_2_iommu(ev: event);
299
300 /*
301 * Since the IOMMU PMU only support counting mode,
302 * the counter always start with value zero.
303 */
304 amd_iommu_pc_set_reg(iommu, bank: hwc->iommu_bank, cntr: hwc->iommu_cntr,
305 IOMMU_PC_COUNTER_REG, value: &count);
306 }
307
308 perf_event_update_userpage(event);
309}
310
311static void perf_iommu_read(struct perf_event *event)
312{
313 u64 count;
314 struct hw_perf_event *hwc = &event->hw;
315 struct amd_iommu *iommu = perf_event_2_iommu(ev: event);
316
317 if (amd_iommu_pc_get_reg(iommu, bank: hwc->iommu_bank, cntr: hwc->iommu_cntr,
318 IOMMU_PC_COUNTER_REG, value: &count))
319 return;
320
321 /* IOMMU pc counter register is only 48 bits */
322 count &= GENMASK_ULL(47, 0);
323
324 /*
325 * Since the counter always start with value zero,
326 * simply just accumulate the count for the event.
327 */
328 local64_add(count, &event->count);
329}
330
331static void perf_iommu_stop(struct perf_event *event, int flags)
332{
333 struct hw_perf_event *hwc = &event->hw;
334
335 if (hwc->state & PERF_HES_UPTODATE)
336 return;
337
338 /*
339 * To account for power-gating, in which reading the counter would
340 * return zero, we need to read the register before disabling.
341 */
342 perf_iommu_read(event);
343 hwc->state |= PERF_HES_UPTODATE;
344
345 perf_iommu_disable_event(event);
346 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
347 hwc->state |= PERF_HES_STOPPED;
348}
349
350static int perf_iommu_add(struct perf_event *event, int flags)
351{
352 int retval;
353
354 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
355
356 /* request an iommu bank/counter */
357 retval = get_next_avail_iommu_bnk_cntr(event);
358 if (retval)
359 return retval;
360
361 if (flags & PERF_EF_START)
362 perf_iommu_start(event, PERF_EF_RELOAD);
363
364 return 0;
365}
366
367static void perf_iommu_del(struct perf_event *event, int flags)
368{
369 struct hw_perf_event *hwc = &event->hw;
370 struct perf_amd_iommu *perf_iommu =
371 container_of(event->pmu, struct perf_amd_iommu, pmu);
372
373 perf_iommu_stop(event, PERF_EF_UPDATE);
374
375 /* clear the assigned iommu bank/counter */
376 clear_avail_iommu_bnk_cntr(perf_iommu,
377 bank: hwc->iommu_bank, cntr: hwc->iommu_cntr);
378
379 perf_event_update_userpage(event);
380}
381
382static __init int _init_events_attrs(void)
383{
384 int i = 0, j;
385 struct attribute **attrs;
386
387 while (amd_iommu_v2_event_descs[i].attr.attr.name)
388 i++;
389
390 attrs = kcalloc(i + 1, sizeof(*attrs), GFP_KERNEL);
391 if (!attrs)
392 return -ENOMEM;
393
394 for (j = 0; j < i; j++)
395 attrs[j] = &amd_iommu_v2_event_descs[j].attr.attr;
396
397 amd_iommu_events_group.attrs = attrs;
398 return 0;
399}
400
401static const struct attribute_group *amd_iommu_attr_groups[] = {
402 &amd_iommu_format_group,
403 &amd_iommu_cpumask_group,
404 &amd_iommu_events_group,
405 NULL,
406};
407
408static const struct pmu iommu_pmu __initconst = {
409 .event_init = perf_iommu_event_init,
410 .add = perf_iommu_add,
411 .del = perf_iommu_del,
412 .start = perf_iommu_start,
413 .stop = perf_iommu_stop,
414 .read = perf_iommu_read,
415 .task_ctx_nr = perf_invalid_context,
416 .attr_groups = amd_iommu_attr_groups,
417 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
418};
419
420static __init int init_one_iommu(unsigned int idx)
421{
422 struct perf_amd_iommu *perf_iommu;
423 int ret;
424
425 perf_iommu = kzalloc(sizeof(struct perf_amd_iommu), GFP_KERNEL);
426 if (!perf_iommu)
427 return -ENOMEM;
428
429 raw_spin_lock_init(&perf_iommu->lock);
430
431 perf_iommu->pmu = iommu_pmu;
432 perf_iommu->iommu = get_amd_iommu(idx);
433 perf_iommu->max_banks = amd_iommu_pc_get_max_banks(idx);
434 perf_iommu->max_counters = amd_iommu_pc_get_max_counters(idx);
435
436 if (!perf_iommu->iommu ||
437 !perf_iommu->max_banks ||
438 !perf_iommu->max_counters) {
439 kfree(objp: perf_iommu);
440 return -EINVAL;
441 }
442
443 snprintf(buf: perf_iommu->name, IOMMU_NAME_SIZE, fmt: "amd_iommu_%u", idx);
444
445 ret = perf_pmu_register(pmu: &perf_iommu->pmu, name: perf_iommu->name, type: -1);
446 if (!ret) {
447 pr_info("Detected AMD IOMMU #%d (%d banks, %d counters/bank).\n",
448 idx, perf_iommu->max_banks, perf_iommu->max_counters);
449 list_add_tail(new: &perf_iommu->list, head: &perf_amd_iommu_list);
450 } else {
451 pr_warn("Error initializing IOMMU %d.\n", idx);
452 kfree(objp: perf_iommu);
453 }
454 return ret;
455}
456
457static __init int amd_iommu_pc_init(void)
458{
459 unsigned int i, cnt = 0;
460 int ret;
461
462 /* Make sure the IOMMU PC resource is available */
463 if (!amd_iommu_pc_supported())
464 return -ENODEV;
465
466 ret = _init_events_attrs();
467 if (ret)
468 return ret;
469
470 /*
471 * An IOMMU PMU is specific to an IOMMU, and can function independently.
472 * So we go through all IOMMUs and ignore the one that fails init
473 * unless all IOMMU are failing.
474 */
475 for (i = 0; i < amd_iommu_get_num_iommus(); i++) {
476 ret = init_one_iommu(idx: i);
477 if (!ret)
478 cnt++;
479 }
480
481 if (!cnt) {
482 kfree(objp: amd_iommu_events_group.attrs);
483 return -ENODEV;
484 }
485
486 /* Init cpumask attributes to only core 0 */
487 cpumask_set_cpu(cpu: 0, dstp: &iommu_cpumask);
488 return 0;
489}
490
491device_initcall(amd_iommu_pc_init);
492