1/*
2 * Support cstate residency counters
3 *
4 * Copyright (C) 2015, Intel Corp.
5 * Author: Kan Liang (kan.liang@intel.com)
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 */
18
19/*
20 * This file export cstate related free running (read-only) counters
21 * for perf. These counters may be use simultaneously by other tools,
22 * such as turbostat. However, it still make sense to implement them
23 * in perf. Because we can conveniently collect them together with
24 * other events, and allow to use them from tools without special MSR
25 * access code.
26 *
27 * The events only support system-wide mode counting. There is no
28 * sampling support because it is not supported by the hardware.
29 *
30 * According to counters' scope and category, two PMUs are registered
31 * with the perf_event core subsystem.
32 * - 'cstate_core': The counter is available for each physical core.
33 * The counters include CORE_C*_RESIDENCY.
34 * - 'cstate_pkg': The counter is available for each physical package.
35 * The counters include PKG_C*_RESIDENCY.
36 *
37 * All of these counters are specified in the Intel® 64 and IA-32
38 * Architectures Software Developer.s Manual Vol3b.
39 *
40 * Model specific counters:
41 * MSR_CORE_C1_RES: CORE C1 Residency Counter
42 * perf code: 0x00
43 * Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL
44 * MTL,SRF,GRR,ARL,LNL
45 * Scope: Core (each processor core has a MSR)
46 * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
47 * perf code: 0x01
48 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,GLM,
49 * CNL,KBL,CML,TNT
50 * Scope: Core
51 * MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter
52 * perf code: 0x02
53 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
54 * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
55 * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
56 * GRR,ARL,LNL
57 * Scope: Core
58 * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
59 * perf code: 0x03
60 * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML,
61 * ICL,TGL,RKL,ADL,RPL,MTL,ARL,LNL
62 * Scope: Core
63 * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter.
64 * perf code: 0x00
65 * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL,
66 * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL,
67 * RPL,SPR,MTL,ARL,LNL,SRF
68 * Scope: Package (physical package)
69 * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter.
70 * perf code: 0x01
71 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL,
72 * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL,
73 * ADL,RPL,MTL,ARL,LNL
74 * Scope: Package (physical package)
75 * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter.
76 * perf code: 0x02
77 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
78 * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
79 * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
80 * ARL,LNL
81 * Scope: Package (physical package)
82 * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter.
83 * perf code: 0x03
84 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,CNL,
85 * KBL,CML,ICL,TGL,RKL
86 * Scope: Package (physical package)
87 * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter.
88 * perf code: 0x04
89 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL,
90 * ADL,RPL,MTL,ARL
91 * Scope: Package (physical package)
92 * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter.
93 * perf code: 0x05
94 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL
95 * Scope: Package (physical package)
96 * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
97 * perf code: 0x06
98 * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,
99 * TNT,RKL,ADL,RPL,MTL,ARL,LNL
100 * Scope: Package (physical package)
101 * MSR_MODULE_C6_RES_MS: Module C6 Residency Counter.
102 * perf code: 0x00
103 * Available model: SRF,GRR
104 * Scope: A cluster of cores shared L2 cache
105 *
106 */
107
108#include <linux/module.h>
109#include <linux/slab.h>
110#include <linux/perf_event.h>
111#include <linux/nospec.h>
112#include <asm/cpu_device_id.h>
113#include <asm/intel-family.h>
114#include <asm/msr.h>
115#include "../perf_event.h"
116#include "../probe.h"
117
118MODULE_DESCRIPTION("Support for Intel cstate performance events");
119MODULE_LICENSE("GPL");
120
121#define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format) \
122static ssize_t __cstate_##_var##_show(struct device *dev, \
123 struct device_attribute *attr, \
124 char *page) \
125{ \
126 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
127 return sprintf(page, _format "\n"); \
128} \
129static struct device_attribute format_attr_##_var = \
130 __ATTR(_name, 0444, __cstate_##_var##_show, NULL)
131
132/* Model -> events mapping */
133struct cstate_model {
134 unsigned long core_events;
135 unsigned long pkg_events;
136 unsigned long module_events;
137 unsigned long quirks;
138};
139
140/* Quirk flags */
141#define SLM_PKG_C6_USE_C7_MSR (1UL << 0)
142#define KNL_CORE_C6_MSR (1UL << 1)
143
144/* cstate_core PMU */
145static struct pmu cstate_core_pmu;
146static bool has_cstate_core;
147
148enum perf_cstate_core_events {
149 PERF_CSTATE_CORE_C1_RES = 0,
150 PERF_CSTATE_CORE_C3_RES,
151 PERF_CSTATE_CORE_C6_RES,
152 PERF_CSTATE_CORE_C7_RES,
153
154 PERF_CSTATE_CORE_EVENT_MAX,
155};
156
157PMU_EVENT_ATTR_STRING(c1-residency, attr_cstate_core_c1, "event=0x00");
158PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_core_c3, "event=0x01");
159PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_core_c6, "event=0x02");
160PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_core_c7, "event=0x03");
161
162static unsigned long core_msr_mask;
163
164PMU_EVENT_GROUP(events, cstate_core_c1);
165PMU_EVENT_GROUP(events, cstate_core_c3);
166PMU_EVENT_GROUP(events, cstate_core_c6);
167PMU_EVENT_GROUP(events, cstate_core_c7);
168
169static bool test_msr(int idx, void *data)
170{
171 return test_bit(idx, (unsigned long *) data);
172}
173
174static struct perf_msr core_msr[] = {
175 [PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES, &group_cstate_core_c1, test_msr },
176 [PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY, .grp: &group_cstate_core_c3, .test: test_msr },
177 [PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY, .grp: &group_cstate_core_c6, .test: test_msr },
178 [PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY, .grp: &group_cstate_core_c7, .test: test_msr },
179};
180
181static struct attribute *attrs_empty[] = {
182 NULL,
183};
184
185/*
186 * There are no default events, but we need to create
187 * "events" group (with empty attrs) before updating
188 * it with detected events.
189 */
190static struct attribute_group cstate_events_attr_group = {
191 .name = "events",
192 .attrs = attrs_empty,
193};
194
195DEFINE_CSTATE_FORMAT_ATTR(cstate_event, event, "config:0-63");
196static struct attribute *cstate_format_attrs[] = {
197 &format_attr_cstate_event.attr,
198 NULL,
199};
200
201static struct attribute_group cstate_format_attr_group = {
202 .name = "format",
203 .attrs = cstate_format_attrs,
204};
205
206static const struct attribute_group *cstate_attr_groups[] = {
207 &cstate_events_attr_group,
208 &cstate_format_attr_group,
209 NULL,
210};
211
212/* cstate_pkg PMU */
213static struct pmu cstate_pkg_pmu;
214static bool has_cstate_pkg;
215
216enum perf_cstate_pkg_events {
217 PERF_CSTATE_PKG_C2_RES = 0,
218 PERF_CSTATE_PKG_C3_RES,
219 PERF_CSTATE_PKG_C6_RES,
220 PERF_CSTATE_PKG_C7_RES,
221 PERF_CSTATE_PKG_C8_RES,
222 PERF_CSTATE_PKG_C9_RES,
223 PERF_CSTATE_PKG_C10_RES,
224
225 PERF_CSTATE_PKG_EVENT_MAX,
226};
227
228PMU_EVENT_ATTR_STRING(c2-residency, attr_cstate_pkg_c2, "event=0x00");
229PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_pkg_c3, "event=0x01");
230PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_pkg_c6, "event=0x02");
231PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_pkg_c7, "event=0x03");
232PMU_EVENT_ATTR_STRING(c8-residency, attr_cstate_pkg_c8, "event=0x04");
233PMU_EVENT_ATTR_STRING(c9-residency, attr_cstate_pkg_c9, "event=0x05");
234PMU_EVENT_ATTR_STRING(c10-residency, attr_cstate_pkg_c10, "event=0x06");
235
236static unsigned long pkg_msr_mask;
237
238PMU_EVENT_GROUP(events, cstate_pkg_c2);
239PMU_EVENT_GROUP(events, cstate_pkg_c3);
240PMU_EVENT_GROUP(events, cstate_pkg_c6);
241PMU_EVENT_GROUP(events, cstate_pkg_c7);
242PMU_EVENT_GROUP(events, cstate_pkg_c8);
243PMU_EVENT_GROUP(events, cstate_pkg_c9);
244PMU_EVENT_GROUP(events, cstate_pkg_c10);
245
246static struct perf_msr pkg_msr[] = {
247 [PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY, .grp: &group_cstate_pkg_c2, .test: test_msr },
248 [PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY, .grp: &group_cstate_pkg_c3, .test: test_msr },
249 [PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY, .grp: &group_cstate_pkg_c6, .test: test_msr },
250 [PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY, .grp: &group_cstate_pkg_c7, .test: test_msr },
251 [PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY, .grp: &group_cstate_pkg_c8, .test: test_msr },
252 [PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY, .grp: &group_cstate_pkg_c9, .test: test_msr },
253 [PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY, .grp: &group_cstate_pkg_c10, .test: test_msr },
254};
255
256/* cstate_module PMU */
257static struct pmu cstate_module_pmu;
258static bool has_cstate_module;
259
260enum perf_cstate_module_events {
261 PERF_CSTATE_MODULE_C6_RES = 0,
262
263 PERF_CSTATE_MODULE_EVENT_MAX,
264};
265
266PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_module_c6, "event=0x00");
267
268static unsigned long module_msr_mask;
269
270PMU_EVENT_GROUP(events, cstate_module_c6);
271
272static struct perf_msr module_msr[] = {
273 [PERF_CSTATE_MODULE_C6_RES] = { MSR_MODULE_C6_RES_MS, .grp: &group_cstate_module_c6, .test: test_msr },
274};
275
276static int cstate_pmu_event_init(struct perf_event *event)
277{
278 u64 cfg = event->attr.config;
279
280 if (event->attr.type != event->pmu->type)
281 return -ENOENT;
282
283 /* unsupported modes and filters */
284 if (event->attr.sample_period) /* no sampling */
285 return -EINVAL;
286
287 if (event->cpu < 0)
288 return -EINVAL;
289
290 if (event->pmu == &cstate_core_pmu) {
291 if (cfg >= PERF_CSTATE_CORE_EVENT_MAX)
292 return -EINVAL;
293 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_CORE_EVENT_MAX);
294 if (!(core_msr_mask & (1 << cfg)))
295 return -EINVAL;
296 event->hw.event_base = core_msr[cfg].msr;
297 } else if (event->pmu == &cstate_pkg_pmu) {
298 if (cfg >= PERF_CSTATE_PKG_EVENT_MAX)
299 return -EINVAL;
300 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX);
301 if (!(pkg_msr_mask & (1 << cfg)))
302 return -EINVAL;
303 event->hw.event_base = pkg_msr[cfg].msr;
304 } else if (event->pmu == &cstate_module_pmu) {
305 if (cfg >= PERF_CSTATE_MODULE_EVENT_MAX)
306 return -EINVAL;
307 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_MODULE_EVENT_MAX);
308 if (!(module_msr_mask & (1 << cfg)))
309 return -EINVAL;
310 event->hw.event_base = module_msr[cfg].msr;
311 } else {
312 return -ENOENT;
313 }
314
315 event->hw.config = cfg;
316 event->hw.idx = -1;
317 return 0;
318}
319
320static inline u64 cstate_pmu_read_counter(struct perf_event *event)
321{
322 u64 val;
323
324 rdmsrq(event->hw.event_base, val);
325 return val;
326}
327
328static void cstate_pmu_event_update(struct perf_event *event)
329{
330 struct hw_perf_event *hwc = &event->hw;
331 u64 prev_raw_count, new_raw_count;
332
333 prev_raw_count = local64_read(&hwc->prev_count);
334 do {
335 new_raw_count = cstate_pmu_read_counter(event);
336 } while (!local64_try_cmpxchg(l: &hwc->prev_count,
337 old: &prev_raw_count, new: new_raw_count));
338
339 local64_add(new_raw_count - prev_raw_count, &event->count);
340}
341
342static void cstate_pmu_event_start(struct perf_event *event, int mode)
343{
344 local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event));
345}
346
347static void cstate_pmu_event_stop(struct perf_event *event, int mode)
348{
349 cstate_pmu_event_update(event);
350}
351
352static void cstate_pmu_event_del(struct perf_event *event, int mode)
353{
354 cstate_pmu_event_stop(event, PERF_EF_UPDATE);
355}
356
357static int cstate_pmu_event_add(struct perf_event *event, int mode)
358{
359 if (mode & PERF_EF_START)
360 cstate_pmu_event_start(event, mode);
361
362 return 0;
363}
364
365static const struct attribute_group *core_attr_update[] = {
366 &group_cstate_core_c1,
367 &group_cstate_core_c3,
368 &group_cstate_core_c6,
369 &group_cstate_core_c7,
370 NULL,
371};
372
373static const struct attribute_group *pkg_attr_update[] = {
374 &group_cstate_pkg_c2,
375 &group_cstate_pkg_c3,
376 &group_cstate_pkg_c6,
377 &group_cstate_pkg_c7,
378 &group_cstate_pkg_c8,
379 &group_cstate_pkg_c9,
380 &group_cstate_pkg_c10,
381 NULL,
382};
383
384static const struct attribute_group *module_attr_update[] = {
385 &group_cstate_module_c6,
386 NULL
387};
388
389static struct pmu cstate_core_pmu = {
390 .attr_groups = cstate_attr_groups,
391 .attr_update = core_attr_update,
392 .name = "cstate_core",
393 .task_ctx_nr = perf_invalid_context,
394 .event_init = cstate_pmu_event_init,
395 .add = cstate_pmu_event_add,
396 .del = cstate_pmu_event_del,
397 .start = cstate_pmu_event_start,
398 .stop = cstate_pmu_event_stop,
399 .read = cstate_pmu_event_update,
400 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
401 .scope = PERF_PMU_SCOPE_CORE,
402 .module = THIS_MODULE,
403};
404
405static struct pmu cstate_pkg_pmu = {
406 .attr_groups = cstate_attr_groups,
407 .attr_update = pkg_attr_update,
408 .name = "cstate_pkg",
409 .task_ctx_nr = perf_invalid_context,
410 .event_init = cstate_pmu_event_init,
411 .add = cstate_pmu_event_add,
412 .del = cstate_pmu_event_del,
413 .start = cstate_pmu_event_start,
414 .stop = cstate_pmu_event_stop,
415 .read = cstate_pmu_event_update,
416 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
417 .scope = PERF_PMU_SCOPE_PKG,
418 .module = THIS_MODULE,
419};
420
421static struct pmu cstate_module_pmu = {
422 .attr_groups = cstate_attr_groups,
423 .attr_update = module_attr_update,
424 .name = "cstate_module",
425 .task_ctx_nr = perf_invalid_context,
426 .event_init = cstate_pmu_event_init,
427 .add = cstate_pmu_event_add,
428 .del = cstate_pmu_event_del,
429 .start = cstate_pmu_event_start,
430 .stop = cstate_pmu_event_stop,
431 .read = cstate_pmu_event_update,
432 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
433 .scope = PERF_PMU_SCOPE_CLUSTER,
434 .module = THIS_MODULE,
435};
436
437static const struct cstate_model nhm_cstates __initconst = {
438 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
439 BIT(PERF_CSTATE_CORE_C6_RES),
440
441 .pkg_events = BIT(PERF_CSTATE_PKG_C3_RES) |
442 BIT(PERF_CSTATE_PKG_C6_RES) |
443 BIT(PERF_CSTATE_PKG_C7_RES),
444};
445
446static const struct cstate_model snb_cstates __initconst = {
447 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
448 BIT(PERF_CSTATE_CORE_C6_RES) |
449 BIT(PERF_CSTATE_CORE_C7_RES),
450
451 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
452 BIT(PERF_CSTATE_PKG_C3_RES) |
453 BIT(PERF_CSTATE_PKG_C6_RES) |
454 BIT(PERF_CSTATE_PKG_C7_RES),
455};
456
457static const struct cstate_model hswult_cstates __initconst = {
458 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
459 BIT(PERF_CSTATE_CORE_C6_RES) |
460 BIT(PERF_CSTATE_CORE_C7_RES),
461
462 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
463 BIT(PERF_CSTATE_PKG_C3_RES) |
464 BIT(PERF_CSTATE_PKG_C6_RES) |
465 BIT(PERF_CSTATE_PKG_C7_RES) |
466 BIT(PERF_CSTATE_PKG_C8_RES) |
467 BIT(PERF_CSTATE_PKG_C9_RES) |
468 BIT(PERF_CSTATE_PKG_C10_RES),
469};
470
471static const struct cstate_model cnl_cstates __initconst = {
472 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
473 BIT(PERF_CSTATE_CORE_C3_RES) |
474 BIT(PERF_CSTATE_CORE_C6_RES) |
475 BIT(PERF_CSTATE_CORE_C7_RES),
476
477 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
478 BIT(PERF_CSTATE_PKG_C3_RES) |
479 BIT(PERF_CSTATE_PKG_C6_RES) |
480 BIT(PERF_CSTATE_PKG_C7_RES) |
481 BIT(PERF_CSTATE_PKG_C8_RES) |
482 BIT(PERF_CSTATE_PKG_C9_RES) |
483 BIT(PERF_CSTATE_PKG_C10_RES),
484};
485
486static const struct cstate_model icl_cstates __initconst = {
487 .core_events = BIT(PERF_CSTATE_CORE_C6_RES) |
488 BIT(PERF_CSTATE_CORE_C7_RES),
489
490 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
491 BIT(PERF_CSTATE_PKG_C3_RES) |
492 BIT(PERF_CSTATE_PKG_C6_RES) |
493 BIT(PERF_CSTATE_PKG_C7_RES) |
494 BIT(PERF_CSTATE_PKG_C8_RES) |
495 BIT(PERF_CSTATE_PKG_C9_RES) |
496 BIT(PERF_CSTATE_PKG_C10_RES),
497};
498
499static const struct cstate_model icx_cstates __initconst = {
500 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
501 BIT(PERF_CSTATE_CORE_C6_RES),
502
503 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
504 BIT(PERF_CSTATE_PKG_C6_RES),
505};
506
507static const struct cstate_model adl_cstates __initconst = {
508 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
509 BIT(PERF_CSTATE_CORE_C6_RES) |
510 BIT(PERF_CSTATE_CORE_C7_RES),
511
512 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
513 BIT(PERF_CSTATE_PKG_C3_RES) |
514 BIT(PERF_CSTATE_PKG_C6_RES) |
515 BIT(PERF_CSTATE_PKG_C8_RES) |
516 BIT(PERF_CSTATE_PKG_C10_RES),
517};
518
519static const struct cstate_model lnl_cstates __initconst = {
520 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
521 BIT(PERF_CSTATE_CORE_C6_RES) |
522 BIT(PERF_CSTATE_CORE_C7_RES),
523
524 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
525 BIT(PERF_CSTATE_PKG_C3_RES) |
526 BIT(PERF_CSTATE_PKG_C6_RES) |
527 BIT(PERF_CSTATE_PKG_C10_RES),
528};
529
530static const struct cstate_model slm_cstates __initconst = {
531 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
532 BIT(PERF_CSTATE_CORE_C6_RES),
533
534 .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES),
535 .quirks = SLM_PKG_C6_USE_C7_MSR,
536};
537
538
539static const struct cstate_model knl_cstates __initconst = {
540 .core_events = BIT(PERF_CSTATE_CORE_C6_RES),
541
542 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
543 BIT(PERF_CSTATE_PKG_C3_RES) |
544 BIT(PERF_CSTATE_PKG_C6_RES),
545 .quirks = KNL_CORE_C6_MSR,
546};
547
548
549static const struct cstate_model glm_cstates __initconst = {
550 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
551 BIT(PERF_CSTATE_CORE_C3_RES) |
552 BIT(PERF_CSTATE_CORE_C6_RES),
553
554 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
555 BIT(PERF_CSTATE_PKG_C3_RES) |
556 BIT(PERF_CSTATE_PKG_C6_RES) |
557 BIT(PERF_CSTATE_PKG_C10_RES),
558};
559
560static const struct cstate_model grr_cstates __initconst = {
561 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
562 BIT(PERF_CSTATE_CORE_C6_RES),
563
564 .module_events = BIT(PERF_CSTATE_MODULE_C6_RES),
565};
566
567static const struct cstate_model srf_cstates __initconst = {
568 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
569 BIT(PERF_CSTATE_CORE_C6_RES),
570
571 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
572 BIT(PERF_CSTATE_PKG_C6_RES),
573
574 .module_events = BIT(PERF_CSTATE_MODULE_C6_RES),
575};
576
577
578static const struct x86_cpu_id intel_cstates_match[] __initconst = {
579 X86_MATCH_VFM(INTEL_NEHALEM, &nhm_cstates),
580 X86_MATCH_VFM(INTEL_NEHALEM_EP, &nhm_cstates),
581 X86_MATCH_VFM(INTEL_NEHALEM_EX, &nhm_cstates),
582
583 X86_MATCH_VFM(INTEL_WESTMERE, &nhm_cstates),
584 X86_MATCH_VFM(INTEL_WESTMERE_EP, &nhm_cstates),
585 X86_MATCH_VFM(INTEL_WESTMERE_EX, &nhm_cstates),
586
587 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &snb_cstates),
588 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &snb_cstates),
589
590 X86_MATCH_VFM(INTEL_IVYBRIDGE, &snb_cstates),
591 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &snb_cstates),
592
593 X86_MATCH_VFM(INTEL_HASWELL, &snb_cstates),
594 X86_MATCH_VFM(INTEL_HASWELL_X, &snb_cstates),
595 X86_MATCH_VFM(INTEL_HASWELL_G, &snb_cstates),
596
597 X86_MATCH_VFM(INTEL_HASWELL_L, &hswult_cstates),
598
599 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &slm_cstates),
600 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &slm_cstates),
601 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &slm_cstates),
602
603 X86_MATCH_VFM(INTEL_BROADWELL, &snb_cstates),
604 X86_MATCH_VFM(INTEL_BROADWELL_D, &snb_cstates),
605 X86_MATCH_VFM(INTEL_BROADWELL_G, &snb_cstates),
606 X86_MATCH_VFM(INTEL_BROADWELL_X, &snb_cstates),
607
608 X86_MATCH_VFM(INTEL_SKYLAKE_L, &snb_cstates),
609 X86_MATCH_VFM(INTEL_SKYLAKE, &snb_cstates),
610 X86_MATCH_VFM(INTEL_SKYLAKE_X, &snb_cstates),
611
612 X86_MATCH_VFM(INTEL_KABYLAKE_L, &hswult_cstates),
613 X86_MATCH_VFM(INTEL_KABYLAKE, &hswult_cstates),
614 X86_MATCH_VFM(INTEL_COMETLAKE_L, &hswult_cstates),
615 X86_MATCH_VFM(INTEL_COMETLAKE, &hswult_cstates),
616
617 X86_MATCH_VFM(INTEL_CANNONLAKE_L, &cnl_cstates),
618
619 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &knl_cstates),
620 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &knl_cstates),
621
622 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &glm_cstates),
623 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &glm_cstates),
624 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &glm_cstates),
625 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &glm_cstates),
626 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &glm_cstates),
627 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &glm_cstates),
628 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &adl_cstates),
629 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &srf_cstates),
630 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &grr_cstates),
631
632 X86_MATCH_VFM(INTEL_ICELAKE_L, &icl_cstates),
633 X86_MATCH_VFM(INTEL_ICELAKE, &icl_cstates),
634 X86_MATCH_VFM(INTEL_ICELAKE_X, &icx_cstates),
635 X86_MATCH_VFM(INTEL_ICELAKE_D, &icx_cstates),
636 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &icx_cstates),
637 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &icx_cstates),
638 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &icx_cstates),
639 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &icx_cstates),
640
641 X86_MATCH_VFM(INTEL_TIGERLAKE_L, &icl_cstates),
642 X86_MATCH_VFM(INTEL_TIGERLAKE, &icl_cstates),
643 X86_MATCH_VFM(INTEL_ROCKETLAKE, &icl_cstates),
644 X86_MATCH_VFM(INTEL_ALDERLAKE, &adl_cstates),
645 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &adl_cstates),
646 X86_MATCH_VFM(INTEL_RAPTORLAKE, &adl_cstates),
647 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, &adl_cstates),
648 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, &adl_cstates),
649 X86_MATCH_VFM(INTEL_METEORLAKE, &adl_cstates),
650 X86_MATCH_VFM(INTEL_METEORLAKE_L, &adl_cstates),
651 X86_MATCH_VFM(INTEL_ARROWLAKE, &adl_cstates),
652 X86_MATCH_VFM(INTEL_ARROWLAKE_H, &adl_cstates),
653 X86_MATCH_VFM(INTEL_ARROWLAKE_U, &adl_cstates),
654 X86_MATCH_VFM(INTEL_LUNARLAKE_M, &lnl_cstates),
655 { },
656};
657MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);
658
659static int __init cstate_probe(const struct cstate_model *cm)
660{
661 /* SLM has different MSR for PKG C6 */
662 if (cm->quirks & SLM_PKG_C6_USE_C7_MSR)
663 pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY;
664
665 /* KNL has different MSR for CORE C6 */
666 if (cm->quirks & KNL_CORE_C6_MSR)
667 pkg_msr[PERF_CSTATE_CORE_C6_RES].msr = MSR_KNL_CORE_C6_RESIDENCY;
668
669
670 core_msr_mask = perf_msr_probe(msr: core_msr, cnt: PERF_CSTATE_CORE_EVENT_MAX,
671 no_zero: true, data: (void *) &cm->core_events);
672
673 pkg_msr_mask = perf_msr_probe(msr: pkg_msr, cnt: PERF_CSTATE_PKG_EVENT_MAX,
674 no_zero: true, data: (void *) &cm->pkg_events);
675
676 module_msr_mask = perf_msr_probe(msr: module_msr, cnt: PERF_CSTATE_MODULE_EVENT_MAX,
677 no_zero: true, data: (void *) &cm->module_events);
678
679 has_cstate_core = !!core_msr_mask;
680 has_cstate_pkg = !!pkg_msr_mask;
681 has_cstate_module = !!module_msr_mask;
682
683 return (has_cstate_core || has_cstate_pkg || has_cstate_module) ? 0 : -ENODEV;
684}
685
686static inline void cstate_cleanup(void)
687{
688 if (has_cstate_core)
689 perf_pmu_unregister(pmu: &cstate_core_pmu);
690
691 if (has_cstate_pkg)
692 perf_pmu_unregister(pmu: &cstate_pkg_pmu);
693
694 if (has_cstate_module)
695 perf_pmu_unregister(pmu: &cstate_module_pmu);
696}
697
698static int __init cstate_init(void)
699{
700 int err;
701
702 if (has_cstate_core) {
703 err = perf_pmu_register(pmu: &cstate_core_pmu, name: cstate_core_pmu.name, type: -1);
704 if (err) {
705 has_cstate_core = false;
706 pr_info("Failed to register cstate core pmu\n");
707 cstate_cleanup();
708 return err;
709 }
710 }
711
712 if (has_cstate_pkg) {
713 if (topology_max_dies_per_package() > 1) {
714 /* CLX-AP is multi-die and the cstate is die-scope */
715 cstate_pkg_pmu.scope = PERF_PMU_SCOPE_DIE;
716 err = perf_pmu_register(pmu: &cstate_pkg_pmu,
717 name: "cstate_die", type: -1);
718 } else {
719 err = perf_pmu_register(pmu: &cstate_pkg_pmu,
720 name: cstate_pkg_pmu.name, type: -1);
721 }
722 if (err) {
723 has_cstate_pkg = false;
724 pr_info("Failed to register cstate pkg pmu\n");
725 cstate_cleanup();
726 return err;
727 }
728 }
729
730 if (has_cstate_module) {
731 err = perf_pmu_register(pmu: &cstate_module_pmu, name: cstate_module_pmu.name, type: -1);
732 if (err) {
733 has_cstate_module = false;
734 pr_info("Failed to register cstate cluster pmu\n");
735 cstate_cleanup();
736 return err;
737 }
738 }
739 return 0;
740}
741
742static int __init cstate_pmu_init(void)
743{
744 const struct x86_cpu_id *id;
745 int err;
746
747 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
748 return -ENODEV;
749
750 id = x86_match_cpu(match: intel_cstates_match);
751 if (!id)
752 return -ENODEV;
753
754 err = cstate_probe(cm: (const struct cstate_model *) id->driver_data);
755 if (err)
756 return err;
757
758 return cstate_init();
759}
760module_init(cstate_pmu_init);
761
762static void __exit cstate_pmu_exit(void)
763{
764 cstate_cleanup();
765}
766module_exit(cstate_pmu_exit);
767