2 * Support cstate residency counters
4 * Copyright (C) 2015, Intel Corp.
5 * Author: Kan Liang (kan.liang@intel.com)
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
20 * This file export cstate related free running (read-only) counters
21 * for perf. These counters may be use simultaneously by other tools,
22 * such as turbostat. However, it still make sense to implement them
23 * in perf. Because we can conveniently collect them together with
24 * other events, and allow to use them from tools without special MSR
27 * The events only support system-wide mode counting. There is no
28 * sampling support because it is not supported by the hardware.
30 * According to counters' scope and category, two PMUs are registered
31 * with the perf_event core subsystem.
32 * - 'cstate_core': The counter is available for each physical core.
33 * The counters include CORE_C*_RESIDENCY.
34 * - 'cstate_pkg': The counter is available for each physical package.
35 * The counters include PKG_C*_RESIDENCY.
37 * All of these counters are specified in the IntelĀ® 64 and IA-32
38 * Architectures Software Developer.s Manual Vol3b.
40 * Model specific counters:
41 * MSR_CORE_C1_RES: CORE C1 Residency Counter
43 * Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL
45 * Scope: Core (each processor core has a MSR)
46 * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
48 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,GLM,
51 * MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter
53 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
54 * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
55 * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
58 * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
60 * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML,
61 * ICL,TGL,RKL,ADL,RPL,MTL
63 * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter.
65 * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL,
66 * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL,
68 * Scope: Package (physical package)
69 * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter.
71 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL,
72 * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL,
74 * Scope: Package (physical package)
75 * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter.
77 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
78 * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
79 * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF
80 * Scope: Package (physical package)
81 * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter.
83 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,CNL,
84 * KBL,CML,ICL,TGL,RKL,ADL,RPL,MTL
85 * Scope: Package (physical package)
86 * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter.
88 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL,
90 * Scope: Package (physical package)
91 * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter.
93 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL,
95 * Scope: Package (physical package)
96 * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
98 * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,
100 * Scope: Package (physical package)
101 * MSR_MODULE_C6_RES_MS: Module C6 Residency Counter.
103 * Available model: SRF,GRR
104 * Scope: A cluster of cores shared L2 cache
108 #include <linux/module.h>
109 #include <linux/slab.h>
110 #include <linux/perf_event.h>
111 #include <linux/nospec.h>
112 #include <asm/cpu_device_id.h>
113 #include <asm/intel-family.h>
114 #include "../perf_event.h"
115 #include "../probe.h"
117 MODULE_LICENSE("GPL");
119 #define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format) \
120 static ssize_t __cstate_##_var##_show(struct device *dev, \
121 struct device_attribute *attr, \
124 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
125 return sprintf(page, _format "\n"); \
127 static struct device_attribute format_attr_##_var = \
128 __ATTR(_name, 0444, __cstate_##_var##_show, NULL)
130 static ssize_t cstate_get_attr_cpumask(struct device *dev,
131 struct device_attribute *attr,
134 /* Model -> events mapping */
135 struct cstate_model {
136 unsigned long core_events;
137 unsigned long pkg_events;
138 unsigned long module_events;
139 unsigned long quirks;
143 #define SLM_PKG_C6_USE_C7_MSR (1UL << 0)
144 #define KNL_CORE_C6_MSR (1UL << 1)
146 struct perf_cstate_msr {
148 struct perf_pmu_events_attr *attr;
152 /* cstate_core PMU */
153 static struct pmu cstate_core_pmu;
154 static bool has_cstate_core;
156 enum perf_cstate_core_events {
157 PERF_CSTATE_CORE_C1_RES = 0,
158 PERF_CSTATE_CORE_C3_RES,
159 PERF_CSTATE_CORE_C6_RES,
160 PERF_CSTATE_CORE_C7_RES,
162 PERF_CSTATE_CORE_EVENT_MAX,
165 PMU_EVENT_ATTR_STRING(c1-residency, attr_cstate_core_c1, "event=0x00");
166 PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_core_c3, "event=0x01");
167 PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_core_c6, "event=0x02");
168 PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_core_c7, "event=0x03");
170 static unsigned long core_msr_mask;
172 PMU_EVENT_GROUP(events, cstate_core_c1);
173 PMU_EVENT_GROUP(events, cstate_core_c3);
174 PMU_EVENT_GROUP(events, cstate_core_c6);
175 PMU_EVENT_GROUP(events, cstate_core_c7);
177 static bool test_msr(int idx, void *data)
179 return test_bit(idx, (unsigned long *) data);
182 static struct perf_msr core_msr[] = {
183 [PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES, &group_cstate_core_c1, test_msr },
184 [PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY, &group_cstate_core_c3, test_msr },
185 [PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY, &group_cstate_core_c6, test_msr },
186 [PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY, &group_cstate_core_c7, test_msr },
189 static struct attribute *attrs_empty[] = {
194 * There are no default events, but we need to create
195 * "events" group (with empty attrs) before updating
196 * it with detected events.
198 static struct attribute_group cstate_events_attr_group = {
200 .attrs = attrs_empty,
203 DEFINE_CSTATE_FORMAT_ATTR(cstate_event, event, "config:0-63");
204 static struct attribute *cstate_format_attrs[] = {
205 &format_attr_cstate_event.attr,
209 static struct attribute_group cstate_format_attr_group = {
211 .attrs = cstate_format_attrs,
214 static cpumask_t cstate_core_cpu_mask;
215 static DEVICE_ATTR(cpumask, S_IRUGO, cstate_get_attr_cpumask, NULL);
217 static struct attribute *cstate_cpumask_attrs[] = {
218 &dev_attr_cpumask.attr,
222 static struct attribute_group cpumask_attr_group = {
223 .attrs = cstate_cpumask_attrs,
226 static const struct attribute_group *cstate_attr_groups[] = {
227 &cstate_events_attr_group,
228 &cstate_format_attr_group,
234 static struct pmu cstate_pkg_pmu;
235 static bool has_cstate_pkg;
237 enum perf_cstate_pkg_events {
238 PERF_CSTATE_PKG_C2_RES = 0,
239 PERF_CSTATE_PKG_C3_RES,
240 PERF_CSTATE_PKG_C6_RES,
241 PERF_CSTATE_PKG_C7_RES,
242 PERF_CSTATE_PKG_C8_RES,
243 PERF_CSTATE_PKG_C9_RES,
244 PERF_CSTATE_PKG_C10_RES,
246 PERF_CSTATE_PKG_EVENT_MAX,
249 PMU_EVENT_ATTR_STRING(c2-residency, attr_cstate_pkg_c2, "event=0x00");
250 PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_pkg_c3, "event=0x01");
251 PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_pkg_c6, "event=0x02");
252 PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_pkg_c7, "event=0x03");
253 PMU_EVENT_ATTR_STRING(c8-residency, attr_cstate_pkg_c8, "event=0x04");
254 PMU_EVENT_ATTR_STRING(c9-residency, attr_cstate_pkg_c9, "event=0x05");
255 PMU_EVENT_ATTR_STRING(c10-residency, attr_cstate_pkg_c10, "event=0x06");
257 static unsigned long pkg_msr_mask;
259 PMU_EVENT_GROUP(events, cstate_pkg_c2);
260 PMU_EVENT_GROUP(events, cstate_pkg_c3);
261 PMU_EVENT_GROUP(events, cstate_pkg_c6);
262 PMU_EVENT_GROUP(events, cstate_pkg_c7);
263 PMU_EVENT_GROUP(events, cstate_pkg_c8);
264 PMU_EVENT_GROUP(events, cstate_pkg_c9);
265 PMU_EVENT_GROUP(events, cstate_pkg_c10);
267 static struct perf_msr pkg_msr[] = {
268 [PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY, &group_cstate_pkg_c2, test_msr },
269 [PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY, &group_cstate_pkg_c3, test_msr },
270 [PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY, &group_cstate_pkg_c6, test_msr },
271 [PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY, &group_cstate_pkg_c7, test_msr },
272 [PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY, &group_cstate_pkg_c8, test_msr },
273 [PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY, &group_cstate_pkg_c9, test_msr },
274 [PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY, &group_cstate_pkg_c10, test_msr },
277 static cpumask_t cstate_pkg_cpu_mask;
279 /* cstate_module PMU */
280 static struct pmu cstate_module_pmu;
281 static bool has_cstate_module;
283 enum perf_cstate_module_events {
284 PERF_CSTATE_MODULE_C6_RES = 0,
286 PERF_CSTATE_MODULE_EVENT_MAX,
289 PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_module_c6, "event=0x00");
291 static unsigned long module_msr_mask;
293 PMU_EVENT_GROUP(events, cstate_module_c6);
295 static struct perf_msr module_msr[] = {
296 [PERF_CSTATE_MODULE_C6_RES] = { MSR_MODULE_C6_RES_MS, &group_cstate_module_c6, test_msr },
299 static cpumask_t cstate_module_cpu_mask;
301 static ssize_t cstate_get_attr_cpumask(struct device *dev,
302 struct device_attribute *attr,
305 struct pmu *pmu = dev_get_drvdata(dev);
307 if (pmu == &cstate_core_pmu)
308 return cpumap_print_to_pagebuf(true, buf, &cstate_core_cpu_mask);
309 else if (pmu == &cstate_pkg_pmu)
310 return cpumap_print_to_pagebuf(true, buf, &cstate_pkg_cpu_mask);
311 else if (pmu == &cstate_module_pmu)
312 return cpumap_print_to_pagebuf(true, buf, &cstate_module_cpu_mask);
317 static int cstate_pmu_event_init(struct perf_event *event)
319 u64 cfg = event->attr.config;
322 if (event->attr.type != event->pmu->type)
325 /* unsupported modes and filters */
326 if (event->attr.sample_period) /* no sampling */
332 if (event->pmu == &cstate_core_pmu) {
333 if (cfg >= PERF_CSTATE_CORE_EVENT_MAX)
335 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_CORE_EVENT_MAX);
336 if (!(core_msr_mask & (1 << cfg)))
338 event->hw.event_base = core_msr[cfg].msr;
339 cpu = cpumask_any_and(&cstate_core_cpu_mask,
340 topology_sibling_cpumask(event->cpu));
341 } else if (event->pmu == &cstate_pkg_pmu) {
342 if (cfg >= PERF_CSTATE_PKG_EVENT_MAX)
344 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX);
345 if (!(pkg_msr_mask & (1 << cfg)))
348 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
350 event->hw.event_base = pkg_msr[cfg].msr;
351 cpu = cpumask_any_and(&cstate_pkg_cpu_mask,
352 topology_die_cpumask(event->cpu));
353 } else if (event->pmu == &cstate_module_pmu) {
354 if (cfg >= PERF_CSTATE_MODULE_EVENT_MAX)
356 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_MODULE_EVENT_MAX);
357 if (!(module_msr_mask & (1 << cfg)))
359 event->hw.event_base = module_msr[cfg].msr;
360 cpu = cpumask_any_and(&cstate_module_cpu_mask,
361 topology_cluster_cpumask(event->cpu));
366 if (cpu >= nr_cpu_ids)
370 event->hw.config = cfg;
375 static inline u64 cstate_pmu_read_counter(struct perf_event *event)
379 rdmsrl(event->hw.event_base, val);
383 static void cstate_pmu_event_update(struct perf_event *event)
385 struct hw_perf_event *hwc = &event->hw;
386 u64 prev_raw_count, new_raw_count;
388 prev_raw_count = local64_read(&hwc->prev_count);
390 new_raw_count = cstate_pmu_read_counter(event);
391 } while (!local64_try_cmpxchg(&hwc->prev_count,
392 &prev_raw_count, new_raw_count));
394 local64_add(new_raw_count - prev_raw_count, &event->count);
397 static void cstate_pmu_event_start(struct perf_event *event, int mode)
399 local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event));
402 static void cstate_pmu_event_stop(struct perf_event *event, int mode)
404 cstate_pmu_event_update(event);
407 static void cstate_pmu_event_del(struct perf_event *event, int mode)
409 cstate_pmu_event_stop(event, PERF_EF_UPDATE);
412 static int cstate_pmu_event_add(struct perf_event *event, int mode)
414 if (mode & PERF_EF_START)
415 cstate_pmu_event_start(event, mode);
421 * Check if exiting cpu is the designated reader. If so migrate the
422 * events when there is a valid target available
424 static int cstate_cpu_exit(unsigned int cpu)
428 if (has_cstate_core &&
429 cpumask_test_and_clear_cpu(cpu, &cstate_core_cpu_mask)) {
431 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
432 /* Migrate events if there is a valid target */
433 if (target < nr_cpu_ids) {
434 cpumask_set_cpu(target, &cstate_core_cpu_mask);
435 perf_pmu_migrate_context(&cstate_core_pmu, cpu, target);
439 if (has_cstate_pkg &&
440 cpumask_test_and_clear_cpu(cpu, &cstate_pkg_cpu_mask)) {
442 target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
443 /* Migrate events if there is a valid target */
444 if (target < nr_cpu_ids) {
445 cpumask_set_cpu(target, &cstate_pkg_cpu_mask);
446 perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target);
450 if (has_cstate_module &&
451 cpumask_test_and_clear_cpu(cpu, &cstate_module_cpu_mask)) {
453 target = cpumask_any_but(topology_cluster_cpumask(cpu), cpu);
454 /* Migrate events if there is a valid target */
455 if (target < nr_cpu_ids) {
456 cpumask_set_cpu(target, &cstate_module_cpu_mask);
457 perf_pmu_migrate_context(&cstate_module_pmu, cpu, target);
463 static int cstate_cpu_init(unsigned int cpu)
468 * If this is the first online thread of that core, set it in
469 * the core cpu mask as the designated reader.
471 target = cpumask_any_and(&cstate_core_cpu_mask,
472 topology_sibling_cpumask(cpu));
474 if (has_cstate_core && target >= nr_cpu_ids)
475 cpumask_set_cpu(cpu, &cstate_core_cpu_mask);
478 * If this is the first online thread of that package, set it
479 * in the package cpu mask as the designated reader.
481 target = cpumask_any_and(&cstate_pkg_cpu_mask,
482 topology_die_cpumask(cpu));
483 if (has_cstate_pkg && target >= nr_cpu_ids)
484 cpumask_set_cpu(cpu, &cstate_pkg_cpu_mask);
487 * If this is the first online thread of that cluster, set it
488 * in the cluster cpu mask as the designated reader.
490 target = cpumask_any_and(&cstate_module_cpu_mask,
491 topology_cluster_cpumask(cpu));
492 if (has_cstate_module && target >= nr_cpu_ids)
493 cpumask_set_cpu(cpu, &cstate_module_cpu_mask);
498 static const struct attribute_group *core_attr_update[] = {
499 &group_cstate_core_c1,
500 &group_cstate_core_c3,
501 &group_cstate_core_c6,
502 &group_cstate_core_c7,
506 static const struct attribute_group *pkg_attr_update[] = {
507 &group_cstate_pkg_c2,
508 &group_cstate_pkg_c3,
509 &group_cstate_pkg_c6,
510 &group_cstate_pkg_c7,
511 &group_cstate_pkg_c8,
512 &group_cstate_pkg_c9,
513 &group_cstate_pkg_c10,
517 static const struct attribute_group *module_attr_update[] = {
518 &group_cstate_module_c6,
522 static struct pmu cstate_core_pmu = {
523 .attr_groups = cstate_attr_groups,
524 .attr_update = core_attr_update,
525 .name = "cstate_core",
526 .task_ctx_nr = perf_invalid_context,
527 .event_init = cstate_pmu_event_init,
528 .add = cstate_pmu_event_add,
529 .del = cstate_pmu_event_del,
530 .start = cstate_pmu_event_start,
531 .stop = cstate_pmu_event_stop,
532 .read = cstate_pmu_event_update,
533 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
534 .module = THIS_MODULE,
537 static struct pmu cstate_pkg_pmu = {
538 .attr_groups = cstate_attr_groups,
539 .attr_update = pkg_attr_update,
540 .name = "cstate_pkg",
541 .task_ctx_nr = perf_invalid_context,
542 .event_init = cstate_pmu_event_init,
543 .add = cstate_pmu_event_add,
544 .del = cstate_pmu_event_del,
545 .start = cstate_pmu_event_start,
546 .stop = cstate_pmu_event_stop,
547 .read = cstate_pmu_event_update,
548 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
549 .module = THIS_MODULE,
552 static struct pmu cstate_module_pmu = {
553 .attr_groups = cstate_attr_groups,
554 .attr_update = module_attr_update,
555 .name = "cstate_module",
556 .task_ctx_nr = perf_invalid_context,
557 .event_init = cstate_pmu_event_init,
558 .add = cstate_pmu_event_add,
559 .del = cstate_pmu_event_del,
560 .start = cstate_pmu_event_start,
561 .stop = cstate_pmu_event_stop,
562 .read = cstate_pmu_event_update,
563 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
564 .module = THIS_MODULE,
567 static const struct cstate_model nhm_cstates __initconst = {
568 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
569 BIT(PERF_CSTATE_CORE_C6_RES),
571 .pkg_events = BIT(PERF_CSTATE_PKG_C3_RES) |
572 BIT(PERF_CSTATE_PKG_C6_RES) |
573 BIT(PERF_CSTATE_PKG_C7_RES),
576 static const struct cstate_model snb_cstates __initconst = {
577 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
578 BIT(PERF_CSTATE_CORE_C6_RES) |
579 BIT(PERF_CSTATE_CORE_C7_RES),
581 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
582 BIT(PERF_CSTATE_PKG_C3_RES) |
583 BIT(PERF_CSTATE_PKG_C6_RES) |
584 BIT(PERF_CSTATE_PKG_C7_RES),
587 static const struct cstate_model hswult_cstates __initconst = {
588 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
589 BIT(PERF_CSTATE_CORE_C6_RES) |
590 BIT(PERF_CSTATE_CORE_C7_RES),
592 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
593 BIT(PERF_CSTATE_PKG_C3_RES) |
594 BIT(PERF_CSTATE_PKG_C6_RES) |
595 BIT(PERF_CSTATE_PKG_C7_RES) |
596 BIT(PERF_CSTATE_PKG_C8_RES) |
597 BIT(PERF_CSTATE_PKG_C9_RES) |
598 BIT(PERF_CSTATE_PKG_C10_RES),
601 static const struct cstate_model cnl_cstates __initconst = {
602 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
603 BIT(PERF_CSTATE_CORE_C3_RES) |
604 BIT(PERF_CSTATE_CORE_C6_RES) |
605 BIT(PERF_CSTATE_CORE_C7_RES),
607 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
608 BIT(PERF_CSTATE_PKG_C3_RES) |
609 BIT(PERF_CSTATE_PKG_C6_RES) |
610 BIT(PERF_CSTATE_PKG_C7_RES) |
611 BIT(PERF_CSTATE_PKG_C8_RES) |
612 BIT(PERF_CSTATE_PKG_C9_RES) |
613 BIT(PERF_CSTATE_PKG_C10_RES),
616 static const struct cstate_model icl_cstates __initconst = {
617 .core_events = BIT(PERF_CSTATE_CORE_C6_RES) |
618 BIT(PERF_CSTATE_CORE_C7_RES),
620 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
621 BIT(PERF_CSTATE_PKG_C3_RES) |
622 BIT(PERF_CSTATE_PKG_C6_RES) |
623 BIT(PERF_CSTATE_PKG_C7_RES) |
624 BIT(PERF_CSTATE_PKG_C8_RES) |
625 BIT(PERF_CSTATE_PKG_C9_RES) |
626 BIT(PERF_CSTATE_PKG_C10_RES),
629 static const struct cstate_model icx_cstates __initconst = {
630 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
631 BIT(PERF_CSTATE_CORE_C6_RES),
633 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
634 BIT(PERF_CSTATE_PKG_C6_RES),
637 static const struct cstate_model adl_cstates __initconst = {
638 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
639 BIT(PERF_CSTATE_CORE_C6_RES) |
640 BIT(PERF_CSTATE_CORE_C7_RES),
642 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
643 BIT(PERF_CSTATE_PKG_C3_RES) |
644 BIT(PERF_CSTATE_PKG_C6_RES) |
645 BIT(PERF_CSTATE_PKG_C7_RES) |
646 BIT(PERF_CSTATE_PKG_C8_RES) |
647 BIT(PERF_CSTATE_PKG_C9_RES) |
648 BIT(PERF_CSTATE_PKG_C10_RES),
651 static const struct cstate_model slm_cstates __initconst = {
652 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
653 BIT(PERF_CSTATE_CORE_C6_RES),
655 .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES),
656 .quirks = SLM_PKG_C6_USE_C7_MSR,
660 static const struct cstate_model knl_cstates __initconst = {
661 .core_events = BIT(PERF_CSTATE_CORE_C6_RES),
663 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
664 BIT(PERF_CSTATE_PKG_C3_RES) |
665 BIT(PERF_CSTATE_PKG_C6_RES),
666 .quirks = KNL_CORE_C6_MSR,
670 static const struct cstate_model glm_cstates __initconst = {
671 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
672 BIT(PERF_CSTATE_CORE_C3_RES) |
673 BIT(PERF_CSTATE_CORE_C6_RES),
675 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
676 BIT(PERF_CSTATE_PKG_C3_RES) |
677 BIT(PERF_CSTATE_PKG_C6_RES) |
678 BIT(PERF_CSTATE_PKG_C10_RES),
681 static const struct cstate_model grr_cstates __initconst = {
682 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
683 BIT(PERF_CSTATE_CORE_C6_RES),
685 .module_events = BIT(PERF_CSTATE_MODULE_C6_RES),
688 static const struct cstate_model srf_cstates __initconst = {
689 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
690 BIT(PERF_CSTATE_CORE_C6_RES),
692 .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES),
694 .module_events = BIT(PERF_CSTATE_MODULE_C6_RES),
698 static const struct x86_cpu_id intel_cstates_match[] __initconst = {
699 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &nhm_cstates),
700 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &nhm_cstates),
701 X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &nhm_cstates),
703 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &nhm_cstates),
704 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &nhm_cstates),
705 X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &nhm_cstates),
707 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &snb_cstates),
708 X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &snb_cstates),
710 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &snb_cstates),
711 X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &snb_cstates),
713 X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &snb_cstates),
714 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &snb_cstates),
715 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &snb_cstates),
717 X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &hswult_cstates),
719 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &slm_cstates),
720 X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &slm_cstates),
721 X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &slm_cstates),
723 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &snb_cstates),
724 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &snb_cstates),
725 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &snb_cstates),
726 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &snb_cstates),
728 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &snb_cstates),
729 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &snb_cstates),
730 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &snb_cstates),
732 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &hswult_cstates),
733 X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &hswult_cstates),
734 X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &hswult_cstates),
735 X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &hswult_cstates),
737 X86_MATCH_INTEL_FAM6_MODEL(CANNONLAKE_L, &cnl_cstates),
739 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &knl_cstates),
740 X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &knl_cstates),
742 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &glm_cstates),
743 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &glm_cstates),
744 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &glm_cstates),
745 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &glm_cstates),
746 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, &glm_cstates),
747 X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L, &glm_cstates),
748 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, &adl_cstates),
749 X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT_X, &srf_cstates),
750 X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT, &grr_cstates),
752 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &icl_cstates),
753 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &icl_cstates),
754 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &icx_cstates),
755 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &icx_cstates),
756 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &icx_cstates),
757 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &icx_cstates),
758 X86_MATCH_INTEL_FAM6_MODEL(GRANITERAPIDS_X, &icx_cstates),
759 X86_MATCH_INTEL_FAM6_MODEL(GRANITERAPIDS_D, &icx_cstates),
761 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &icl_cstates),
762 X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &icl_cstates),
763 X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &icl_cstates),
764 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_cstates),
765 X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &adl_cstates),
766 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_cstates),
767 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &adl_cstates),
768 X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &adl_cstates),
769 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE, &adl_cstates),
770 X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &adl_cstates),
773 MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);
775 static int __init cstate_probe(const struct cstate_model *cm)
777 /* SLM has different MSR for PKG C6 */
778 if (cm->quirks & SLM_PKG_C6_USE_C7_MSR)
779 pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY;
781 /* KNL has different MSR for CORE C6 */
782 if (cm->quirks & KNL_CORE_C6_MSR)
783 pkg_msr[PERF_CSTATE_CORE_C6_RES].msr = MSR_KNL_CORE_C6_RESIDENCY;
786 core_msr_mask = perf_msr_probe(core_msr, PERF_CSTATE_CORE_EVENT_MAX,
787 true, (void *) &cm->core_events);
789 pkg_msr_mask = perf_msr_probe(pkg_msr, PERF_CSTATE_PKG_EVENT_MAX,
790 true, (void *) &cm->pkg_events);
792 module_msr_mask = perf_msr_probe(module_msr, PERF_CSTATE_MODULE_EVENT_MAX,
793 true, (void *) &cm->module_events);
795 has_cstate_core = !!core_msr_mask;
796 has_cstate_pkg = !!pkg_msr_mask;
797 has_cstate_module = !!module_msr_mask;
799 return (has_cstate_core || has_cstate_pkg || has_cstate_module) ? 0 : -ENODEV;
802 static inline void cstate_cleanup(void)
804 cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_ONLINE);
805 cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_STARTING);
808 perf_pmu_unregister(&cstate_core_pmu);
811 perf_pmu_unregister(&cstate_pkg_pmu);
813 if (has_cstate_module)
814 perf_pmu_unregister(&cstate_module_pmu);
817 static int __init cstate_init(void)
821 cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_STARTING,
822 "perf/x86/cstate:starting", cstate_cpu_init, NULL);
823 cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_ONLINE,
824 "perf/x86/cstate:online", NULL, cstate_cpu_exit);
826 if (has_cstate_core) {
827 err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1);
829 has_cstate_core = false;
830 pr_info("Failed to register cstate core pmu\n");
836 if (has_cstate_pkg) {
837 if (topology_max_dies_per_package() > 1) {
838 err = perf_pmu_register(&cstate_pkg_pmu,
841 err = perf_pmu_register(&cstate_pkg_pmu,
842 cstate_pkg_pmu.name, -1);
845 has_cstate_pkg = false;
846 pr_info("Failed to register cstate pkg pmu\n");
852 if (has_cstate_module) {
853 err = perf_pmu_register(&cstate_module_pmu, cstate_module_pmu.name, -1);
855 has_cstate_module = false;
856 pr_info("Failed to register cstate cluster pmu\n");
864 static int __init cstate_pmu_init(void)
866 const struct x86_cpu_id *id;
869 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
872 id = x86_match_cpu(intel_cstates_match);
876 err = cstate_probe((const struct cstate_model *) id->driver_data);
880 return cstate_init();
882 module_init(cstate_pmu_init);
884 static void __exit cstate_pmu_exit(void)
888 module_exit(cstate_pmu_exit);