arch/x86/events/intel/uncore.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #include <linux/slab.h>
   3 #include <linux/pci.h>
   4 #include <asm/apicdef.h>
   5 #include <linux/io-64-nonatomic-lo-hi.h>
   6
   7 #include <linux/perf_event.h>
   8 #include "../perf_event.h"
   9
  10 #define UNCORE_PMU_NAME_LEN             32
  11 #define UNCORE_PMU_HRTIMER_INTERVAL     (60LL * NSEC_PER_SEC)
  12 #define UNCORE_SNB_IMC_HRTIMER_INTERVAL (5ULL * NSEC_PER_SEC)
  13
  14 #define UNCORE_FIXED_EVENT              0xff
  15 #define UNCORE_PMC_IDX_MAX_GENERIC      8
  16 #define UNCORE_PMC_IDX_MAX_FIXED        1
  17 #define UNCORE_PMC_IDX_MAX_FREERUNNING  1
  18 #define UNCORE_PMC_IDX_FIXED            UNCORE_PMC_IDX_MAX_GENERIC
  19 #define UNCORE_PMC_IDX_FREERUNNING      (UNCORE_PMC_IDX_FIXED + \
  20                                         UNCORE_PMC_IDX_MAX_FIXED)
  21 #define UNCORE_PMC_IDX_MAX              (UNCORE_PMC_IDX_FREERUNNING + \
  22                                         UNCORE_PMC_IDX_MAX_FREERUNNING)
  23
  24 #define UNCORE_PCI_DEV_FULL_DATA(dev, func, type, idx)  \
  25                 ((dev << 24) | (func << 16) | (type << 8) | idx)
  26 #define UNCORE_PCI_DEV_DATA(type, idx)  ((type << 8) | idx)
  27 #define UNCORE_PCI_DEV_DEV(data)        ((data >> 24) & 0xff)
  28 #define UNCORE_PCI_DEV_FUNC(data)       ((data >> 16) & 0xff)
  29 #define UNCORE_PCI_DEV_TYPE(data)       ((data >> 8) & 0xff)
  30 #define UNCORE_PCI_DEV_IDX(data)        (data & 0xff)
  31 #define UNCORE_EXTRA_PCI_DEV            0xff
  32 #define UNCORE_EXTRA_PCI_DEV_MAX        4
  33
  34 #define UNCORE_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, 0xff)
  35
  36 struct pci_extra_dev {
  37         struct pci_dev *dev[UNCORE_EXTRA_PCI_DEV_MAX];
  38 };
  39
  40 struct intel_uncore_ops;
  41 struct intel_uncore_pmu;
  42 struct intel_uncore_box;
  43 struct uncore_event_desc;
  44 struct freerunning_counters;
  45
  46 struct intel_uncore_type {
  47         const char *name;
  48         int num_counters;
  49         int num_boxes;
  50         int perf_ctr_bits;
  51         int fixed_ctr_bits;
  52         int num_freerunning_types;
  53         unsigned perf_ctr;
  54         unsigned event_ctl;
  55         unsigned event_mask;
  56         unsigned event_mask_ext;
  57         unsigned fixed_ctr;
  58         unsigned fixed_ctl;
  59         unsigned box_ctl;
  60         union {
  61                 unsigned msr_offset;
  62                 unsigned mmio_offset;
  63         };
  64         unsigned mmio_map_size;
  65         unsigned num_shared_regs:8;
  66         unsigned single_fixed:1;
  67         unsigned pair_ctr_ctl:1;
  68         unsigned *msr_offsets;
  69         struct event_constraint unconstrainted;
  70         struct event_constraint *constraints;
  71         struct intel_uncore_pmu *pmus;
  72         struct intel_uncore_ops *ops;
  73         struct uncore_event_desc *event_descs;
  74         struct freerunning_counters *freerunning;
  75         const struct attribute_group *attr_groups[4];
  76         struct pmu *pmu; /* for custom pmu ops */
  77 };
  78
  79 #define pmu_group attr_groups[0]
  80 #define format_group attr_groups[1]
  81 #define events_group attr_groups[2]
  82
  83 struct intel_uncore_ops {
  84         void (*init_box)(struct intel_uncore_box *);
  85         void (*exit_box)(struct intel_uncore_box *);
  86         void (*disable_box)(struct intel_uncore_box *);
  87         void (*enable_box)(struct intel_uncore_box *);
  88         void (*disable_event)(struct intel_uncore_box *, struct perf_event *);
  89         void (*enable_event)(struct intel_uncore_box *, struct perf_event *);
  90         u64 (*read_counter)(struct intel_uncore_box *, struct perf_event *);
  91         int (*hw_config)(struct intel_uncore_box *, struct perf_event *);
  92         struct event_constraint *(*get_constraint)(struct intel_uncore_box *,
  93                                                    struct perf_event *);
  94         void (*put_constraint)(struct intel_uncore_box *, struct perf_event *);
  95 };
  96
  97 struct intel_uncore_pmu {
  98         struct pmu                      pmu;
  99         char                            name[UNCORE_PMU_NAME_LEN];
 100         int                             pmu_idx;
 101         int                             func_id;
 102         bool                            registered;
 103         atomic_t                        activeboxes;
 104         struct intel_uncore_type        *type;
 105         struct intel_uncore_box         **boxes;
 106 };
 107
 108 struct intel_uncore_extra_reg {
 109         raw_spinlock_t lock;
 110         u64 config, config1, config2;
 111         atomic_t ref;
 112 };
 113
 114 struct intel_uncore_box {
 115         int pci_phys_id;
 116         int dieid;      /* Logical die ID */
 117         int n_active;   /* number of active events */
 118         int n_events;
 119         int cpu;        /* cpu to collect events */
 120         unsigned long flags;
 121         atomic_t refcnt;
 122         struct perf_event *events[UNCORE_PMC_IDX_MAX];
 123         struct perf_event *event_list[UNCORE_PMC_IDX_MAX];
 124         struct event_constraint *event_constraint[UNCORE_PMC_IDX_MAX];
 125         unsigned long active_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
 126         u64 tags[UNCORE_PMC_IDX_MAX];
 127         struct pci_dev *pci_dev;
 128         struct intel_uncore_pmu *pmu;
 129         u64 hrtimer_duration; /* hrtimer timeout for this box */
 130         struct hrtimer hrtimer;
 131         struct list_head list;
 132         struct list_head active_list;
 133         void __iomem *io_addr;
 134         struct intel_uncore_extra_reg shared_regs[];
 135 };
 136
 137 /* CFL uncore 8th cbox MSRs */
 138 #define CFL_UNC_CBO_7_PERFEVTSEL0               0xf70
 139 #define CFL_UNC_CBO_7_PER_CTR0                  0xf76
 140
 141 #define UNCORE_BOX_FLAG_INITIATED               0
 142 /* event config registers are 8-byte apart */
 143 #define UNCORE_BOX_FLAG_CTL_OFFS8               1
 144 /* CFL 8th CBOX has different MSR space */
 145 #define UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS      2
 146
 147 struct uncore_event_desc {
 148         struct kobj_attribute attr;
 149         const char *config;
 150 };
 151
 152 struct freerunning_counters {
 153         unsigned int counter_base;
 154         unsigned int counter_offset;
 155         unsigned int box_offset;
 156         unsigned int num_counters;
 157         unsigned int bits;
 158         unsigned *box_offsets;
 159 };
 160
 161 struct pci2phy_map {
 162         struct list_head list;
 163         int segment;
 164         int pbus_to_physid[256];
 165 };
 166
 167 struct pci2phy_map *__find_pci2phy_map(int segment);
 168 int uncore_pcibus_to_physid(struct pci_bus *bus);
 169
 170 ssize_t uncore_event_show(struct kobject *kobj,
 171                           struct kobj_attribute *attr, char *buf);
 172
 173 #define INTEL_UNCORE_EVENT_DESC(_name, _config)                 \
 174 {                                                               \
 175         .attr   = __ATTR(_name, 0444, uncore_event_show, NULL), \
 176         .config = _config,                                      \
 177 }
 178
 179 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format)                 \
 180 static ssize_t __uncore_##_var##_show(struct kobject *kobj,             \
 181                                 struct kobj_attribute *attr,            \
 182                                 char *page)                             \
 183 {                                                                       \
 184         BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);                     \
 185         return sprintf(page, _format "\n");                             \
 186 }                                                                       \
 187 static struct kobj_attribute format_attr_##_var =                       \
 188         __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
 189
 190 static inline bool uncore_pmc_fixed(int idx)
 191 {
 192         return idx == UNCORE_PMC_IDX_FIXED;
 193 }
 194
 195 static inline bool uncore_pmc_freerunning(int idx)
 196 {
 197         return idx == UNCORE_PMC_IDX_FREERUNNING;
 198 }
 199
 200 static inline
 201 unsigned int uncore_mmio_box_ctl(struct intel_uncore_box *box)
 202 {
 203         return box->pmu->type->box_ctl +
 204                box->pmu->type->mmio_offset * box->pmu->pmu_idx;
 205 }
 206
 207 static inline unsigned uncore_pci_box_ctl(struct intel_uncore_box *box)
 208 {
 209         return box->pmu->type->box_ctl;
 210 }
 211
 212 static inline unsigned uncore_pci_fixed_ctl(struct intel_uncore_box *box)
 213 {
 214         return box->pmu->type->fixed_ctl;
 215 }
 216
 217 static inline unsigned uncore_pci_fixed_ctr(struct intel_uncore_box *box)
 218 {
 219         return box->pmu->type->fixed_ctr;
 220 }
 221
 222 static inline
 223 unsigned uncore_pci_event_ctl(struct intel_uncore_box *box, int idx)
 224 {
 225         if (test_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags))
 226                 return idx * 8 + box->pmu->type->event_ctl;
 227
 228         return idx * 4 + box->pmu->type->event_ctl;
 229 }
 230
 231 static inline
 232 unsigned uncore_pci_perf_ctr(struct intel_uncore_box *box, int idx)
 233 {
 234         return idx * 8 + box->pmu->type->perf_ctr;
 235 }
 236
 237 static inline unsigned uncore_msr_box_offset(struct intel_uncore_box *box)
 238 {
 239         struct intel_uncore_pmu *pmu = box->pmu;
 240         return pmu->type->msr_offsets ?
 241                 pmu->type->msr_offsets[pmu->pmu_idx] :
 242                 pmu->type->msr_offset * pmu->pmu_idx;
 243 }
 244
 245 static inline unsigned uncore_msr_box_ctl(struct intel_uncore_box *box)
 246 {
 247         if (!box->pmu->type->box_ctl)
 248                 return 0;
 249         return box->pmu->type->box_ctl + uncore_msr_box_offset(box);
 250 }
 251
 252 static inline unsigned uncore_msr_fixed_ctl(struct intel_uncore_box *box)
 253 {
 254         if (!box->pmu->type->fixed_ctl)
 255                 return 0;
 256         return box->pmu->type->fixed_ctl + uncore_msr_box_offset(box);
 257 }
 258
 259 static inline unsigned uncore_msr_fixed_ctr(struct intel_uncore_box *box)
 260 {
 261         return box->pmu->type->fixed_ctr + uncore_msr_box_offset(box);
 262 }
 263
 264
 265 /*
 266  * In the uncore document, there is no event-code assigned to free running
 267  * counters. Some events need to be defined to indicate the free running
 268  * counters. The events are encoded as event-code + umask-code.
 269  *
 270  * The event-code for all free running counters is 0xff, which is the same as
 271  * the fixed counters.
 272  *
 273  * The umask-code is used to distinguish a fixed counter and a free running
 274  * counter, and different types of free running counters.
 275  * - For fixed counters, the umask-code is 0x0X.
 276  *   X indicates the index of the fixed counter, which starts from 0.
 277  * - For free running counters, the umask-code uses the rest of the space.
 278  *   It would bare the format of 0xXY.
 279  *   X stands for the type of free running counters, which starts from 1.
 280  *   Y stands for the index of free running counters of same type, which
 281  *   starts from 0.
 282  *
 283  * For example, there are three types of IIO free running counters on Skylake
 284  * server, IO CLOCKS counters, BANDWIDTH counters and UTILIZATION counters.
 285  * The event-code for all the free running counters is 0xff.
 286  * 'ioclk' is the first counter of IO CLOCKS. IO CLOCKS is the first type,
 287  * which umask-code starts from 0x10.
 288  * So 'ioclk' is encoded as event=0xff,umask=0x10
 289  * 'bw_in_port2' is the third counter of BANDWIDTH counters. BANDWIDTH is
 290  * the second type, which umask-code starts from 0x20.
 291  * So 'bw_in_port2' is encoded as event=0xff,umask=0x22
 292  */
 293 static inline unsigned int uncore_freerunning_idx(u64 config)
 294 {
 295         return ((config >> 8) & 0xf);
 296 }
 297
 298 #define UNCORE_FREERUNNING_UMASK_START          0x10
 299
 300 static inline unsigned int uncore_freerunning_type(u64 config)
 301 {
 302         return ((((config >> 8) - UNCORE_FREERUNNING_UMASK_START) >> 4) & 0xf);
 303 }
 304
 305 static inline
 306 unsigned int uncore_freerunning_counter(struct intel_uncore_box *box,
 307                                         struct perf_event *event)
 308 {
 309         unsigned int type = uncore_freerunning_type(event->hw.config);
 310         unsigned int idx = uncore_freerunning_idx(event->hw.config);
 311         struct intel_uncore_pmu *pmu = box->pmu;
 312
 313         return pmu->type->freerunning[type].counter_base +
 314                pmu->type->freerunning[type].counter_offset * idx +
 315                (pmu->type->freerunning[type].box_offsets ?
 316                 pmu->type->freerunning[type].box_offsets[pmu->pmu_idx] :
 317                 pmu->type->freerunning[type].box_offset * pmu->pmu_idx);
 318 }
 319
 320 static inline
 321 unsigned uncore_msr_event_ctl(struct intel_uncore_box *box, int idx)
 322 {
 323         if (test_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags)) {
 324                 return CFL_UNC_CBO_7_PERFEVTSEL0 +
 325                        (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx);
 326         } else {
 327                 return box->pmu->type->event_ctl +
 328                        (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
 329                        uncore_msr_box_offset(box);
 330         }
 331 }
 332
 333 static inline
 334 unsigned uncore_msr_perf_ctr(struct intel_uncore_box *box, int idx)
 335 {
 336         if (test_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags)) {
 337                 return CFL_UNC_CBO_7_PER_CTR0 +
 338                        (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx);
 339         } else {
 340                 return box->pmu->type->perf_ctr +
 341                        (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
 342                        uncore_msr_box_offset(box);
 343         }
 344 }
 345
 346 static inline
 347 unsigned uncore_fixed_ctl(struct intel_uncore_box *box)
 348 {
 349         if (box->pci_dev || box->io_addr)
 350                 return uncore_pci_fixed_ctl(box);
 351         else
 352                 return uncore_msr_fixed_ctl(box);
 353 }
 354
 355 static inline
 356 unsigned uncore_fixed_ctr(struct intel_uncore_box *box)
 357 {
 358         if (box->pci_dev || box->io_addr)
 359                 return uncore_pci_fixed_ctr(box);
 360         else
 361                 return uncore_msr_fixed_ctr(box);
 362 }
 363
 364 static inline
 365 unsigned uncore_event_ctl(struct intel_uncore_box *box, int idx)
 366 {
 367         if (box->pci_dev || box->io_addr)
 368                 return uncore_pci_event_ctl(box, idx);
 369         else
 370                 return uncore_msr_event_ctl(box, idx);
 371 }
 372
 373 static inline
 374 unsigned uncore_perf_ctr(struct intel_uncore_box *box, int idx)
 375 {
 376         if (box->pci_dev || box->io_addr)
 377                 return uncore_pci_perf_ctr(box, idx);
 378         else
 379                 return uncore_msr_perf_ctr(box, idx);
 380 }
 381
 382 static inline int uncore_perf_ctr_bits(struct intel_uncore_box *box)
 383 {
 384         return box->pmu->type->perf_ctr_bits;
 385 }
 386
 387 static inline int uncore_fixed_ctr_bits(struct intel_uncore_box *box)
 388 {
 389         return box->pmu->type->fixed_ctr_bits;
 390 }
 391
 392 static inline
 393 unsigned int uncore_freerunning_bits(struct intel_uncore_box *box,
 394                                      struct perf_event *event)
 395 {
 396         unsigned int type = uncore_freerunning_type(event->hw.config);
 397
 398         return box->pmu->type->freerunning[type].bits;
 399 }
 400
 401 static inline int uncore_num_freerunning(struct intel_uncore_box *box,
 402                                          struct perf_event *event)
 403 {
 404         unsigned int type = uncore_freerunning_type(event->hw.config);
 405
 406         return box->pmu->type->freerunning[type].num_counters;
 407 }
 408
 409 static inline int uncore_num_freerunning_types(struct intel_uncore_box *box,
 410                                                struct perf_event *event)
 411 {
 412         return box->pmu->type->num_freerunning_types;
 413 }
 414
 415 static inline bool check_valid_freerunning_event(struct intel_uncore_box *box,
 416                                                  struct perf_event *event)
 417 {
 418         unsigned int type = uncore_freerunning_type(event->hw.config);
 419         unsigned int idx = uncore_freerunning_idx(event->hw.config);
 420
 421         return (type < uncore_num_freerunning_types(box, event)) &&
 422                (idx < uncore_num_freerunning(box, event));
 423 }
 424
 425 static inline int uncore_num_counters(struct intel_uncore_box *box)
 426 {
 427         return box->pmu->type->num_counters;
 428 }
 429
 430 static inline bool is_freerunning_event(struct perf_event *event)
 431 {
 432         u64 cfg = event->attr.config;
 433
 434         return ((cfg & UNCORE_FIXED_EVENT) == UNCORE_FIXED_EVENT) &&
 435                (((cfg >> 8) & 0xff) >= UNCORE_FREERUNNING_UMASK_START);
 436 }
 437
 438 /* Check and reject invalid config */
 439 static inline int uncore_freerunning_hw_config(struct intel_uncore_box *box,
 440                                                struct perf_event *event)
 441 {
 442         if (is_freerunning_event(event))
 443                 return 0;
 444
 445         return -EINVAL;
 446 }
 447
 448 static inline void uncore_disable_event(struct intel_uncore_box *box,
 449                                 struct perf_event *event)
 450 {
 451         box->pmu->type->ops->disable_event(box, event);
 452 }
 453
 454 static inline void uncore_enable_event(struct intel_uncore_box *box,
 455                                 struct perf_event *event)
 456 {
 457         box->pmu->type->ops->enable_event(box, event);
 458 }
 459
 460 static inline u64 uncore_read_counter(struct intel_uncore_box *box,
 461                                 struct perf_event *event)
 462 {
 463         return box->pmu->type->ops->read_counter(box, event);
 464 }
 465
 466 static inline void uncore_box_init(struct intel_uncore_box *box)
 467 {
 468         if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) {
 469                 if (box->pmu->type->ops->init_box)
 470                         box->pmu->type->ops->init_box(box);
 471         }
 472 }
 473
 474 static inline void uncore_box_exit(struct intel_uncore_box *box)
 475 {
 476         if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) {
 477                 if (box->pmu->type->ops->exit_box)
 478                         box->pmu->type->ops->exit_box(box);
 479         }
 480 }
 481
 482 static inline bool uncore_box_is_fake(struct intel_uncore_box *box)
 483 {
 484         return (box->dieid < 0);
 485 }
 486
 487 static inline struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event)
 488 {
 489         return container_of(event->pmu, struct intel_uncore_pmu, pmu);
 490 }
 491
 492 static inline struct intel_uncore_box *uncore_event_to_box(struct perf_event *event)
 493 {
 494         return event->pmu_private;
 495 }
 496
 497 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu);
 498 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event);
 499 void uncore_mmio_exit_box(struct intel_uncore_box *box);
 500 u64 uncore_mmio_read_counter(struct intel_uncore_box *box,
 501                              struct perf_event *event);
 502 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box);
 503 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box);
 504 void uncore_pmu_event_start(struct perf_event *event, int flags);
 505 void uncore_pmu_event_stop(struct perf_event *event, int flags);
 506 int uncore_pmu_event_add(struct perf_event *event, int flags);
 507 void uncore_pmu_event_del(struct perf_event *event, int flags);
 508 void uncore_pmu_event_read(struct perf_event *event);
 509 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event);
 510 struct event_constraint *
 511 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event);
 512 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event);
 513 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx);
 514
 515 extern struct intel_uncore_type **uncore_msr_uncores;
 516 extern struct intel_uncore_type **uncore_pci_uncores;
 517 extern struct intel_uncore_type **uncore_mmio_uncores;
 518 extern struct pci_driver *uncore_pci_driver;
 519 extern raw_spinlock_t pci2phy_map_lock;
 520 extern struct list_head pci2phy_map_head;
 521 extern struct pci_extra_dev *uncore_extra_pci_dev;
 522 extern struct event_constraint uncore_constraint_empty;
 523
 524 /* uncore_snb.c */
 525 int snb_uncore_pci_init(void);
 526 int ivb_uncore_pci_init(void);
 527 int hsw_uncore_pci_init(void);
 528 int bdw_uncore_pci_init(void);
 529 int skl_uncore_pci_init(void);
 530 void snb_uncore_cpu_init(void);
 531 void nhm_uncore_cpu_init(void);
 532 void skl_uncore_cpu_init(void);
 533 void icl_uncore_cpu_init(void);
 534 void tgl_uncore_mmio_init(void);
 535 void tgl_l_uncore_mmio_init(void);
 536 int snb_pci2phy_map_init(int devid);
 537
 538 /* uncore_snbep.c */
 539 int snbep_uncore_pci_init(void);
 540 void snbep_uncore_cpu_init(void);
 541 int ivbep_uncore_pci_init(void);
 542 void ivbep_uncore_cpu_init(void);
 543 int hswep_uncore_pci_init(void);
 544 void hswep_uncore_cpu_init(void);
 545 int bdx_uncore_pci_init(void);
 546 void bdx_uncore_cpu_init(void);
 547 int knl_uncore_pci_init(void);
 548 void knl_uncore_cpu_init(void);
 549 int skx_uncore_pci_init(void);
 550 void skx_uncore_cpu_init(void);
 551 int snr_uncore_pci_init(void);
 552 void snr_uncore_cpu_init(void);
 553 void snr_uncore_mmio_init(void);
 554 int icx_uncore_pci_init(void);
 555 void icx_uncore_cpu_init(void);
 556 void icx_uncore_mmio_init(void);
 557
 558 /* uncore_nhmex.c */
 559 void nhmex_uncore_cpu_init(void);