arch/x86/events/intel/uncore_discovery.c

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Support Intel uncore PerfMon discovery mechanism.
   4  * Copyright(c) 2021 Intel Corporation.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8 #include "uncore.h"
   9 #include "uncore_discovery.h"
  10
  11 static struct rb_root discovery_tables = RB_ROOT;
  12 static int num_discovered_types[UNCORE_ACCESS_MAX];
  13
  14 static bool has_generic_discovery_table(void)
  15 {
  16         struct pci_dev *dev;
  17         int dvsec;
  18
  19         dev = pci_get_device(PCI_VENDOR_ID_INTEL, UNCORE_DISCOVERY_TABLE_DEVICE, NULL);
  20         if (!dev)
  21                 return false;
  22
  23         /* A discovery table device has the unique capability ID. */
  24         dvsec = pci_find_next_ext_capability(dev, 0, UNCORE_EXT_CAP_ID_DISCOVERY);
  25         pci_dev_put(dev);
  26         if (dvsec)
  27                 return true;
  28
  29         return false;
  30 }
  31
  32 static int logical_die_id;
  33
  34 static int get_device_die_id(struct pci_dev *dev)
  35 {
  36         int cpu, node = pcibus_to_node(dev->bus);
  37
  38         /*
  39          * If the NUMA info is not available, assume that the logical die id is
  40          * continuous in the order in which the discovery table devices are
  41          * detected.
  42          */
  43         if (node < 0)
  44                 return logical_die_id++;
  45
  46         for_each_cpu(cpu, cpumask_of_node(node)) {
  47                 struct cpuinfo_x86 *c = &cpu_data(cpu);
  48
  49                 if (c->initialized && cpu_to_node(cpu) == node)
  50                         return c->logical_die_id;
  51         }
  52
  53         /*
  54          * All CPUs of a node may be offlined. For this case,
  55          * the PCI and MMIO type of uncore blocks which are
  56          * enumerated by the device will be unavailable.
  57          */
  58         return -1;
  59 }
  60
  61 #define __node_2_type(cur)      \
  62         rb_entry((cur), struct intel_uncore_discovery_type, node)
  63
  64 static inline int __type_cmp(const void *key, const struct rb_node *b)
  65 {
  66         struct intel_uncore_discovery_type *type_b = __node_2_type(b);
  67         const u16 *type_id = key;
  68
  69         if (type_b->type > *type_id)
  70                 return -1;
  71         else if (type_b->type < *type_id)
  72                 return 1;
  73
  74         return 0;
  75 }
  76
  77 static inline struct intel_uncore_discovery_type *
  78 search_uncore_discovery_type(u16 type_id)
  79 {
  80         struct rb_node *node = rb_find(&type_id, &discovery_tables, __type_cmp);
  81
  82         return (node) ? __node_2_type(node) : NULL;
  83 }
  84
  85 static inline bool __type_less(struct rb_node *a, const struct rb_node *b)
  86 {
  87         return (__node_2_type(a)->type < __node_2_type(b)->type);
  88 }
  89
  90 static struct intel_uncore_discovery_type *
  91 add_uncore_discovery_type(struct uncore_unit_discovery *unit)
  92 {
  93         struct intel_uncore_discovery_type *type;
  94
  95         if (unit->access_type >= UNCORE_ACCESS_MAX) {
  96                 pr_warn("Unsupported access type %d\n", unit->access_type);
  97                 return NULL;
  98         }
  99
 100         type = kzalloc(sizeof(struct intel_uncore_discovery_type), GFP_KERNEL);
 101         if (!type)
 102                 return NULL;
 103
 104         type->box_ctrl_die = kcalloc(__uncore_max_dies, sizeof(u64), GFP_KERNEL);
 105         if (!type->box_ctrl_die)
 106                 goto free_type;
 107
 108         type->access_type = unit->access_type;
 109         num_discovered_types[type->access_type]++;
 110         type->type = unit->box_type;
 111
 112         rb_add(&type->node, &discovery_tables, __type_less);
 113
 114         return type;
 115
 116 free_type:
 117         kfree(type);
 118
 119         return NULL;
 120
 121 }
 122
 123 static struct intel_uncore_discovery_type *
 124 get_uncore_discovery_type(struct uncore_unit_discovery *unit)
 125 {
 126         struct intel_uncore_discovery_type *type;
 127
 128         type = search_uncore_discovery_type(unit->box_type);
 129         if (type)
 130                 return type;
 131
 132         return add_uncore_discovery_type(unit);
 133 }
 134
 135 static void
 136 uncore_insert_box_info(struct uncore_unit_discovery *unit,
 137                        int die, bool parsed)
 138 {
 139         struct intel_uncore_discovery_type *type;
 140         unsigned int *box_offset, *ids;
 141         int i;
 142
 143         if (WARN_ON_ONCE(!unit->ctl || !unit->ctl_offset || !unit->ctr_offset))
 144                 return;
 145
 146         if (parsed) {
 147                 type = search_uncore_discovery_type(unit->box_type);
 148                 if (WARN_ON_ONCE(!type))
 149                         return;
 150                 /* Store the first box of each die */
 151                 if (!type->box_ctrl_die[die])
 152                         type->box_ctrl_die[die] = unit->ctl;
 153                 return;
 154         }
 155
 156         type = get_uncore_discovery_type(unit);
 157         if (!type)
 158                 return;
 159
 160         box_offset = kcalloc(type->num_boxes + 1, sizeof(unsigned int), GFP_KERNEL);
 161         if (!box_offset)
 162                 return;
 163
 164         ids = kcalloc(type->num_boxes + 1, sizeof(unsigned int), GFP_KERNEL);
 165         if (!ids)
 166                 goto free_box_offset;
 167
 168         /* Store generic information for the first box */
 169         if (!type->num_boxes) {
 170                 type->box_ctrl = unit->ctl;
 171                 type->box_ctrl_die[die] = unit->ctl;
 172                 type->num_counters = unit->num_regs;
 173                 type->counter_width = unit->bit_width;
 174                 type->ctl_offset = unit->ctl_offset;
 175                 type->ctr_offset = unit->ctr_offset;
 176                 *ids = unit->box_id;
 177                 goto end;
 178         }
 179
 180         for (i = 0; i < type->num_boxes; i++) {
 181                 ids[i] = type->ids[i];
 182                 box_offset[i] = type->box_offset[i];
 183
 184                 if (WARN_ON_ONCE(unit->box_id == ids[i]))
 185                         goto free_ids;
 186         }
 187         ids[i] = unit->box_id;
 188         box_offset[i] = unit->ctl - type->box_ctrl;
 189         kfree(type->ids);
 190         kfree(type->box_offset);
 191 end:
 192         type->ids = ids;
 193         type->box_offset = box_offset;
 194         type->num_boxes++;
 195         return;
 196
 197 free_ids:
 198         kfree(ids);
 199
 200 free_box_offset:
 201         kfree(box_offset);
 202
 203 }
 204
 205 static int parse_discovery_table(struct pci_dev *dev, int die,
 206                                  u32 bar_offset, bool *parsed)
 207 {
 208         struct uncore_global_discovery global;
 209         struct uncore_unit_discovery unit;
 210         void __iomem *io_addr;
 211         resource_size_t addr;
 212         unsigned long size;
 213         u32 val;
 214         int i;
 215
 216         pci_read_config_dword(dev, bar_offset, &val);
 217
 218         if (val & UNCORE_DISCOVERY_MASK)
 219                 return -EINVAL;
 220
 221         addr = (resource_size_t)(val & ~UNCORE_DISCOVERY_MASK);
 222         size = UNCORE_DISCOVERY_GLOBAL_MAP_SIZE;
 223         io_addr = ioremap(addr, size);
 224         if (!io_addr)
 225                 return -ENOMEM;
 226
 227         /* Read Global Discovery State */
 228         memcpy_fromio(&global, io_addr, sizeof(struct uncore_global_discovery));
 229         if (uncore_discovery_invalid_unit(global)) {
 230                 pr_info("Invalid Global Discovery State: 0x%llx 0x%llx 0x%llx\n",
 231                         global.table1, global.ctl, global.table3);
 232                 iounmap(io_addr);
 233                 return -EINVAL;
 234         }
 235         iounmap(io_addr);
 236
 237         size = (1 + global.max_units) * global.stride * 8;
 238         io_addr = ioremap(addr, size);
 239         if (!io_addr)
 240                 return -ENOMEM;
 241
 242         /* Parsing Unit Discovery State */
 243         for (i = 0; i < global.max_units; i++) {
 244                 memcpy_fromio(&unit, io_addr + (i + 1) * (global.stride * 8),
 245                               sizeof(struct uncore_unit_discovery));
 246
 247                 if (uncore_discovery_invalid_unit(unit))
 248                         continue;
 249
 250                 if (unit.access_type >= UNCORE_ACCESS_MAX)
 251                         continue;
 252
 253                 uncore_insert_box_info(&unit, die, *parsed);
 254         }
 255
 256         *parsed = true;
 257         iounmap(io_addr);
 258         return 0;
 259 }
 260
 261 bool intel_uncore_has_discovery_tables(void)
 262 {
 263         u32 device, val, entry_id, bar_offset;
 264         int die, dvsec = 0, ret = true;
 265         struct pci_dev *dev = NULL;
 266         bool parsed = false;
 267
 268         if (has_generic_discovery_table())
 269                 device = UNCORE_DISCOVERY_TABLE_DEVICE;
 270         else
 271                 device = PCI_ANY_ID;
 272
 273         /*
 274          * Start a new search and iterates through the list of
 275          * the discovery table devices.
 276          */
 277         while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) {
 278                 while ((dvsec = pci_find_next_ext_capability(dev, dvsec, UNCORE_EXT_CAP_ID_DISCOVERY))) {
 279                         pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC_OFFSET, &val);
 280                         entry_id = val & UNCORE_DISCOVERY_DVSEC_ID_MASK;
 281                         if (entry_id != UNCORE_DISCOVERY_DVSEC_ID_PMON)
 282                                 continue;
 283
 284                         pci_read_config_dword(dev, dvsec + UNCORE_DISCOVERY_DVSEC2_OFFSET, &val);
 285
 286                         if (val & ~UNCORE_DISCOVERY_DVSEC2_BIR_MASK) {
 287                                 ret = false;
 288                                 goto err;
 289                         }
 290                         bar_offset = UNCORE_DISCOVERY_BIR_BASE +
 291                                      (val & UNCORE_DISCOVERY_DVSEC2_BIR_MASK) * UNCORE_DISCOVERY_BIR_STEP;
 292
 293                         die = get_device_die_id(dev);
 294                         if (die < 0)
 295                                 continue;
 296
 297                         parse_discovery_table(dev, die, bar_offset, &parsed);
 298                 }
 299         }
 300
 301         /* None of the discovery tables are available */
 302         if (!parsed)
 303                 ret = false;
 304 err:
 305         pci_dev_put(dev);
 306
 307         return ret;
 308 }
 309
 310 void intel_uncore_clear_discovery_tables(void)
 311 {
 312         struct intel_uncore_discovery_type *type, *next;
 313
 314         rbtree_postorder_for_each_entry_safe(type, next, &discovery_tables, node) {
 315                 kfree(type->box_ctrl_die);
 316                 kfree(type);
 317         }
 318 }
 319
 320 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
 321 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
 322 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
 323 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
 324 DEFINE_UNCORE_FORMAT_ATTR(thresh, thresh, "config:24-31");
 325
 326 static struct attribute *generic_uncore_formats_attr[] = {
 327         &format_attr_event.attr,
 328         &format_attr_umask.attr,
 329         &format_attr_edge.attr,
 330         &format_attr_inv.attr,
 331         &format_attr_thresh.attr,
 332         NULL,
 333 };
 334
 335 static const struct attribute_group generic_uncore_format_group = {
 336         .name = "format",
 337         .attrs = generic_uncore_formats_attr,
 338 };
 339
 340 void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box)
 341 {
 342         wrmsrl(uncore_msr_box_ctl(box), GENERIC_PMON_BOX_CTL_INT);
 343 }
 344
 345 void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box)
 346 {
 347         wrmsrl(uncore_msr_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ);
 348 }
 349
 350 void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box)
 351 {
 352         wrmsrl(uncore_msr_box_ctl(box), 0);
 353 }
 354
 355 static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box,
 356                                             struct perf_event *event)
 357 {
 358         struct hw_perf_event *hwc = &event->hw;
 359
 360         wrmsrl(hwc->config_base, hwc->config);
 361 }
 362
 363 static void intel_generic_uncore_msr_disable_event(struct intel_uncore_box *box,
 364                                              struct perf_event *event)
 365 {
 366         struct hw_perf_event *hwc = &event->hw;
 367
 368         wrmsrl(hwc->config_base, 0);
 369 }
 370
 371 static struct intel_uncore_ops generic_uncore_msr_ops = {
 372         .init_box               = intel_generic_uncore_msr_init_box,
 373         .disable_box            = intel_generic_uncore_msr_disable_box,
 374         .enable_box             = intel_generic_uncore_msr_enable_box,
 375         .disable_event          = intel_generic_uncore_msr_disable_event,
 376         .enable_event           = intel_generic_uncore_msr_enable_event,
 377         .read_counter           = uncore_msr_read_counter,
 378 };
 379
 380 void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box)
 381 {
 382         struct pci_dev *pdev = box->pci_dev;
 383         int box_ctl = uncore_pci_box_ctl(box);
 384
 385         __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
 386         pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT);
 387 }
 388
 389 void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box)
 390 {
 391         struct pci_dev *pdev = box->pci_dev;
 392         int box_ctl = uncore_pci_box_ctl(box);
 393
 394         pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ);
 395 }
 396
 397 void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box)
 398 {
 399         struct pci_dev *pdev = box->pci_dev;
 400         int box_ctl = uncore_pci_box_ctl(box);
 401
 402         pci_write_config_dword(pdev, box_ctl, 0);
 403 }
 404
 405 static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box,
 406                                             struct perf_event *event)
 407 {
 408         struct pci_dev *pdev = box->pci_dev;
 409         struct hw_perf_event *hwc = &event->hw;
 410
 411         pci_write_config_dword(pdev, hwc->config_base, hwc->config);
 412 }
 413
 414 void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box,
 415                                             struct perf_event *event)
 416 {
 417         struct pci_dev *pdev = box->pci_dev;
 418         struct hw_perf_event *hwc = &event->hw;
 419
 420         pci_write_config_dword(pdev, hwc->config_base, 0);
 421 }
 422
 423 u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box,
 424                                           struct perf_event *event)
 425 {
 426         struct pci_dev *pdev = box->pci_dev;
 427         struct hw_perf_event *hwc = &event->hw;
 428         u64 count = 0;
 429
 430         pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
 431         pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
 432
 433         return count;
 434 }
 435
 436 static struct intel_uncore_ops generic_uncore_pci_ops = {
 437         .init_box       = intel_generic_uncore_pci_init_box,
 438         .disable_box    = intel_generic_uncore_pci_disable_box,
 439         .enable_box     = intel_generic_uncore_pci_enable_box,
 440         .disable_event  = intel_generic_uncore_pci_disable_event,
 441         .enable_event   = intel_generic_uncore_pci_enable_event,
 442         .read_counter   = intel_generic_uncore_pci_read_counter,
 443 };
 444
 445 #define UNCORE_GENERIC_MMIO_SIZE                0x4000
 446
 447 static unsigned int generic_uncore_mmio_box_ctl(struct intel_uncore_box *box)
 448 {
 449         struct intel_uncore_type *type = box->pmu->type;
 450
 451         if (!type->box_ctls || !type->box_ctls[box->dieid] || !type->mmio_offsets)
 452                 return 0;
 453
 454         return type->box_ctls[box->dieid] + type->mmio_offsets[box->pmu->pmu_idx];
 455 }
 456
 457 void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box)
 458 {
 459         unsigned int box_ctl = generic_uncore_mmio_box_ctl(box);
 460         struct intel_uncore_type *type = box->pmu->type;
 461         resource_size_t addr;
 462
 463         if (!box_ctl) {
 464                 pr_warn("Uncore type %d box %d: Invalid box control address.\n",
 465                         type->type_id, type->box_ids[box->pmu->pmu_idx]);
 466                 return;
 467         }
 468
 469         addr = box_ctl;
 470         box->io_addr = ioremap(addr, UNCORE_GENERIC_MMIO_SIZE);
 471         if (!box->io_addr) {
 472                 pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n",
 473                         type->type_id, type->box_ids[box->pmu->pmu_idx],
 474                         (unsigned long long)addr);
 475                 return;
 476         }
 477
 478         writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr);
 479 }
 480
 481 void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box)
 482 {
 483         if (!box->io_addr)
 484                 return;
 485
 486         writel(GENERIC_PMON_BOX_CTL_FRZ, box->io_addr);
 487 }
 488
 489 void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box)
 490 {
 491         if (!box->io_addr)
 492                 return;
 493
 494         writel(0, box->io_addr);
 495 }
 496
 497 static void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box,
 498                                              struct perf_event *event)
 499 {
 500         struct hw_perf_event *hwc = &event->hw;
 501
 502         if (!box->io_addr)
 503                 return;
 504
 505         writel(hwc->config, box->io_addr + hwc->config_base);
 506 }
 507
 508 void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box,
 509                                              struct perf_event *event)
 510 {
 511         struct hw_perf_event *hwc = &event->hw;
 512
 513         if (!box->io_addr)
 514                 return;
 515
 516         writel(0, box->io_addr + hwc->config_base);
 517 }
 518
 519 static struct intel_uncore_ops generic_uncore_mmio_ops = {
 520         .init_box       = intel_generic_uncore_mmio_init_box,
 521         .exit_box       = uncore_mmio_exit_box,
 522         .disable_box    = intel_generic_uncore_mmio_disable_box,
 523         .enable_box     = intel_generic_uncore_mmio_enable_box,
 524         .disable_event  = intel_generic_uncore_mmio_disable_event,
 525         .enable_event   = intel_generic_uncore_mmio_enable_event,
 526         .read_counter   = uncore_mmio_read_counter,
 527 };
 528
 529 static bool uncore_update_uncore_type(enum uncore_access_type type_id,
 530                                       struct intel_uncore_type *uncore,
 531                                       struct intel_uncore_discovery_type *type)
 532 {
 533         uncore->type_id = type->type;
 534         uncore->num_boxes = type->num_boxes;
 535         uncore->num_counters = type->num_counters;
 536         uncore->perf_ctr_bits = type->counter_width;
 537         uncore->box_ids = type->ids;
 538
 539         switch (type_id) {
 540         case UNCORE_ACCESS_MSR:
 541                 uncore->ops = &generic_uncore_msr_ops;
 542                 uncore->perf_ctr = (unsigned int)type->box_ctrl + type->ctr_offset;
 543                 uncore->event_ctl = (unsigned int)type->box_ctrl + type->ctl_offset;
 544                 uncore->box_ctl = (unsigned int)type->box_ctrl;
 545                 uncore->msr_offsets = type->box_offset;
 546                 break;
 547         case UNCORE_ACCESS_PCI:
 548                 uncore->ops = &generic_uncore_pci_ops;
 549                 uncore->perf_ctr = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl) + type->ctr_offset;
 550                 uncore->event_ctl = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl) + type->ctl_offset;
 551                 uncore->box_ctl = (unsigned int)UNCORE_DISCOVERY_PCI_BOX_CTRL(type->box_ctrl);
 552                 uncore->box_ctls = type->box_ctrl_die;
 553                 uncore->pci_offsets = type->box_offset;
 554                 break;
 555         case UNCORE_ACCESS_MMIO:
 556                 uncore->ops = &generic_uncore_mmio_ops;
 557                 uncore->perf_ctr = (unsigned int)type->ctr_offset;
 558                 uncore->event_ctl = (unsigned int)type->ctl_offset;
 559                 uncore->box_ctl = (unsigned int)type->box_ctrl;
 560                 uncore->box_ctls = type->box_ctrl_die;
 561                 uncore->mmio_offsets = type->box_offset;
 562                 uncore->mmio_map_size = UNCORE_GENERIC_MMIO_SIZE;
 563                 break;
 564         default:
 565                 return false;
 566         }
 567
 568         return true;
 569 }
 570
 571 struct intel_uncore_type **
 572 intel_uncore_generic_init_uncores(enum uncore_access_type type_id, int num_extra)
 573 {
 574         struct intel_uncore_discovery_type *type;
 575         struct intel_uncore_type **uncores;
 576         struct intel_uncore_type *uncore;
 577         struct rb_node *node;
 578         int i = 0;
 579
 580         uncores = kcalloc(num_discovered_types[type_id] + num_extra + 1,
 581                           sizeof(struct intel_uncore_type *), GFP_KERNEL);
 582         if (!uncores)
 583                 return empty_uncore;
 584
 585         for (node = rb_first(&discovery_tables); node; node = rb_next(node)) {
 586                 type = rb_entry(node, struct intel_uncore_discovery_type, node);
 587                 if (type->access_type != type_id)
 588                         continue;
 589
 590                 uncore = kzalloc(sizeof(struct intel_uncore_type), GFP_KERNEL);
 591                 if (!uncore)
 592                         break;
 593
 594                 uncore->event_mask = GENERIC_PMON_RAW_EVENT_MASK;
 595                 uncore->format_group = &generic_uncore_format_group;
 596
 597                 if (!uncore_update_uncore_type(type_id, uncore, type)) {
 598                         kfree(uncore);
 599                         continue;
 600                 }
 601                 uncores[i++] = uncore;
 602         }
 603
 604         return uncores;
 605 }
 606
 607 void intel_uncore_generic_uncore_cpu_init(void)
 608 {
 609         uncore_msr_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MSR, 0);
 610 }
 611
 612 int intel_uncore_generic_uncore_pci_init(void)
 613 {
 614         uncore_pci_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_PCI, 0);
 615
 616         return 0;
 617 }
 618
 619 void intel_uncore_generic_uncore_mmio_init(void)
 620 {
 621         uncore_mmio_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MMIO, 0);
 622 }