drivers/perf/arm_cspmu/arm_cspmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * ARM CoreSight Architecture PMU driver.
   4  *
   5  * This driver adds support for uncore PMU based on ARM CoreSight Performance
   6  * Monitoring Unit Architecture. The PMU is accessible via MMIO registers and
   7  * like other uncore PMUs, it does not support process specific events and
   8  * cannot be used in sampling mode.
   9  *
  10  * This code is based on other uncore PMUs like ARM DSU PMU. It provides a
  11  * generic implementation to operate the PMU according to CoreSight PMU
  12  * architecture and ACPI ARM PMU table (APMT) documents below:
  13  *   - ARM CoreSight PMU architecture document number: ARM IHI 0091 A.a-00bet0.
  14  *   - APMT document number: ARM DEN0117.
  15  *
  16  * The user should refer to the vendor technical documentation to get details
  17  * about the supported events.
  18  *
  19  * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  20  *
  21  */
  22
  23 #include <linux/acpi.h>
  24 #include <linux/cacheinfo.h>
  25 #include <linux/ctype.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/io-64-nonatomic-lo-hi.h>
  28 #include <linux/module.h>
  29 #include <linux/perf_event.h>
  30 #include <linux/platform_device.h>
  31
  32 #include "arm_cspmu.h"
  33 #include "nvidia_cspmu.h"
  34
  35 #define PMUNAME "arm_cspmu"
  36 #define DRVNAME "arm-cs-arch-pmu"
  37
  38 #define ARM_CSPMU_CPUMASK_ATTR(_name, _config)                  \
  39         ARM_CSPMU_EXT_ATTR(_name, arm_cspmu_cpumask_show,       \
  40                                 (unsigned long)_config)
  41
  42 /*
  43  * CoreSight PMU Arch register offsets.
  44  */
  45 #define PMEVCNTR_LO                                     0x0
  46 #define PMEVCNTR_HI                                     0x4
  47 #define PMEVTYPER                                       0x400
  48 #define PMCCFILTR                                       0x47C
  49 #define PMEVFILTR                                       0xA00
  50 #define PMCNTENSET                                      0xC00
  51 #define PMCNTENCLR                                      0xC20
  52 #define PMINTENSET                                      0xC40
  53 #define PMINTENCLR                                      0xC60
  54 #define PMOVSCLR                                        0xC80
  55 #define PMOVSSET                                        0xCC0
  56 #define PMCFGR                                          0xE00
  57 #define PMCR                                            0xE04
  58 #define PMIIDR                                          0xE08
  59
  60 /* PMCFGR register field */
  61 #define PMCFGR_NCG                                      GENMASK(31, 28)
  62 #define PMCFGR_HDBG                                     BIT(24)
  63 #define PMCFGR_TRO                                      BIT(23)
  64 #define PMCFGR_SS                                       BIT(22)
  65 #define PMCFGR_FZO                                      BIT(21)
  66 #define PMCFGR_MSI                                      BIT(20)
  67 #define PMCFGR_UEN                                      BIT(19)
  68 #define PMCFGR_NA                                       BIT(17)
  69 #define PMCFGR_EX                                       BIT(16)
  70 #define PMCFGR_CCD                                      BIT(15)
  71 #define PMCFGR_CC                                       BIT(14)
  72 #define PMCFGR_SIZE                                     GENMASK(13, 8)
  73 #define PMCFGR_N                                        GENMASK(7, 0)
  74
  75 /* PMCR register field */
  76 #define PMCR_TRO                                        BIT(11)
  77 #define PMCR_HDBG                                       BIT(10)
  78 #define PMCR_FZO                                        BIT(9)
  79 #define PMCR_NA                                         BIT(8)
  80 #define PMCR_DP                                         BIT(5)
  81 #define PMCR_X                                          BIT(4)
  82 #define PMCR_D                                          BIT(3)
  83 #define PMCR_C                                          BIT(2)
  84 #define PMCR_P                                          BIT(1)
  85 #define PMCR_E                                          BIT(0)
  86
  87 /* Each SET/CLR register supports up to 32 counters. */
  88 #define ARM_CSPMU_SET_CLR_COUNTER_SHIFT         5
  89 #define ARM_CSPMU_SET_CLR_COUNTER_NUM           \
  90         (1 << ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
  91
  92 /* Convert counter idx into SET/CLR register number. */
  93 #define COUNTER_TO_SET_CLR_ID(idx)                      \
  94         (idx >> ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
  95
  96 /* Convert counter idx into SET/CLR register bit. */
  97 #define COUNTER_TO_SET_CLR_BIT(idx)                     \
  98         (idx & (ARM_CSPMU_SET_CLR_COUNTER_NUM - 1))
  99
 100 #define ARM_CSPMU_ACTIVE_CPU_MASK               0x0
 101 #define ARM_CSPMU_ASSOCIATED_CPU_MASK           0x1
 102
 103 /* Check and use default if implementer doesn't provide attribute callback */
 104 #define CHECK_DEFAULT_IMPL_OPS(ops, callback)                   \
 105         do {                                                    \
 106                 if (!ops->callback)                             \
 107                         ops->callback = arm_cspmu_ ## callback; \
 108         } while (0)
 109
 110 /*
 111  * Maximum poll count for reading counter value using high-low-high sequence.
 112  */
 113 #define HILOHI_MAX_POLL 1000
 114
 115 /* JEDEC-assigned JEP106 identification code */
 116 #define ARM_CSPMU_IMPL_ID_NVIDIA                0x36B
 117
 118 static unsigned long arm_cspmu_cpuhp_state;
 119
 120 static struct acpi_apmt_node *arm_cspmu_apmt_node(struct device *dev)
 121 {
 122         return *(struct acpi_apmt_node **)dev_get_platdata(dev);
 123 }
 124
 125 /*
 126  * In CoreSight PMU architecture, all of the MMIO registers are 32-bit except
 127  * counter register. The counter register can be implemented as 32-bit or 64-bit
 128  * register depending on the value of PMCFGR.SIZE field. For 64-bit access,
 129  * single-copy 64-bit atomic support is implementation defined. APMT node flag
 130  * is used to identify if the PMU supports 64-bit single copy atomic. If 64-bit
 131  * single copy atomic is not supported, the driver treats the register as a pair
 132  * of 32-bit register.
 133  */
 134
 135 /*
 136  * Read 64-bit register as a pair of 32-bit registers using hi-lo-hi sequence.
 137  */
 138 static u64 read_reg64_hilohi(const void __iomem *addr, u32 max_poll_count)
 139 {
 140         u32 val_lo, val_hi;
 141         u64 val;
 142
 143         /* Use high-low-high sequence to avoid tearing */
 144         do {
 145                 if (max_poll_count-- == 0) {
 146                         pr_err("ARM CSPMU: timeout hi-low-high sequence\n");
 147                         return 0;
 148                 }
 149
 150                 val_hi = readl(addr + 4);
 151                 val_lo = readl(addr);
 152         } while (val_hi != readl(addr + 4));
 153
 154         val = (((u64)val_hi << 32) | val_lo);
 155
 156         return val;
 157 }
 158
 159 /* Check if cycle counter is supported. */
 160 static inline bool supports_cycle_counter(const struct arm_cspmu *cspmu)
 161 {
 162         return (cspmu->pmcfgr & PMCFGR_CC);
 163 }
 164
 165 /* Get counter size, which is (PMCFGR_SIZE + 1). */
 166 static inline u32 counter_size(const struct arm_cspmu *cspmu)
 167 {
 168         return FIELD_GET(PMCFGR_SIZE, cspmu->pmcfgr) + 1;
 169 }
 170
 171 /* Get counter mask. */
 172 static inline u64 counter_mask(const struct arm_cspmu *cspmu)
 173 {
 174         return GENMASK_ULL(counter_size(cspmu) - 1, 0);
 175 }
 176
 177 /* Check if counter is implemented as 64-bit register. */
 178 static inline bool use_64b_counter_reg(const struct arm_cspmu *cspmu)
 179 {
 180         return (counter_size(cspmu) > 32);
 181 }
 182
 183 ssize_t arm_cspmu_sysfs_event_show(struct device *dev,
 184                                 struct device_attribute *attr, char *buf)
 185 {
 186         struct perf_pmu_events_attr *pmu_attr;
 187
 188         pmu_attr = container_of(attr, typeof(*pmu_attr), attr);
 189         return sysfs_emit(buf, "event=0x%llx\n", pmu_attr->id);
 190 }
 191 EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_event_show);
 192
 193 /* Default event list. */
 194 static struct attribute *arm_cspmu_event_attrs[] = {
 195         ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
 196         NULL,
 197 };
 198
 199 static struct attribute **
 200 arm_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
 201 {
 202         struct attribute **attrs;
 203
 204         attrs = devm_kmemdup(cspmu->dev, arm_cspmu_event_attrs,
 205                 sizeof(arm_cspmu_event_attrs), GFP_KERNEL);
 206
 207         return attrs;
 208 }
 209
 210 static umode_t
 211 arm_cspmu_event_attr_is_visible(struct kobject *kobj,
 212                                 struct attribute *attr, int unused)
 213 {
 214         struct device *dev = kobj_to_dev(kobj);
 215         struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
 216         struct perf_pmu_events_attr *eattr;
 217
 218         eattr = container_of(attr, typeof(*eattr), attr.attr);
 219
 220         /* Hide cycle event if not supported */
 221         if (!supports_cycle_counter(cspmu) &&
 222             eattr->id == ARM_CSPMU_EVT_CYCLES_DEFAULT)
 223                 return 0;
 224
 225         return attr->mode;
 226 }
 227
 228 ssize_t arm_cspmu_sysfs_format_show(struct device *dev,
 229                                 struct device_attribute *attr,
 230                                 char *buf)
 231 {
 232         struct dev_ext_attribute *eattr =
 233                 container_of(attr, struct dev_ext_attribute, attr);
 234         return sysfs_emit(buf, "%s\n", (char *)eattr->var);
 235 }
 236 EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_format_show);
 237
 238 static struct attribute *arm_cspmu_format_attrs[] = {
 239         ARM_CSPMU_FORMAT_EVENT_ATTR,
 240         ARM_CSPMU_FORMAT_FILTER_ATTR,
 241         NULL,
 242 };
 243
 244 static struct attribute **
 245 arm_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
 246 {
 247         struct attribute **attrs;
 248
 249         attrs = devm_kmemdup(cspmu->dev, arm_cspmu_format_attrs,
 250                 sizeof(arm_cspmu_format_attrs), GFP_KERNEL);
 251
 252         return attrs;
 253 }
 254
 255 static u32 arm_cspmu_event_type(const struct perf_event *event)
 256 {
 257         return event->attr.config & ARM_CSPMU_EVENT_MASK;
 258 }
 259
 260 static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event)
 261 {
 262         return (event->attr.config == ARM_CSPMU_EVT_CYCLES_DEFAULT);
 263 }
 264
 265 static u32 arm_cspmu_event_filter(const struct perf_event *event)
 266 {
 267         return event->attr.config1 & ARM_CSPMU_FILTER_MASK;
 268 }
 269
 270 static ssize_t arm_cspmu_identifier_show(struct device *dev,
 271                                          struct device_attribute *attr,
 272                                          char *page)
 273 {
 274         struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
 275
 276         return sysfs_emit(page, "%s\n", cspmu->identifier);
 277 }
 278
 279 static struct device_attribute arm_cspmu_identifier_attr =
 280         __ATTR(identifier, 0444, arm_cspmu_identifier_show, NULL);
 281
 282 static struct attribute *arm_cspmu_identifier_attrs[] = {
 283         &arm_cspmu_identifier_attr.attr,
 284         NULL,
 285 };
 286
 287 static struct attribute_group arm_cspmu_identifier_attr_group = {
 288         .attrs = arm_cspmu_identifier_attrs,
 289 };
 290
 291 static const char *arm_cspmu_get_identifier(const struct arm_cspmu *cspmu)
 292 {
 293         const char *identifier =
 294                 devm_kasprintf(cspmu->dev, GFP_KERNEL, "%x",
 295                                cspmu->impl.pmiidr);
 296         return identifier;
 297 }
 298
 299 static const char *arm_cspmu_type_str[ACPI_APMT_NODE_TYPE_COUNT] = {
 300         "mc",
 301         "smmu",
 302         "pcie",
 303         "acpi",
 304         "cache",
 305 };
 306
 307 static const char *arm_cspmu_get_name(const struct arm_cspmu *cspmu)
 308 {
 309         struct device *dev;
 310         struct acpi_apmt_node *apmt_node;
 311         u8 pmu_type;
 312         char *name;
 313         char acpi_hid_string[ACPI_ID_LEN] = { 0 };
 314         static atomic_t pmu_idx[ACPI_APMT_NODE_TYPE_COUNT] = { 0 };
 315
 316         dev = cspmu->dev;
 317         apmt_node = arm_cspmu_apmt_node(dev);
 318         pmu_type = apmt_node->type;
 319
 320         if (pmu_type >= ACPI_APMT_NODE_TYPE_COUNT) {
 321                 dev_err(dev, "unsupported PMU type-%u\n", pmu_type);
 322                 return NULL;
 323         }
 324
 325         if (pmu_type == ACPI_APMT_NODE_TYPE_ACPI) {
 326                 memcpy(acpi_hid_string,
 327                         &apmt_node->inst_primary,
 328                         sizeof(apmt_node->inst_primary));
 329                 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME,
 330                                       arm_cspmu_type_str[pmu_type],
 331                                       acpi_hid_string,
 332                                       apmt_node->inst_secondary);
 333         } else {
 334                 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%d", PMUNAME,
 335                                       arm_cspmu_type_str[pmu_type],
 336                                       atomic_fetch_inc(&pmu_idx[pmu_type]));
 337         }
 338
 339         return name;
 340 }
 341
 342 static ssize_t arm_cspmu_cpumask_show(struct device *dev,
 343                                       struct device_attribute *attr,
 344                                       char *buf)
 345 {
 346         struct pmu *pmu = dev_get_drvdata(dev);
 347         struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
 348         struct dev_ext_attribute *eattr =
 349                 container_of(attr, struct dev_ext_attribute, attr);
 350         unsigned long mask_id = (unsigned long)eattr->var;
 351         const cpumask_t *cpumask;
 352
 353         switch (mask_id) {
 354         case ARM_CSPMU_ACTIVE_CPU_MASK:
 355                 cpumask = &cspmu->active_cpu;
 356                 break;
 357         case ARM_CSPMU_ASSOCIATED_CPU_MASK:
 358                 cpumask = &cspmu->associated_cpus;
 359                 break;
 360         default:
 361                 return 0;
 362         }
 363         return cpumap_print_to_pagebuf(true, buf, cpumask);
 364 }
 365
 366 static struct attribute *arm_cspmu_cpumask_attrs[] = {
 367         ARM_CSPMU_CPUMASK_ATTR(cpumask, ARM_CSPMU_ACTIVE_CPU_MASK),
 368         ARM_CSPMU_CPUMASK_ATTR(associated_cpus, ARM_CSPMU_ASSOCIATED_CPU_MASK),
 369         NULL,
 370 };
 371
 372 static struct attribute_group arm_cspmu_cpumask_attr_group = {
 373         .attrs = arm_cspmu_cpumask_attrs,
 374 };
 375
 376 struct impl_match {
 377         u32 pmiidr;
 378         u32 mask;
 379         int (*impl_init_ops)(struct arm_cspmu *cspmu);
 380 };
 381
 382 static const struct impl_match impl_match[] = {
 383         {
 384           .pmiidr = ARM_CSPMU_IMPL_ID_NVIDIA,
 385           .mask = ARM_CSPMU_PMIIDR_IMPLEMENTER,
 386           .impl_init_ops = nv_cspmu_init_ops
 387         },
 388         {}
 389 };
 390
 391 static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu)
 392 {
 393         int ret;
 394         struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
 395         struct acpi_apmt_node *apmt_node = arm_cspmu_apmt_node(cspmu->dev);
 396         const struct impl_match *match = impl_match;
 397
 398         /*
 399          * Get PMU implementer and product id from APMT node.
 400          * If APMT node doesn't have implementer/product id, try get it
 401          * from PMIIDR.
 402          */
 403         cspmu->impl.pmiidr =
 404                 (apmt_node->impl_id) ? apmt_node->impl_id :
 405                                        readl(cspmu->base0 + PMIIDR);
 406
 407         /* Find implementer specific attribute ops. */
 408         for (; match->pmiidr; match++) {
 409                 const u32 mask = match->mask;
 410
 411                 if ((match->pmiidr & mask) == (cspmu->impl.pmiidr & mask)) {
 412                         ret = match->impl_init_ops(cspmu);
 413                         if (ret)
 414                                 return ret;
 415
 416                         break;
 417                 }
 418         }
 419
 420         /* Use default callbacks if implementer doesn't provide one. */
 421         CHECK_DEFAULT_IMPL_OPS(impl_ops, get_event_attrs);
 422         CHECK_DEFAULT_IMPL_OPS(impl_ops, get_format_attrs);
 423         CHECK_DEFAULT_IMPL_OPS(impl_ops, get_identifier);
 424         CHECK_DEFAULT_IMPL_OPS(impl_ops, get_name);
 425         CHECK_DEFAULT_IMPL_OPS(impl_ops, is_cycle_counter_event);
 426         CHECK_DEFAULT_IMPL_OPS(impl_ops, event_type);
 427         CHECK_DEFAULT_IMPL_OPS(impl_ops, event_filter);
 428         CHECK_DEFAULT_IMPL_OPS(impl_ops, event_attr_is_visible);
 429
 430         return 0;
 431 }
 432
 433 static struct attribute_group *
 434 arm_cspmu_alloc_event_attr_group(struct arm_cspmu *cspmu)
 435 {
 436         struct attribute_group *event_group;
 437         struct device *dev = cspmu->dev;
 438         const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
 439
 440         event_group =
 441                 devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
 442         if (!event_group)
 443                 return NULL;
 444
 445         event_group->name = "events";
 446         event_group->is_visible = impl_ops->event_attr_is_visible;
 447         event_group->attrs = impl_ops->get_event_attrs(cspmu);
 448
 449         if (!event_group->attrs)
 450                 return NULL;
 451
 452         return event_group;
 453 }
 454
 455 static struct attribute_group *
 456 arm_cspmu_alloc_format_attr_group(struct arm_cspmu *cspmu)
 457 {
 458         struct attribute_group *format_group;
 459         struct device *dev = cspmu->dev;
 460
 461         format_group =
 462                 devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
 463         if (!format_group)
 464                 return NULL;
 465
 466         format_group->name = "format";
 467         format_group->attrs = cspmu->impl.ops.get_format_attrs(cspmu);
 468
 469         if (!format_group->attrs)
 470                 return NULL;
 471
 472         return format_group;
 473 }
 474
 475 static struct attribute_group **
 476 arm_cspmu_alloc_attr_group(struct arm_cspmu *cspmu)
 477 {
 478         struct attribute_group **attr_groups = NULL;
 479         struct device *dev = cspmu->dev;
 480         const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
 481         int ret;
 482
 483         ret = arm_cspmu_init_impl_ops(cspmu);
 484         if (ret)
 485                 return NULL;
 486
 487         cspmu->identifier = impl_ops->get_identifier(cspmu);
 488         cspmu->name = impl_ops->get_name(cspmu);
 489
 490         if (!cspmu->identifier || !cspmu->name)
 491                 return NULL;
 492
 493         attr_groups = devm_kcalloc(dev, 5, sizeof(struct attribute_group *),
 494                                    GFP_KERNEL);
 495         if (!attr_groups)
 496                 return NULL;
 497
 498         attr_groups[0] = arm_cspmu_alloc_event_attr_group(cspmu);
 499         attr_groups[1] = arm_cspmu_alloc_format_attr_group(cspmu);
 500         attr_groups[2] = &arm_cspmu_identifier_attr_group;
 501         attr_groups[3] = &arm_cspmu_cpumask_attr_group;
 502
 503         if (!attr_groups[0] || !attr_groups[1])
 504                 return NULL;
 505
 506         return attr_groups;
 507 }
 508
 509 static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu)
 510 {
 511         u32 pmcr = 0;
 512
 513         pmcr |= PMCR_P;
 514         pmcr |= PMCR_C;
 515         writel(pmcr, cspmu->base0 + PMCR);
 516 }
 517
 518 static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu)
 519 {
 520         writel(PMCR_E, cspmu->base0 + PMCR);
 521 }
 522
 523 static inline void arm_cspmu_stop_counters(struct arm_cspmu *cspmu)
 524 {
 525         writel(0, cspmu->base0 + PMCR);
 526 }
 527
 528 static void arm_cspmu_enable(struct pmu *pmu)
 529 {
 530         bool disabled;
 531         struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
 532
 533         disabled = bitmap_empty(cspmu->hw_events.used_ctrs,
 534                                 cspmu->num_logical_ctrs);
 535
 536         if (disabled)
 537                 return;
 538
 539         arm_cspmu_start_counters(cspmu);
 540 }
 541
 542 static void arm_cspmu_disable(struct pmu *pmu)
 543 {
 544         struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
 545
 546         arm_cspmu_stop_counters(cspmu);
 547 }
 548
 549 static int arm_cspmu_get_event_idx(struct arm_cspmu_hw_events *hw_events,
 550                                 struct perf_event *event)
 551 {
 552         int idx;
 553         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 554
 555         if (supports_cycle_counter(cspmu)) {
 556                 if (cspmu->impl.ops.is_cycle_counter_event(event)) {
 557                         /* Search for available cycle counter. */
 558                         if (test_and_set_bit(cspmu->cycle_counter_logical_idx,
 559                                              hw_events->used_ctrs))
 560                                 return -EAGAIN;
 561
 562                         return cspmu->cycle_counter_logical_idx;
 563                 }
 564
 565                 /*
 566                  * Search a regular counter from the used counter bitmap.
 567                  * The cycle counter divides the bitmap into two parts. Search
 568                  * the first then second half to exclude the cycle counter bit.
 569                  */
 570                 idx = find_first_zero_bit(hw_events->used_ctrs,
 571                                           cspmu->cycle_counter_logical_idx);
 572                 if (idx >= cspmu->cycle_counter_logical_idx) {
 573                         idx = find_next_zero_bit(
 574                                 hw_events->used_ctrs,
 575                                 cspmu->num_logical_ctrs,
 576                                 cspmu->cycle_counter_logical_idx + 1);
 577                 }
 578         } else {
 579                 idx = find_first_zero_bit(hw_events->used_ctrs,
 580                                           cspmu->num_logical_ctrs);
 581         }
 582
 583         if (idx >= cspmu->num_logical_ctrs)
 584                 return -EAGAIN;
 585
 586         set_bit(idx, hw_events->used_ctrs);
 587
 588         return idx;
 589 }
 590
 591 static bool arm_cspmu_validate_event(struct pmu *pmu,
 592                                  struct arm_cspmu_hw_events *hw_events,
 593                                  struct perf_event *event)
 594 {
 595         if (is_software_event(event))
 596                 return true;
 597
 598         /* Reject groups spanning multiple HW PMUs. */
 599         if (event->pmu != pmu)
 600                 return false;
 601
 602         return (arm_cspmu_get_event_idx(hw_events, event) >= 0);
 603 }
 604
 605 /*
 606  * Make sure the group of events can be scheduled at once
 607  * on the PMU.
 608  */
 609 static bool arm_cspmu_validate_group(struct perf_event *event)
 610 {
 611         struct perf_event *sibling, *leader = event->group_leader;
 612         struct arm_cspmu_hw_events fake_hw_events;
 613
 614         if (event->group_leader == event)
 615                 return true;
 616
 617         memset(&fake_hw_events, 0, sizeof(fake_hw_events));
 618
 619         if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events, leader))
 620                 return false;
 621
 622         for_each_sibling_event(sibling, leader) {
 623                 if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events,
 624                                                   sibling))
 625                         return false;
 626         }
 627
 628         return arm_cspmu_validate_event(event->pmu, &fake_hw_events, event);
 629 }
 630
 631 static int arm_cspmu_event_init(struct perf_event *event)
 632 {
 633         struct arm_cspmu *cspmu;
 634         struct hw_perf_event *hwc = &event->hw;
 635
 636         cspmu = to_arm_cspmu(event->pmu);
 637
 638         /*
 639          * Following other "uncore" PMUs, we do not support sampling mode or
 640          * attach to a task (per-process mode).
 641          */
 642         if (is_sampling_event(event)) {
 643                 dev_dbg(cspmu->pmu.dev,
 644                         "Can't support sampling events\n");
 645                 return -EOPNOTSUPP;
 646         }
 647
 648         if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
 649                 dev_dbg(cspmu->pmu.dev,
 650                         "Can't support per-task counters\n");
 651                 return -EINVAL;
 652         }
 653
 654         /*
 655          * Make sure the CPU assignment is on one of the CPUs associated with
 656          * this PMU.
 657          */
 658         if (!cpumask_test_cpu(event->cpu, &cspmu->associated_cpus)) {
 659                 dev_dbg(cspmu->pmu.dev,
 660                         "Requested cpu is not associated with the PMU\n");
 661                 return -EINVAL;
 662         }
 663
 664         /* Enforce the current active CPU to handle the events in this PMU. */
 665         event->cpu = cpumask_first(&cspmu->active_cpu);
 666         if (event->cpu >= nr_cpu_ids)
 667                 return -EINVAL;
 668
 669         if (!arm_cspmu_validate_group(event))
 670                 return -EINVAL;
 671
 672         /*
 673          * The logical counter id is tracked with hw_perf_event.extra_reg.idx.
 674          * The physical counter id is tracked with hw_perf_event.idx.
 675          * We don't assign an index until we actually place the event onto
 676          * hardware. Use -1 to signify that we haven't decided where to put it
 677          * yet.
 678          */
 679         hwc->idx = -1;
 680         hwc->extra_reg.idx = -1;
 681         hwc->config = cspmu->impl.ops.event_type(event);
 682
 683         return 0;
 684 }
 685
 686 static inline u32 counter_offset(u32 reg_sz, u32 ctr_idx)
 687 {
 688         return (PMEVCNTR_LO + (reg_sz * ctr_idx));
 689 }
 690
 691 static void arm_cspmu_write_counter(struct perf_event *event, u64 val)
 692 {
 693         u32 offset;
 694         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 695
 696         if (use_64b_counter_reg(cspmu)) {
 697                 offset = counter_offset(sizeof(u64), event->hw.idx);
 698
 699                 writeq(val, cspmu->base1 + offset);
 700         } else {
 701                 offset = counter_offset(sizeof(u32), event->hw.idx);
 702
 703                 writel(lower_32_bits(val), cspmu->base1 + offset);
 704         }
 705 }
 706
 707 static u64 arm_cspmu_read_counter(struct perf_event *event)
 708 {
 709         u32 offset;
 710         const void __iomem *counter_addr;
 711         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 712
 713         if (use_64b_counter_reg(cspmu)) {
 714                 offset = counter_offset(sizeof(u64), event->hw.idx);
 715                 counter_addr = cspmu->base1 + offset;
 716
 717                 return cspmu->has_atomic_dword ?
 718                                readq(counter_addr) :
 719                                read_reg64_hilohi(counter_addr, HILOHI_MAX_POLL);
 720         }
 721
 722         offset = counter_offset(sizeof(u32), event->hw.idx);
 723         return readl(cspmu->base1 + offset);
 724 }
 725
 726 /*
 727  * arm_cspmu_set_event_period: Set the period for the counter.
 728  *
 729  * To handle cases of extreme interrupt latency, we program
 730  * the counter with half of the max count for the counters.
 731  */
 732 static void arm_cspmu_set_event_period(struct perf_event *event)
 733 {
 734         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 735         u64 val = counter_mask(cspmu) >> 1ULL;
 736
 737         local64_set(&event->hw.prev_count, val);
 738         arm_cspmu_write_counter(event, val);
 739 }
 740
 741 static void arm_cspmu_enable_counter(struct arm_cspmu *cspmu, int idx)
 742 {
 743         u32 reg_id, reg_bit, inten_off, cnten_off;
 744
 745         reg_id = COUNTER_TO_SET_CLR_ID(idx);
 746         reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
 747
 748         inten_off = PMINTENSET + (4 * reg_id);
 749         cnten_off = PMCNTENSET + (4 * reg_id);
 750
 751         writel(BIT(reg_bit), cspmu->base0 + inten_off);
 752         writel(BIT(reg_bit), cspmu->base0 + cnten_off);
 753 }
 754
 755 static void arm_cspmu_disable_counter(struct arm_cspmu *cspmu, int idx)
 756 {
 757         u32 reg_id, reg_bit, inten_off, cnten_off;
 758
 759         reg_id = COUNTER_TO_SET_CLR_ID(idx);
 760         reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
 761
 762         inten_off = PMINTENCLR + (4 * reg_id);
 763         cnten_off = PMCNTENCLR + (4 * reg_id);
 764
 765         writel(BIT(reg_bit), cspmu->base0 + cnten_off);
 766         writel(BIT(reg_bit), cspmu->base0 + inten_off);
 767 }
 768
 769 static void arm_cspmu_event_update(struct perf_event *event)
 770 {
 771         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 772         struct hw_perf_event *hwc = &event->hw;
 773         u64 delta, prev, now;
 774
 775         do {
 776                 prev = local64_read(&hwc->prev_count);
 777                 now = arm_cspmu_read_counter(event);
 778         } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
 779
 780         delta = (now - prev) & counter_mask(cspmu);
 781         local64_add(delta, &event->count);
 782 }
 783
 784 static inline void arm_cspmu_set_event(struct arm_cspmu *cspmu,
 785                                         struct hw_perf_event *hwc)
 786 {
 787         u32 offset = PMEVTYPER + (4 * hwc->idx);
 788
 789         writel(hwc->config, cspmu->base0 + offset);
 790 }
 791
 792 static inline void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu,
 793                                            struct hw_perf_event *hwc,
 794                                            u32 filter)
 795 {
 796         u32 offset = PMEVFILTR + (4 * hwc->idx);
 797
 798         writel(filter, cspmu->base0 + offset);
 799 }
 800
 801 static inline void arm_cspmu_set_cc_filter(struct arm_cspmu *cspmu, u32 filter)
 802 {
 803         u32 offset = PMCCFILTR;
 804
 805         writel(filter, cspmu->base0 + offset);
 806 }
 807
 808 static void arm_cspmu_start(struct perf_event *event, int pmu_flags)
 809 {
 810         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 811         struct hw_perf_event *hwc = &event->hw;
 812         u32 filter;
 813
 814         /* We always reprogram the counter */
 815         if (pmu_flags & PERF_EF_RELOAD)
 816                 WARN_ON(!(hwc->state & PERF_HES_UPTODATE));
 817
 818         arm_cspmu_set_event_period(event);
 819
 820         filter = cspmu->impl.ops.event_filter(event);
 821
 822         if (event->hw.extra_reg.idx == cspmu->cycle_counter_logical_idx) {
 823                 arm_cspmu_set_cc_filter(cspmu, filter);
 824         } else {
 825                 arm_cspmu_set_event(cspmu, hwc);
 826                 arm_cspmu_set_ev_filter(cspmu, hwc, filter);
 827         }
 828
 829         hwc->state = 0;
 830
 831         arm_cspmu_enable_counter(cspmu, hwc->idx);
 832 }
 833
 834 static void arm_cspmu_stop(struct perf_event *event, int pmu_flags)
 835 {
 836         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 837         struct hw_perf_event *hwc = &event->hw;
 838
 839         if (hwc->state & PERF_HES_STOPPED)
 840                 return;
 841
 842         arm_cspmu_disable_counter(cspmu, hwc->idx);
 843         arm_cspmu_event_update(event);
 844
 845         hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 846 }
 847
 848 static inline u32 to_phys_idx(struct arm_cspmu *cspmu, u32 idx)
 849 {
 850         return (idx == cspmu->cycle_counter_logical_idx) ?
 851                 ARM_CSPMU_CYCLE_CNTR_IDX : idx;
 852 }
 853
 854 static int arm_cspmu_add(struct perf_event *event, int flags)
 855 {
 856         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 857         struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
 858         struct hw_perf_event *hwc = &event->hw;
 859         int idx;
 860
 861         if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
 862                                            &cspmu->associated_cpus)))
 863                 return -ENOENT;
 864
 865         idx = arm_cspmu_get_event_idx(hw_events, event);
 866         if (idx < 0)
 867                 return idx;
 868
 869         hw_events->events[idx] = event;
 870         hwc->idx = to_phys_idx(cspmu, idx);
 871         hwc->extra_reg.idx = idx;
 872         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 873
 874         if (flags & PERF_EF_START)
 875                 arm_cspmu_start(event, PERF_EF_RELOAD);
 876
 877         /* Propagate changes to the userspace mapping. */
 878         perf_event_update_userpage(event);
 879
 880         return 0;
 881 }
 882
 883 static void arm_cspmu_del(struct perf_event *event, int flags)
 884 {
 885         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 886         struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
 887         struct hw_perf_event *hwc = &event->hw;
 888         int idx = hwc->extra_reg.idx;
 889
 890         arm_cspmu_stop(event, PERF_EF_UPDATE);
 891
 892         hw_events->events[idx] = NULL;
 893
 894         clear_bit(idx, hw_events->used_ctrs);
 895
 896         perf_event_update_userpage(event);
 897 }
 898
 899 static void arm_cspmu_read(struct perf_event *event)
 900 {
 901         arm_cspmu_event_update(event);
 902 }
 903
 904 static struct arm_cspmu *arm_cspmu_alloc(struct platform_device *pdev)
 905 {
 906         struct acpi_apmt_node *apmt_node;
 907         struct arm_cspmu *cspmu;
 908         struct device *dev = &pdev->dev;
 909
 910         cspmu = devm_kzalloc(dev, sizeof(*cspmu), GFP_KERNEL);
 911         if (!cspmu)
 912                 return NULL;
 913
 914         cspmu->dev = dev;
 915         platform_set_drvdata(pdev, cspmu);
 916
 917         apmt_node = arm_cspmu_apmt_node(dev);
 918         cspmu->has_atomic_dword = apmt_node->flags & ACPI_APMT_FLAGS_ATOMIC;
 919
 920         return cspmu;
 921 }
 922
 923 static int arm_cspmu_init_mmio(struct arm_cspmu *cspmu)
 924 {
 925         struct device *dev;
 926         struct platform_device *pdev;
 927
 928         dev = cspmu->dev;
 929         pdev = to_platform_device(dev);
 930
 931         /* Base address for page 0. */
 932         cspmu->base0 = devm_platform_ioremap_resource(pdev, 0);
 933         if (IS_ERR(cspmu->base0)) {
 934                 dev_err(dev, "ioremap failed for page-0 resource\n");
 935                 return PTR_ERR(cspmu->base0);
 936         }
 937
 938         /* Base address for page 1 if supported. Otherwise point to page 0. */
 939         cspmu->base1 = cspmu->base0;
 940         if (platform_get_resource(pdev, IORESOURCE_MEM, 1)) {
 941                 cspmu->base1 = devm_platform_ioremap_resource(pdev, 1);
 942                 if (IS_ERR(cspmu->base1)) {
 943                         dev_err(dev, "ioremap failed for page-1 resource\n");
 944                         return PTR_ERR(cspmu->base1);
 945                 }
 946         }
 947
 948         cspmu->pmcfgr = readl(cspmu->base0 + PMCFGR);
 949
 950         cspmu->num_logical_ctrs = FIELD_GET(PMCFGR_N, cspmu->pmcfgr) + 1;
 951
 952         cspmu->cycle_counter_logical_idx = ARM_CSPMU_MAX_HW_CNTRS;
 953
 954         if (supports_cycle_counter(cspmu)) {
 955                 /*
 956                  * The last logical counter is mapped to cycle counter if
 957                  * there is a gap between regular and cycle counter. Otherwise,
 958                  * logical and physical have 1-to-1 mapping.
 959                  */
 960                 cspmu->cycle_counter_logical_idx =
 961                         (cspmu->num_logical_ctrs <= ARM_CSPMU_CYCLE_CNTR_IDX) ?
 962                                 cspmu->num_logical_ctrs - 1 :
 963                                 ARM_CSPMU_CYCLE_CNTR_IDX;
 964         }
 965
 966         cspmu->num_set_clr_reg =
 967                 DIV_ROUND_UP(cspmu->num_logical_ctrs,
 968                                 ARM_CSPMU_SET_CLR_COUNTER_NUM);
 969
 970         cspmu->hw_events.events =
 971                 devm_kcalloc(dev, cspmu->num_logical_ctrs,
 972                              sizeof(*cspmu->hw_events.events), GFP_KERNEL);
 973
 974         if (!cspmu->hw_events.events)
 975                 return -ENOMEM;
 976
 977         return 0;
 978 }
 979
 980 static inline int arm_cspmu_get_reset_overflow(struct arm_cspmu *cspmu,
 981                                                u32 *pmovs)
 982 {
 983         int i;
 984         u32 pmovclr_offset = PMOVSCLR;
 985         u32 has_overflowed = 0;
 986
 987         for (i = 0; i < cspmu->num_set_clr_reg; ++i) {
 988                 pmovs[i] = readl(cspmu->base1 + pmovclr_offset);
 989                 has_overflowed |= pmovs[i];
 990                 writel(pmovs[i], cspmu->base1 + pmovclr_offset);
 991                 pmovclr_offset += sizeof(u32);
 992         }
 993
 994         return has_overflowed != 0;
 995 }
 996
 997 static irqreturn_t arm_cspmu_handle_irq(int irq_num, void *dev)
 998 {
 999         int idx, has_overflowed;
1000         struct perf_event *event;
1001         struct arm_cspmu *cspmu = dev;
1002         DECLARE_BITMAP(pmovs, ARM_CSPMU_MAX_HW_CNTRS);
1003         bool handled = false;
1004
1005         arm_cspmu_stop_counters(cspmu);
1006
1007         has_overflowed = arm_cspmu_get_reset_overflow(cspmu, (u32 *)pmovs);
1008         if (!has_overflowed)
1009                 goto done;
1010
1011         for_each_set_bit(idx, cspmu->hw_events.used_ctrs,
1012                         cspmu->num_logical_ctrs) {
1013                 event = cspmu->hw_events.events[idx];
1014
1015                 if (!event)
1016                         continue;
1017
1018                 if (!test_bit(event->hw.idx, pmovs))
1019                         continue;
1020
1021                 arm_cspmu_event_update(event);
1022                 arm_cspmu_set_event_period(event);
1023
1024                 handled = true;
1025         }
1026
1027 done:
1028         arm_cspmu_start_counters(cspmu);
1029         return IRQ_RETVAL(handled);
1030 }
1031
1032 static int arm_cspmu_request_irq(struct arm_cspmu *cspmu)
1033 {
1034         int irq, ret;
1035         struct device *dev;
1036         struct platform_device *pdev;
1037
1038         dev = cspmu->dev;
1039         pdev = to_platform_device(dev);
1040
1041         /* Skip IRQ request if the PMU does not support overflow interrupt. */
1042         irq = platform_get_irq_optional(pdev, 0);
1043         if (irq < 0)
1044                 return irq == -ENXIO ? 0 : irq;
1045
1046         ret = devm_request_irq(dev, irq, arm_cspmu_handle_irq,
1047                                IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(dev),
1048                                cspmu);
1049         if (ret) {
1050                 dev_err(dev, "Could not request IRQ %d\n", irq);
1051                 return ret;
1052         }
1053
1054         cspmu->irq = irq;
1055
1056         return 0;
1057 }
1058
1059 #if defined(CONFIG_ACPI) && defined(CONFIG_ARM64)
1060 #include <acpi/processor.h>
1061
1062 static inline int arm_cspmu_find_cpu_container(int cpu, u32 container_uid)
1063 {
1064         u32 acpi_uid;
1065         struct device *cpu_dev;
1066         struct acpi_device *acpi_dev;
1067
1068         cpu_dev = get_cpu_device(cpu);
1069         if (!cpu_dev)
1070                 return -ENODEV;
1071
1072         acpi_dev = ACPI_COMPANION(cpu_dev);
1073         while (acpi_dev) {
1074                 if (!strcmp(acpi_device_hid(acpi_dev),
1075                             ACPI_PROCESSOR_CONTAINER_HID) &&
1076                     !kstrtouint(acpi_device_uid(acpi_dev), 0, &acpi_uid) &&
1077                     acpi_uid == container_uid)
1078                         return 0;
1079
1080                 acpi_dev = acpi_dev_parent(acpi_dev);
1081         }
1082
1083         return -ENODEV;
1084 }
1085
1086 static int arm_cspmu_acpi_get_cpus(struct arm_cspmu *cspmu)
1087 {
1088         struct acpi_apmt_node *apmt_node;
1089         int affinity_flag;
1090         int cpu;
1091
1092         apmt_node = arm_cspmu_apmt_node(cspmu->dev);
1093         affinity_flag = apmt_node->flags & ACPI_APMT_FLAGS_AFFINITY;
1094
1095         if (affinity_flag == ACPI_APMT_FLAGS_AFFINITY_PROC) {
1096                 for_each_possible_cpu(cpu) {
1097                         if (apmt_node->proc_affinity ==
1098                             get_acpi_id_for_cpu(cpu)) {
1099                                 cpumask_set_cpu(cpu, &cspmu->associated_cpus);
1100                                 break;
1101                         }
1102                 }
1103         } else {
1104                 for_each_possible_cpu(cpu) {
1105                         if (arm_cspmu_find_cpu_container(
1106                                     cpu, apmt_node->proc_affinity))
1107                                 continue;
1108
1109                         cpumask_set_cpu(cpu, &cspmu->associated_cpus);
1110                 }
1111         }
1112
1113         if (cpumask_empty(&cspmu->associated_cpus)) {
1114                 dev_dbg(cspmu->dev, "No cpu associated with the PMU\n");
1115                 return -ENODEV;
1116         }
1117
1118         return 0;
1119 }
1120 #else
1121 static int arm_cspmu_acpi_get_cpus(struct arm_cspmu *cspmu)
1122 {
1123         return -ENODEV;
1124 }
1125 #endif
1126
1127 static int arm_cspmu_get_cpus(struct arm_cspmu *cspmu)
1128 {
1129         return arm_cspmu_acpi_get_cpus(cspmu);
1130 }
1131
1132 static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu)
1133 {
1134         int ret, capabilities;
1135         struct attribute_group **attr_groups;
1136
1137         attr_groups = arm_cspmu_alloc_attr_group(cspmu);
1138         if (!attr_groups)
1139                 return -ENOMEM;
1140
1141         ret = cpuhp_state_add_instance(arm_cspmu_cpuhp_state,
1142                                        &cspmu->cpuhp_node);
1143         if (ret)
1144                 return ret;
1145
1146         capabilities = PERF_PMU_CAP_NO_EXCLUDE;
1147         if (cspmu->irq == 0)
1148                 capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1149
1150         cspmu->pmu = (struct pmu){
1151                 .task_ctx_nr    = perf_invalid_context,
1152                 .module         = THIS_MODULE,
1153                 .pmu_enable     = arm_cspmu_enable,
1154                 .pmu_disable    = arm_cspmu_disable,
1155                 .event_init     = arm_cspmu_event_init,
1156                 .add            = arm_cspmu_add,
1157                 .del            = arm_cspmu_del,
1158                 .start          = arm_cspmu_start,
1159                 .stop           = arm_cspmu_stop,
1160                 .read           = arm_cspmu_read,
1161                 .attr_groups    = (const struct attribute_group **)attr_groups,
1162                 .capabilities   = capabilities,
1163         };
1164
1165         /* Hardware counter init */
1166         arm_cspmu_stop_counters(cspmu);
1167         arm_cspmu_reset_counters(cspmu);
1168
1169         ret = perf_pmu_register(&cspmu->pmu, cspmu->name, -1);
1170         if (ret) {
1171                 cpuhp_state_remove_instance(arm_cspmu_cpuhp_state,
1172                                             &cspmu->cpuhp_node);
1173         }
1174
1175         return ret;
1176 }
1177
1178 static int arm_cspmu_device_probe(struct platform_device *pdev)
1179 {
1180         int ret;
1181         struct arm_cspmu *cspmu;
1182
1183         cspmu = arm_cspmu_alloc(pdev);
1184         if (!cspmu)
1185                 return -ENOMEM;
1186
1187         ret = arm_cspmu_init_mmio(cspmu);
1188         if (ret)
1189                 return ret;
1190
1191         ret = arm_cspmu_request_irq(cspmu);
1192         if (ret)
1193                 return ret;
1194
1195         ret = arm_cspmu_get_cpus(cspmu);
1196         if (ret)
1197                 return ret;
1198
1199         ret = arm_cspmu_register_pmu(cspmu);
1200         if (ret)
1201                 return ret;
1202
1203         return 0;
1204 }
1205
1206 static int arm_cspmu_device_remove(struct platform_device *pdev)
1207 {
1208         struct arm_cspmu *cspmu = platform_get_drvdata(pdev);
1209
1210         perf_pmu_unregister(&cspmu->pmu);
1211         cpuhp_state_remove_instance(arm_cspmu_cpuhp_state, &cspmu->cpuhp_node);
1212
1213         return 0;
1214 }
1215
1216 static const struct platform_device_id arm_cspmu_id[] = {
1217         {DRVNAME, 0},
1218         { },
1219 };
1220 MODULE_DEVICE_TABLE(platform, arm_cspmu_id);
1221
1222 static struct platform_driver arm_cspmu_driver = {
1223         .driver = {
1224                         .name = DRVNAME,
1225                         .suppress_bind_attrs = true,
1226                 },
1227         .probe = arm_cspmu_device_probe,
1228         .remove = arm_cspmu_device_remove,
1229         .id_table = arm_cspmu_id,
1230 };
1231
1232 static void arm_cspmu_set_active_cpu(int cpu, struct arm_cspmu *cspmu)
1233 {
1234         cpumask_set_cpu(cpu, &cspmu->active_cpu);
1235         if (cspmu->irq)
1236                 WARN_ON(irq_set_affinity(cspmu->irq, &cspmu->active_cpu));
1237 }
1238
1239 static int arm_cspmu_cpu_online(unsigned int cpu, struct hlist_node *node)
1240 {
1241         struct arm_cspmu *cspmu =
1242                 hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
1243
1244         if (!cpumask_test_cpu(cpu, &cspmu->associated_cpus))
1245                 return 0;
1246
1247         /* If the PMU is already managed, there is nothing to do */
1248         if (!cpumask_empty(&cspmu->active_cpu))
1249                 return 0;
1250
1251         /* Use this CPU for event counting */
1252         arm_cspmu_set_active_cpu(cpu, cspmu);
1253
1254         return 0;
1255 }
1256
1257 static int arm_cspmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1258 {
1259         int dst;
1260         struct cpumask online_supported;
1261
1262         struct arm_cspmu *cspmu =
1263                 hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
1264
1265         /* Nothing to do if this CPU doesn't own the PMU */
1266         if (!cpumask_test_and_clear_cpu(cpu, &cspmu->active_cpu))
1267                 return 0;
1268
1269         /* Choose a new CPU to migrate ownership of the PMU to */
1270         cpumask_and(&online_supported, &cspmu->associated_cpus,
1271                     cpu_online_mask);
1272         dst = cpumask_any_but(&online_supported, cpu);
1273         if (dst >= nr_cpu_ids)
1274                 return 0;
1275
1276         /* Use this CPU for event counting */
1277         perf_pmu_migrate_context(&cspmu->pmu, cpu, dst);
1278         arm_cspmu_set_active_cpu(dst, cspmu);
1279
1280         return 0;
1281 }
1282
1283 static int __init arm_cspmu_init(void)
1284 {
1285         int ret;
1286
1287         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1288                                         "perf/arm/cspmu:online",
1289                                         arm_cspmu_cpu_online,
1290                                         arm_cspmu_cpu_teardown);
1291         if (ret < 0)
1292                 return ret;
1293         arm_cspmu_cpuhp_state = ret;
1294         return platform_driver_register(&arm_cspmu_driver);
1295 }
1296
1297 static void __exit arm_cspmu_exit(void)
1298 {
1299         platform_driver_unregister(&arm_cspmu_driver);
1300         cpuhp_remove_multi_state(arm_cspmu_cpuhp_state);
1301 }
1302
1303 module_init(arm_cspmu_init);
1304 module_exit(arm_cspmu_exit);
1305
1306 MODULE_LICENSE("GPL v2");