drivers/perf/arm-cmn.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2016-2020 Arm Limited
   3 // CMN-600 Coherent Mesh Network PMU driver
   4
   5 #include <linux/acpi.h>
   6 #include <linux/bitfield.h>
   7 #include <linux/bitops.h>
   8 #include <linux/interrupt.h>
   9 #include <linux/io.h>
  10 #include <linux/kernel.h>
  11 #include <linux/list.h>
  12 #include <linux/module.h>
  13 #include <linux/of.h>
  14 #include <linux/perf_event.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/slab.h>
  17 #include <linux/sort.h>
  18
  19 /* Common register stuff */
  20 #define CMN_NODE_INFO                   0x0000
  21 #define CMN_NI_NODE_TYPE                GENMASK_ULL(15, 0)
  22 #define CMN_NI_NODE_ID                  GENMASK_ULL(31, 16)
  23 #define CMN_NI_LOGICAL_ID               GENMASK_ULL(47, 32)
  24
  25 #define CMN_NODEID_DEVID(reg)           ((reg) & 3)
  26 #define CMN_NODEID_PID(reg)             (((reg) >> 2) & 1)
  27 #define CMN_NODEID_X(reg, bits)         ((reg) >> (3 + (bits)))
  28 #define CMN_NODEID_Y(reg, bits)         (((reg) >> 3) & ((1U << (bits)) - 1))
  29
  30 #define CMN_CHILD_INFO                  0x0080
  31 #define CMN_CI_CHILD_COUNT              GENMASK_ULL(15, 0)
  32 #define CMN_CI_CHILD_PTR_OFFSET         GENMASK_ULL(31, 16)
  33
  34 #define CMN_CHILD_NODE_ADDR             GENMASK(27,0)
  35 #define CMN_CHILD_NODE_EXTERNAL         BIT(31)
  36
  37 #define CMN_ADDR_NODE_PTR               GENMASK(27, 14)
  38
  39 #define CMN_NODE_PTR_DEVID(ptr)         (((ptr) >> 2) & 3)
  40 #define CMN_NODE_PTR_PID(ptr)           ((ptr) & 1)
  41 #define CMN_NODE_PTR_X(ptr, bits)       ((ptr) >> (6 + (bits)))
  42 #define CMN_NODE_PTR_Y(ptr, bits)       (((ptr) >> 6) & ((1U << (bits)) - 1))
  43
  44 #define CMN_MAX_XPS                     (8 * 8)
  45
  46 /* The CFG node has one other useful purpose */
  47 #define CMN_CFGM_PERIPH_ID_2            0x0010
  48 #define CMN_CFGM_PID2_REVISION          GENMASK(7, 4)
  49
  50 /* PMU registers occupy the 3rd 4KB page of each node's 16KB space */
  51 #define CMN_PMU_OFFSET                  0x2000
  52
  53 /* For most nodes, this is all there is */
  54 #define CMN_PMU_EVENT_SEL               0x000
  55 #define CMN_PMU_EVENTn_ID_SHIFT(n)      ((n) * 8)
  56
  57 /* DTMs live in the PMU space of XP registers */
  58 #define CMN_DTM_WPn(n)                  (0x1A0 + (n) * 0x18)
  59 #define CMN_DTM_WPn_CONFIG(n)           (CMN_DTM_WPn(n) + 0x00)
  60 #define CMN_DTM_WPn_CONFIG_WP_COMBINE   BIT(6)
  61 #define CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE BIT(5)
  62 #define CMN_DTM_WPn_CONFIG_WP_GRP       BIT(4)
  63 #define CMN_DTM_WPn_CONFIG_WP_CHN_SEL   GENMASK_ULL(3, 1)
  64 #define CMN_DTM_WPn_CONFIG_WP_DEV_SEL   BIT(0)
  65 #define CMN_DTM_WPn_VAL(n)              (CMN_DTM_WPn(n) + 0x08)
  66 #define CMN_DTM_WPn_MASK(n)             (CMN_DTM_WPn(n) + 0x10)
  67
  68 #define CMN_DTM_PMU_CONFIG              0x210
  69 #define CMN__PMEVCNT0_INPUT_SEL         GENMASK_ULL(37, 32)
  70 #define CMN__PMEVCNT0_INPUT_SEL_WP      0x00
  71 #define CMN__PMEVCNT0_INPUT_SEL_XP      0x04
  72 #define CMN__PMEVCNT0_INPUT_SEL_DEV     0x10
  73 #define CMN__PMEVCNT0_GLOBAL_NUM        GENMASK_ULL(18, 16)
  74 #define CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(n)       ((n) * 4)
  75 #define CMN__PMEVCNT_PAIRED(n)          BIT(4 + (n))
  76 #define CMN__PMEVCNT23_COMBINED         BIT(2)
  77 #define CMN__PMEVCNT01_COMBINED         BIT(1)
  78 #define CMN_DTM_PMU_CONFIG_PMU_EN       BIT(0)
  79
  80 #define CMN_DTM_PMEVCNT                 0x220
  81
  82 #define CMN_DTM_PMEVCNTSR               0x240
  83
  84 #define CMN_DTM_NUM_COUNTERS            4
  85
  86 /* The DTC node is where the magic happens */
  87 #define CMN_DT_DTC_CTL                  0x0a00
  88 #define CMN_DT_DTC_CTL_DT_EN            BIT(0)
  89
  90 /* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */
  91 #define _CMN_DT_CNT_REG(n)              ((((n) / 2) * 4 + (n) % 2) * 4)
  92 #define CMN_DT_PMEVCNT(n)               (CMN_PMU_OFFSET + _CMN_DT_CNT_REG(n))
  93 #define CMN_DT_PMCCNTR                  (CMN_PMU_OFFSET + 0x40)
  94
  95 #define CMN_DT_PMEVCNTSR(n)             (CMN_PMU_OFFSET + 0x50 + _CMN_DT_CNT_REG(n))
  96 #define CMN_DT_PMCCNTRSR                (CMN_PMU_OFFSET + 0x90)
  97
  98 #define CMN_DT_PMCR                     (CMN_PMU_OFFSET + 0x100)
  99 #define CMN_DT_PMCR_PMU_EN              BIT(0)
 100 #define CMN_DT_PMCR_CNTR_RST            BIT(5)
 101 #define CMN_DT_PMCR_OVFL_INTR_EN        BIT(6)
 102
 103 #define CMN_DT_PMOVSR                   (CMN_PMU_OFFSET + 0x118)
 104 #define CMN_DT_PMOVSR_CLR               (CMN_PMU_OFFSET + 0x120)
 105
 106 #define CMN_DT_PMSSR                    (CMN_PMU_OFFSET + 0x128)
 107 #define CMN_DT_PMSSR_SS_STATUS(n)       BIT(n)
 108
 109 #define CMN_DT_PMSRR                    (CMN_PMU_OFFSET + 0x130)
 110 #define CMN_DT_PMSRR_SS_REQ             BIT(0)
 111
 112 #define CMN_DT_NUM_COUNTERS             8
 113 #define CMN_MAX_DTCS                    4
 114
 115 /*
 116  * Even in the worst case a DTC counter can't wrap in fewer than 2^42 cycles,
 117  * so throwing away one bit to make overflow handling easy is no big deal.
 118  */
 119 #define CMN_COUNTER_INIT                0x80000000
 120 /* Similarly for the 40-bit cycle counter */
 121 #define CMN_CC_INIT                     0x8000000000ULL
 122
 123
 124 /* Event attributes */
 125 #define CMN_CONFIG_TYPE                 GENMASK(15, 0)
 126 #define CMN_CONFIG_EVENTID              GENMASK(23, 16)
 127 #define CMN_CONFIG_OCCUPID              GENMASK(27, 24)
 128 #define CMN_CONFIG_BYNODEID             BIT(31)
 129 #define CMN_CONFIG_NODEID               GENMASK(47, 32)
 130
 131 #define CMN_EVENT_TYPE(event)           FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config)
 132 #define CMN_EVENT_EVENTID(event)        FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config)
 133 #define CMN_EVENT_OCCUPID(event)        FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config)
 134 #define CMN_EVENT_BYNODEID(event)       FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config)
 135 #define CMN_EVENT_NODEID(event)         FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config)
 136
 137 #define CMN_CONFIG_WP_COMBINE           GENMASK(27, 24)
 138 #define CMN_CONFIG_WP_DEV_SEL           BIT(48)
 139 #define CMN_CONFIG_WP_CHN_SEL           GENMASK(50, 49)
 140 #define CMN_CONFIG_WP_GRP               BIT(52)
 141 #define CMN_CONFIG_WP_EXCLUSIVE         BIT(53)
 142 #define CMN_CONFIG1_WP_VAL              GENMASK(63, 0)
 143 #define CMN_CONFIG2_WP_MASK             GENMASK(63, 0)
 144
 145 #define CMN_EVENT_WP_COMBINE(event)     FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config)
 146 #define CMN_EVENT_WP_DEV_SEL(event)     FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config)
 147 #define CMN_EVENT_WP_CHN_SEL(event)     FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config)
 148 #define CMN_EVENT_WP_GRP(event)         FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config)
 149 #define CMN_EVENT_WP_EXCLUSIVE(event)   FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config)
 150 #define CMN_EVENT_WP_VAL(event)         FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1)
 151 #define CMN_EVENT_WP_MASK(event)        FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2)
 152
 153 /* Made-up event IDs for watchpoint direction */
 154 #define CMN_WP_UP                       0
 155 #define CMN_WP_DOWN                     2
 156
 157
 158 /* r0px probably don't exist in silicon, thankfully */
 159 enum cmn_revision {
 160         CMN600_R1P0,
 161         CMN600_R1P1,
 162         CMN600_R1P2,
 163         CMN600_R1P3,
 164         CMN600_R2P0,
 165         CMN600_R3P0,
 166 };
 167
 168 enum cmn_node_type {
 169         CMN_TYPE_INVALID,
 170         CMN_TYPE_DVM,
 171         CMN_TYPE_CFG,
 172         CMN_TYPE_DTC,
 173         CMN_TYPE_HNI,
 174         CMN_TYPE_HNF,
 175         CMN_TYPE_XP,
 176         CMN_TYPE_SBSX,
 177         CMN_TYPE_RNI = 0xa,
 178         CMN_TYPE_RND = 0xd,
 179         CMN_TYPE_RNSAM = 0xf,
 180         CMN_TYPE_CXRA = 0x100,
 181         CMN_TYPE_CXHA = 0x101,
 182         CMN_TYPE_CXLA = 0x102,
 183         /* Not a real node type */
 184         CMN_TYPE_WP = 0x7770
 185 };
 186
 187 struct arm_cmn_node {
 188         void __iomem *pmu_base;
 189         u16 id, logid;
 190         enum cmn_node_type type;
 191
 192         union {
 193                 /* Device node */
 194                 struct {
 195                         int to_xp;
 196                         /* DN/HN-F/CXHA */
 197                         unsigned int occupid_val;
 198                         unsigned int occupid_count;
 199                 };
 200                 /* XP */
 201                 struct {
 202                         int dtc;
 203                         u32 pmu_config_low;
 204                         union {
 205                                 u8 input_sel[4];
 206                                 __le32 pmu_config_high;
 207                         };
 208                         s8 wp_event[4];
 209                 };
 210         };
 211
 212         union {
 213                 u8 event[4];
 214                 __le32 event_sel;
 215         };
 216 };
 217
 218 struct arm_cmn_dtc {
 219         void __iomem *base;
 220         int irq;
 221         int irq_friend;
 222         bool cc_active;
 223
 224         struct perf_event *counters[CMN_DT_NUM_COUNTERS];
 225         struct perf_event *cycles;
 226 };
 227
 228 #define CMN_STATE_DISABLED      BIT(0)
 229 #define CMN_STATE_TXN           BIT(1)
 230
 231 struct arm_cmn {
 232         struct device *dev;
 233         void __iomem *base;
 234
 235         enum cmn_revision rev;
 236         u8 mesh_x;
 237         u8 mesh_y;
 238         u16 num_xps;
 239         u16 num_dns;
 240         struct arm_cmn_node *xps;
 241         struct arm_cmn_node *dns;
 242
 243         struct arm_cmn_dtc *dtc;
 244         unsigned int num_dtcs;
 245
 246         int cpu;
 247         struct hlist_node cpuhp_node;
 248
 249         unsigned int state;
 250         struct pmu pmu;
 251 };
 252
 253 #define to_cmn(p)       container_of(p, struct arm_cmn, pmu)
 254
 255 static int arm_cmn_hp_state;
 256
 257 struct arm_cmn_hw_event {
 258         struct arm_cmn_node *dn;
 259         u64 dtm_idx[2];
 260         unsigned int dtc_idx;
 261         u8 dtcs_used;
 262         u8 num_dns;
 263 };
 264
 265 #define for_each_hw_dn(hw, dn, i) \
 266         for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++)
 267
 268 static struct arm_cmn_hw_event *to_cmn_hw(struct perf_event *event)
 269 {
 270         BUILD_BUG_ON(sizeof(struct arm_cmn_hw_event) > offsetof(struct hw_perf_event, target));
 271         return (struct arm_cmn_hw_event *)&event->hw;
 272 }
 273
 274 static void arm_cmn_set_index(u64 x[], unsigned int pos, unsigned int val)
 275 {
 276         x[pos / 32] |= (u64)val << ((pos % 32) * 2);
 277 }
 278
 279 static unsigned int arm_cmn_get_index(u64 x[], unsigned int pos)
 280 {
 281         return (x[pos / 32] >> ((pos % 32) * 2)) & 3;
 282 }
 283
 284 struct arm_cmn_event_attr {
 285         struct device_attribute attr;
 286         enum cmn_node_type type;
 287         u8 eventid;
 288         u8 occupid;
 289 };
 290
 291 struct arm_cmn_format_attr {
 292         struct device_attribute attr;
 293         u64 field;
 294         int config;
 295 };
 296
 297 static int arm_cmn_xyidbits(const struct arm_cmn *cmn)
 298 {
 299         return cmn->mesh_x > 4 || cmn->mesh_y > 4 ? 3 : 2;
 300 }
 301
 302 static void arm_cmn_init_node_to_xp(const struct arm_cmn *cmn,
 303                                     struct arm_cmn_node *dn)
 304 {
 305         int bits = arm_cmn_xyidbits(cmn);
 306         int x = CMN_NODEID_X(dn->id, bits);
 307         int y = CMN_NODEID_Y(dn->id, bits);
 308         int xp_idx = cmn->mesh_x * y + x;
 309
 310         dn->to_xp = (cmn->xps + xp_idx) - dn;
 311 }
 312
 313 static struct arm_cmn_node *arm_cmn_node_to_xp(struct arm_cmn_node *dn)
 314 {
 315         return dn->type == CMN_TYPE_XP ? dn : dn + dn->to_xp;
 316 }
 317
 318 static struct arm_cmn_node *arm_cmn_node(const struct arm_cmn *cmn,
 319                                          enum cmn_node_type type)
 320 {
 321         int i;
 322
 323         for (i = 0; i < cmn->num_dns; i++)
 324                 if (cmn->dns[i].type == type)
 325                         return &cmn->dns[i];
 326         return NULL;
 327 }
 328
 329 #define CMN_EVENT_ATTR(_name, _type, _eventid, _occupid)                \
 330         (&((struct arm_cmn_event_attr[]) {{                             \
 331                 .attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL),  \
 332                 .type = _type,                                          \
 333                 .eventid = _eventid,                                    \
 334                 .occupid = _occupid,                                    \
 335         }})[0].attr.attr)
 336
 337 static bool arm_cmn_is_occup_event(enum cmn_node_type type, unsigned int id)
 338 {
 339         return (type == CMN_TYPE_DVM && id == 0x05) ||
 340                (type == CMN_TYPE_HNF && id == 0x0f);
 341 }
 342
 343 static ssize_t arm_cmn_event_show(struct device *dev,
 344                                   struct device_attribute *attr, char *buf)
 345 {
 346         struct arm_cmn_event_attr *eattr;
 347
 348         eattr = container_of(attr, typeof(*eattr), attr);
 349
 350         if (eattr->type == CMN_TYPE_DTC)
 351                 return sysfs_emit(buf, "type=0x%x\n", eattr->type);
 352
 353         if (eattr->type == CMN_TYPE_WP)
 354                 return sysfs_emit(buf,
 355                                   "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n",
 356                                   eattr->type, eattr->eventid);
 357
 358         if (arm_cmn_is_occup_event(eattr->type, eattr->eventid))
 359                 return sysfs_emit(buf, "type=0x%x,eventid=0x%x,occupid=0x%x\n",
 360                                   eattr->type, eattr->eventid, eattr->occupid);
 361
 362         return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type,
 363                           eattr->eventid);
 364 }
 365
 366 static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj,
 367                                              struct attribute *attr,
 368                                              int unused)
 369 {
 370         struct device *dev = kobj_to_dev(kobj);
 371         struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));
 372         struct arm_cmn_event_attr *eattr;
 373         enum cmn_node_type type;
 374
 375         eattr = container_of(attr, typeof(*eattr), attr.attr);
 376         type = eattr->type;
 377
 378         /* Watchpoints aren't nodes */
 379         if (type == CMN_TYPE_WP)
 380                 type = CMN_TYPE_XP;
 381
 382         /* Revision-specific differences */
 383         if (cmn->rev < CMN600_R1P2) {
 384                 if (type == CMN_TYPE_HNF && eattr->eventid == 0x1b)
 385                         return 0;
 386         }
 387
 388         if (!arm_cmn_node(cmn, type))
 389                 return 0;
 390
 391         return attr->mode;
 392 }
 393
 394 #define _CMN_EVENT_DVM(_name, _event, _occup)                   \
 395         CMN_EVENT_ATTR(dn_##_name, CMN_TYPE_DVM, _event, _occup)
 396 #define CMN_EVENT_DTC(_name)                                    \
 397         CMN_EVENT_ATTR(dtc_##_name, CMN_TYPE_DTC, 0, 0)
 398 #define _CMN_EVENT_HNF(_name, _event, _occup)                   \
 399         CMN_EVENT_ATTR(hnf_##_name, CMN_TYPE_HNF, _event, _occup)
 400 #define CMN_EVENT_HNI(_name, _event)                            \
 401         CMN_EVENT_ATTR(hni_##_name, CMN_TYPE_HNI, _event, 0)
 402 #define __CMN_EVENT_XP(_name, _event)                           \
 403         CMN_EVENT_ATTR(mxp_##_name, CMN_TYPE_XP, _event, 0)
 404 #define CMN_EVENT_SBSX(_name, _event)                           \
 405         CMN_EVENT_ATTR(sbsx_##_name, CMN_TYPE_SBSX, _event, 0)
 406 #define CMN_EVENT_RNID(_name, _event)                           \
 407         CMN_EVENT_ATTR(rnid_##_name, CMN_TYPE_RNI, _event, 0)
 408
 409 #define CMN_EVENT_DVM(_name, _event)                            \
 410         _CMN_EVENT_DVM(_name, _event, 0)
 411 #define CMN_EVENT_HNF(_name, _event)                            \
 412         _CMN_EVENT_HNF(_name, _event, 0)
 413 #define _CMN_EVENT_XP(_name, _event)                            \
 414         __CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)),         \
 415         __CMN_EVENT_XP(w_##_name, (_event) | (1 << 2)),         \
 416         __CMN_EVENT_XP(n_##_name, (_event) | (2 << 2)),         \
 417         __CMN_EVENT_XP(s_##_name, (_event) | (3 << 2)),         \
 418         __CMN_EVENT_XP(p0_##_name, (_event) | (4 << 2)),        \
 419         __CMN_EVENT_XP(p1_##_name, (_event) | (5 << 2))
 420
 421 /* Good thing there are only 3 fundamental XP events... */
 422 #define CMN_EVENT_XP(_name, _event)                             \
 423         _CMN_EVENT_XP(req_##_name, (_event) | (0 << 5)),        \
 424         _CMN_EVENT_XP(rsp_##_name, (_event) | (1 << 5)),        \
 425         _CMN_EVENT_XP(snp_##_name, (_event) | (2 << 5)),        \
 426         _CMN_EVENT_XP(dat_##_name, (_event) | (3 << 5))
 427
 428
 429 static struct attribute *arm_cmn_event_attrs[] = {
 430         CMN_EVENT_DTC(cycles),
 431
 432         /*
 433          * DVM node events conflict with HN-I events in the equivalent PMU
 434          * slot, but our lazy short-cut of using the DTM counter index for
 435          * the PMU index as well happens to avoid that by construction.
 436          */
 437         CMN_EVENT_DVM(rxreq_dvmop,      0x01),
 438         CMN_EVENT_DVM(rxreq_dvmsync,    0x02),
 439         CMN_EVENT_DVM(rxreq_dvmop_vmid_filtered, 0x03),
 440         CMN_EVENT_DVM(rxreq_retried,    0x04),
 441         _CMN_EVENT_DVM(rxreq_trk_occupancy_all, 0x05, 0),
 442         _CMN_EVENT_DVM(rxreq_trk_occupancy_dvmop, 0x05, 1),
 443         _CMN_EVENT_DVM(rxreq_trk_occupancy_dvmsync, 0x05, 2),
 444
 445         CMN_EVENT_HNF(cache_miss,       0x01),
 446         CMN_EVENT_HNF(slc_sf_cache_access, 0x02),
 447         CMN_EVENT_HNF(cache_fill,       0x03),
 448         CMN_EVENT_HNF(pocq_retry,       0x04),
 449         CMN_EVENT_HNF(pocq_reqs_recvd,  0x05),
 450         CMN_EVENT_HNF(sf_hit,           0x06),
 451         CMN_EVENT_HNF(sf_evictions,     0x07),
 452         CMN_EVENT_HNF(dir_snoops_sent,  0x08),
 453         CMN_EVENT_HNF(brd_snoops_sent,  0x09),
 454         CMN_EVENT_HNF(slc_eviction,     0x0a),
 455         CMN_EVENT_HNF(slc_fill_invalid_way, 0x0b),
 456         CMN_EVENT_HNF(mc_retries,       0x0c),
 457         CMN_EVENT_HNF(mc_reqs,          0x0d),
 458         CMN_EVENT_HNF(qos_hh_retry,     0x0e),
 459         _CMN_EVENT_HNF(qos_pocq_occupancy_all, 0x0f, 0),
 460         _CMN_EVENT_HNF(qos_pocq_occupancy_read, 0x0f, 1),
 461         _CMN_EVENT_HNF(qos_pocq_occupancy_write, 0x0f, 2),
 462         _CMN_EVENT_HNF(qos_pocq_occupancy_atomic, 0x0f, 3),
 463         _CMN_EVENT_HNF(qos_pocq_occupancy_stash, 0x0f, 4),
 464         CMN_EVENT_HNF(pocq_addrhaz,     0x10),
 465         CMN_EVENT_HNF(pocq_atomic_addrhaz, 0x11),
 466         CMN_EVENT_HNF(ld_st_swp_adq_full, 0x12),
 467         CMN_EVENT_HNF(cmp_adq_full,     0x13),
 468         CMN_EVENT_HNF(txdat_stall,      0x14),
 469         CMN_EVENT_HNF(txrsp_stall,      0x15),
 470         CMN_EVENT_HNF(seq_full,         0x16),
 471         CMN_EVENT_HNF(seq_hit,          0x17),
 472         CMN_EVENT_HNF(snp_sent,         0x18),
 473         CMN_EVENT_HNF(sfbi_dir_snp_sent, 0x19),
 474         CMN_EVENT_HNF(sfbi_brd_snp_sent, 0x1a),
 475         CMN_EVENT_HNF(snp_sent_untrk,   0x1b),
 476         CMN_EVENT_HNF(intv_dirty,       0x1c),
 477         CMN_EVENT_HNF(stash_snp_sent,   0x1d),
 478         CMN_EVENT_HNF(stash_data_pull,  0x1e),
 479         CMN_EVENT_HNF(snp_fwded,        0x1f),
 480
 481         CMN_EVENT_HNI(rrt_rd_occ_cnt_ovfl, 0x20),
 482         CMN_EVENT_HNI(rrt_wr_occ_cnt_ovfl, 0x21),
 483         CMN_EVENT_HNI(rdt_rd_occ_cnt_ovfl, 0x22),
 484         CMN_EVENT_HNI(rdt_wr_occ_cnt_ovfl, 0x23),
 485         CMN_EVENT_HNI(wdb_occ_cnt_ovfl, 0x24),
 486         CMN_EVENT_HNI(rrt_rd_alloc,     0x25),
 487         CMN_EVENT_HNI(rrt_wr_alloc,     0x26),
 488         CMN_EVENT_HNI(rdt_rd_alloc,     0x27),
 489         CMN_EVENT_HNI(rdt_wr_alloc,     0x28),
 490         CMN_EVENT_HNI(wdb_alloc,        0x29),
 491         CMN_EVENT_HNI(txrsp_retryack,   0x2a),
 492         CMN_EVENT_HNI(arvalid_no_arready, 0x2b),
 493         CMN_EVENT_HNI(arready_no_arvalid, 0x2c),
 494         CMN_EVENT_HNI(awvalid_no_awready, 0x2d),
 495         CMN_EVENT_HNI(awready_no_awvalid, 0x2e),
 496         CMN_EVENT_HNI(wvalid_no_wready, 0x2f),
 497         CMN_EVENT_HNI(txdat_stall,      0x30),
 498         CMN_EVENT_HNI(nonpcie_serialization, 0x31),
 499         CMN_EVENT_HNI(pcie_serialization, 0x32),
 500
 501         CMN_EVENT_XP(txflit_valid,      0x01),
 502         CMN_EVENT_XP(txflit_stall,      0x02),
 503         CMN_EVENT_XP(partial_dat_flit,  0x03),
 504         /* We treat watchpoints as a special made-up class of XP events */
 505         CMN_EVENT_ATTR(watchpoint_up, CMN_TYPE_WP, 0, 0),
 506         CMN_EVENT_ATTR(watchpoint_down, CMN_TYPE_WP, 2, 0),
 507
 508         CMN_EVENT_SBSX(rd_req,          0x01),
 509         CMN_EVENT_SBSX(wr_req,          0x02),
 510         CMN_EVENT_SBSX(cmo_req,         0x03),
 511         CMN_EVENT_SBSX(txrsp_retryack,  0x04),
 512         CMN_EVENT_SBSX(txdat_flitv,     0x05),
 513         CMN_EVENT_SBSX(txrsp_flitv,     0x06),
 514         CMN_EVENT_SBSX(rd_req_trkr_occ_cnt_ovfl, 0x11),
 515         CMN_EVENT_SBSX(wr_req_trkr_occ_cnt_ovfl, 0x12),
 516         CMN_EVENT_SBSX(cmo_req_trkr_occ_cnt_ovfl, 0x13),
 517         CMN_EVENT_SBSX(wdb_occ_cnt_ovfl, 0x14),
 518         CMN_EVENT_SBSX(rd_axi_trkr_occ_cnt_ovfl, 0x15),
 519         CMN_EVENT_SBSX(cmo_axi_trkr_occ_cnt_ovfl, 0x16),
 520         CMN_EVENT_SBSX(arvalid_no_arready, 0x21),
 521         CMN_EVENT_SBSX(awvalid_no_awready, 0x22),
 522         CMN_EVENT_SBSX(wvalid_no_wready, 0x23),
 523         CMN_EVENT_SBSX(txdat_stall,     0x24),
 524         CMN_EVENT_SBSX(txrsp_stall,     0x25),
 525
 526         CMN_EVENT_RNID(s0_rdata_beats,  0x01),
 527         CMN_EVENT_RNID(s1_rdata_beats,  0x02),
 528         CMN_EVENT_RNID(s2_rdata_beats,  0x03),
 529         CMN_EVENT_RNID(rxdat_flits,     0x04),
 530         CMN_EVENT_RNID(txdat_flits,     0x05),
 531         CMN_EVENT_RNID(txreq_flits_total, 0x06),
 532         CMN_EVENT_RNID(txreq_flits_retried, 0x07),
 533         CMN_EVENT_RNID(rrt_occ_ovfl,    0x08),
 534         CMN_EVENT_RNID(wrt_occ_ovfl,    0x09),
 535         CMN_EVENT_RNID(txreq_flits_replayed, 0x0a),
 536         CMN_EVENT_RNID(wrcancel_sent,   0x0b),
 537         CMN_EVENT_RNID(s0_wdata_beats,  0x0c),
 538         CMN_EVENT_RNID(s1_wdata_beats,  0x0d),
 539         CMN_EVENT_RNID(s2_wdata_beats,  0x0e),
 540         CMN_EVENT_RNID(rrt_alloc,       0x0f),
 541         CMN_EVENT_RNID(wrt_alloc,       0x10),
 542         CMN_EVENT_RNID(rdb_unord,       0x11),
 543         CMN_EVENT_RNID(rdb_replay,      0x12),
 544         CMN_EVENT_RNID(rdb_hybrid,      0x13),
 545         CMN_EVENT_RNID(rdb_ord,         0x14),
 546
 547         NULL
 548 };
 549
 550 static const struct attribute_group arm_cmn_event_attrs_group = {
 551         .name = "events",
 552         .attrs = arm_cmn_event_attrs,
 553         .is_visible = arm_cmn_event_attr_is_visible,
 554 };
 555
 556 static ssize_t arm_cmn_format_show(struct device *dev,
 557                                    struct device_attribute *attr, char *buf)
 558 {
 559         struct arm_cmn_format_attr *fmt = container_of(attr, typeof(*fmt), attr);
 560         int lo = __ffs(fmt->field), hi = __fls(fmt->field);
 561
 562         if (lo == hi)
 563                 return sysfs_emit(buf, "config:%d\n", lo);
 564
 565         if (!fmt->config)
 566                 return sysfs_emit(buf, "config:%d-%d\n", lo, hi);
 567
 568         return sysfs_emit(buf, "config%d:%d-%d\n", fmt->config, lo, hi);
 569 }
 570
 571 #define _CMN_FORMAT_ATTR(_name, _cfg, _fld)                             \
 572         (&((struct arm_cmn_format_attr[]) {{                            \
 573                 .attr = __ATTR(_name, 0444, arm_cmn_format_show, NULL), \
 574                 .config = _cfg,                                         \
 575                 .field = _fld,                                          \
 576         }})[0].attr.attr)
 577 #define CMN_FORMAT_ATTR(_name, _fld)    _CMN_FORMAT_ATTR(_name, 0, _fld)
 578
 579 static struct attribute *arm_cmn_format_attrs[] = {
 580         CMN_FORMAT_ATTR(type, CMN_CONFIG_TYPE),
 581         CMN_FORMAT_ATTR(eventid, CMN_CONFIG_EVENTID),
 582         CMN_FORMAT_ATTR(occupid, CMN_CONFIG_OCCUPID),
 583         CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID),
 584         CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID),
 585
 586         CMN_FORMAT_ATTR(wp_dev_sel, CMN_CONFIG_WP_DEV_SEL),
 587         CMN_FORMAT_ATTR(wp_chn_sel, CMN_CONFIG_WP_CHN_SEL),
 588         CMN_FORMAT_ATTR(wp_grp, CMN_CONFIG_WP_GRP),
 589         CMN_FORMAT_ATTR(wp_exclusive, CMN_CONFIG_WP_EXCLUSIVE),
 590         CMN_FORMAT_ATTR(wp_combine, CMN_CONFIG_WP_COMBINE),
 591
 592         _CMN_FORMAT_ATTR(wp_val, 1, CMN_CONFIG1_WP_VAL),
 593         _CMN_FORMAT_ATTR(wp_mask, 2, CMN_CONFIG2_WP_MASK),
 594
 595         NULL
 596 };
 597
 598 static const struct attribute_group arm_cmn_format_attrs_group = {
 599         .name = "format",
 600         .attrs = arm_cmn_format_attrs,
 601 };
 602
 603 static ssize_t arm_cmn_cpumask_show(struct device *dev,
 604                                     struct device_attribute *attr, char *buf)
 605 {
 606         struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));
 607
 608         return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu));
 609 }
 610
 611 static struct device_attribute arm_cmn_cpumask_attr =
 612                 __ATTR(cpumask, 0444, arm_cmn_cpumask_show, NULL);
 613
 614 static struct attribute *arm_cmn_cpumask_attrs[] = {
 615         &arm_cmn_cpumask_attr.attr,
 616         NULL,
 617 };
 618
 619 static const struct attribute_group arm_cmn_cpumask_attr_group = {
 620         .attrs = arm_cmn_cpumask_attrs,
 621 };
 622
 623 static const struct attribute_group *arm_cmn_attr_groups[] = {
 624         &arm_cmn_event_attrs_group,
 625         &arm_cmn_format_attrs_group,
 626         &arm_cmn_cpumask_attr_group,
 627         NULL
 628 };
 629
 630 static int arm_cmn_wp_idx(struct perf_event *event)
 631 {
 632         return CMN_EVENT_EVENTID(event) + CMN_EVENT_WP_GRP(event);
 633 }
 634
 635 static u32 arm_cmn_wp_config(struct perf_event *event)
 636 {
 637         u32 config;
 638         u32 dev = CMN_EVENT_WP_DEV_SEL(event);
 639         u32 chn = CMN_EVENT_WP_CHN_SEL(event);
 640         u32 grp = CMN_EVENT_WP_GRP(event);
 641         u32 exc = CMN_EVENT_WP_EXCLUSIVE(event);
 642         u32 combine = CMN_EVENT_WP_COMBINE(event);
 643
 644         config = FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL, dev) |
 645                  FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_CHN_SEL, chn) |
 646                  FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_GRP, grp) |
 647                  FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE, exc);
 648         if (combine && !grp)
 649                 config |= CMN_DTM_WPn_CONFIG_WP_COMBINE;
 650
 651         return config;
 652 }
 653
 654 static void arm_cmn_set_state(struct arm_cmn *cmn, u32 state)
 655 {
 656         if (!cmn->state)
 657                 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_PMCR);
 658         cmn->state |= state;
 659 }
 660
 661 static void arm_cmn_clear_state(struct arm_cmn *cmn, u32 state)
 662 {
 663         cmn->state &= ~state;
 664         if (!cmn->state)
 665                 writel_relaxed(CMN_DT_PMCR_PMU_EN | CMN_DT_PMCR_OVFL_INTR_EN,
 666                                cmn->dtc[0].base + CMN_DT_PMCR);
 667 }
 668
 669 static void arm_cmn_pmu_enable(struct pmu *pmu)
 670 {
 671         arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_DISABLED);
 672 }
 673
 674 static void arm_cmn_pmu_disable(struct pmu *pmu)
 675 {
 676         arm_cmn_set_state(to_cmn(pmu), CMN_STATE_DISABLED);
 677 }
 678
 679 static u64 arm_cmn_read_dtm(struct arm_cmn *cmn, struct arm_cmn_hw_event *hw,
 680                             bool snapshot)
 681 {
 682         struct arm_cmn_node *dn;
 683         unsigned int i, offset;
 684         u64 count = 0;
 685
 686         offset = snapshot ? CMN_DTM_PMEVCNTSR : CMN_DTM_PMEVCNT;
 687         for_each_hw_dn(hw, dn, i) {
 688                 struct arm_cmn_node *xp = arm_cmn_node_to_xp(dn);
 689                 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
 690                 u64 reg = readq_relaxed(xp->pmu_base + offset);
 691                 u16 dtm_count = reg >> (dtm_idx * 16);
 692
 693                 count += dtm_count;
 694         }
 695         return count;
 696 }
 697
 698 static u64 arm_cmn_read_cc(struct arm_cmn_dtc *dtc)
 699 {
 700         u64 val = readq_relaxed(dtc->base + CMN_DT_PMCCNTR);
 701
 702         writeq_relaxed(CMN_CC_INIT, dtc->base + CMN_DT_PMCCNTR);
 703         return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1);
 704 }
 705
 706 static u32 arm_cmn_read_counter(struct arm_cmn_dtc *dtc, int idx)
 707 {
 708         u32 val, pmevcnt = CMN_DT_PMEVCNT(idx);
 709
 710         val = readl_relaxed(dtc->base + pmevcnt);
 711         writel_relaxed(CMN_COUNTER_INIT, dtc->base + pmevcnt);
 712         return val - CMN_COUNTER_INIT;
 713 }
 714
 715 static void arm_cmn_init_counter(struct perf_event *event)
 716 {
 717         struct arm_cmn *cmn = to_cmn(event->pmu);
 718         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
 719         unsigned int i, pmevcnt = CMN_DT_PMEVCNT(hw->dtc_idx);
 720         u64 count;
 721
 722         for (i = 0; hw->dtcs_used & (1U << i); i++) {
 723                 writel_relaxed(CMN_COUNTER_INIT, cmn->dtc[i].base + pmevcnt);
 724                 cmn->dtc[i].counters[hw->dtc_idx] = event;
 725         }
 726
 727         count = arm_cmn_read_dtm(cmn, hw, false);
 728         local64_set(&event->hw.prev_count, count);
 729 }
 730
 731 static void arm_cmn_event_read(struct perf_event *event)
 732 {
 733         struct arm_cmn *cmn = to_cmn(event->pmu);
 734         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
 735         u64 delta, new, prev;
 736         unsigned long flags;
 737         unsigned int i;
 738
 739         if (hw->dtc_idx == CMN_DT_NUM_COUNTERS) {
 740                 i = __ffs(hw->dtcs_used);
 741                 delta = arm_cmn_read_cc(cmn->dtc + i);
 742                 local64_add(delta, &event->count);
 743                 return;
 744         }
 745         new = arm_cmn_read_dtm(cmn, hw, false);
 746         prev = local64_xchg(&event->hw.prev_count, new);
 747
 748         delta = new - prev;
 749
 750         local_irq_save(flags);
 751         for (i = 0; hw->dtcs_used & (1U << i); i++) {
 752                 new = arm_cmn_read_counter(cmn->dtc + i, hw->dtc_idx);
 753                 delta += new << 16;
 754         }
 755         local_irq_restore(flags);
 756         local64_add(delta, &event->count);
 757 }
 758
 759 static void arm_cmn_event_start(struct perf_event *event, int flags)
 760 {
 761         struct arm_cmn *cmn = to_cmn(event->pmu);
 762         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
 763         struct arm_cmn_node *dn;
 764         enum cmn_node_type type = CMN_EVENT_TYPE(event);
 765         int i;
 766
 767         if (type == CMN_TYPE_DTC) {
 768                 i = __ffs(hw->dtcs_used);
 769                 writeq_relaxed(CMN_CC_INIT, cmn->dtc[i].base + CMN_DT_PMCCNTR);
 770                 cmn->dtc[i].cc_active = true;
 771         } else if (type == CMN_TYPE_WP) {
 772                 int wp_idx = arm_cmn_wp_idx(event);
 773                 u64 val = CMN_EVENT_WP_VAL(event);
 774                 u64 mask = CMN_EVENT_WP_MASK(event);
 775
 776                 for_each_hw_dn(hw, dn, i) {
 777                         writeq_relaxed(val, dn->pmu_base + CMN_DTM_WPn_VAL(wp_idx));
 778                         writeq_relaxed(mask, dn->pmu_base + CMN_DTM_WPn_MASK(wp_idx));
 779                 }
 780         } else for_each_hw_dn(hw, dn, i) {
 781                 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
 782
 783                 dn->event[dtm_idx] = CMN_EVENT_EVENTID(event);
 784                 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL);
 785         }
 786 }
 787
 788 static void arm_cmn_event_stop(struct perf_event *event, int flags)
 789 {
 790         struct arm_cmn *cmn = to_cmn(event->pmu);
 791         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
 792         struct arm_cmn_node *dn;
 793         enum cmn_node_type type = CMN_EVENT_TYPE(event);
 794         int i;
 795
 796         if (type == CMN_TYPE_DTC) {
 797                 i = __ffs(hw->dtcs_used);
 798                 cmn->dtc[i].cc_active = false;
 799         } else if (type == CMN_TYPE_WP) {
 800                 int wp_idx = arm_cmn_wp_idx(event);
 801
 802                 for_each_hw_dn(hw, dn, i) {
 803                         writeq_relaxed(0, dn->pmu_base + CMN_DTM_WPn_MASK(wp_idx));
 804                         writeq_relaxed(~0ULL, dn->pmu_base + CMN_DTM_WPn_VAL(wp_idx));
 805                 }
 806         } else for_each_hw_dn(hw, dn, i) {
 807                 int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
 808
 809                 dn->event[dtm_idx] = 0;
 810                 writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL);
 811         }
 812
 813         arm_cmn_event_read(event);
 814 }
 815
 816 struct arm_cmn_val {
 817         u8 dtm_count[CMN_MAX_XPS];
 818         u8 occupid[CMN_MAX_XPS];
 819         u8 wp[CMN_MAX_XPS][4];
 820         int dtc_count;
 821         bool cycles;
 822 };
 823
 824 static void arm_cmn_val_add_event(struct arm_cmn_val *val, struct perf_event *event)
 825 {
 826         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
 827         struct arm_cmn_node *dn;
 828         enum cmn_node_type type;
 829         int i;
 830         u8 occupid;
 831
 832         if (is_software_event(event))
 833                 return;
 834
 835         type = CMN_EVENT_TYPE(event);
 836         if (type == CMN_TYPE_DTC) {
 837                 val->cycles = true;
 838                 return;
 839         }
 840
 841         val->dtc_count++;
 842         if (arm_cmn_is_occup_event(type, CMN_EVENT_EVENTID(event)))
 843                 occupid = CMN_EVENT_OCCUPID(event) + 1;
 844         else
 845                 occupid = 0;
 846
 847         for_each_hw_dn(hw, dn, i) {
 848                 int wp_idx, xp = arm_cmn_node_to_xp(dn)->logid;
 849
 850                 val->dtm_count[xp]++;
 851                 val->occupid[xp] = occupid;
 852
 853                 if (type != CMN_TYPE_WP)
 854                         continue;
 855
 856                 wp_idx = arm_cmn_wp_idx(event);
 857                 val->wp[xp][wp_idx] = CMN_EVENT_WP_COMBINE(event) + 1;
 858         }
 859 }
 860
 861 static int arm_cmn_validate_group(struct perf_event *event)
 862 {
 863         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
 864         struct arm_cmn_node *dn;
 865         struct perf_event *sibling, *leader = event->group_leader;
 866         enum cmn_node_type type;
 867         struct arm_cmn_val val;
 868         int i;
 869         u8 occupid;
 870
 871         if (leader == event)
 872                 return 0;
 873
 874         if (event->pmu != leader->pmu && !is_software_event(leader))
 875                 return -EINVAL;
 876
 877         memset(&val, 0, sizeof(val));
 878
 879         arm_cmn_val_add_event(&val, leader);
 880         for_each_sibling_event(sibling, leader)
 881                 arm_cmn_val_add_event(&val, sibling);
 882
 883         type = CMN_EVENT_TYPE(event);
 884         if (type == CMN_TYPE_DTC)
 885                 return val.cycles ? -EINVAL : 0;
 886
 887         if (val.dtc_count == CMN_DT_NUM_COUNTERS)
 888                 return -EINVAL;
 889
 890         if (arm_cmn_is_occup_event(type, CMN_EVENT_EVENTID(event)))
 891                 occupid = CMN_EVENT_OCCUPID(event) + 1;
 892         else
 893                 occupid = 0;
 894
 895         for_each_hw_dn(hw, dn, i) {
 896                 int wp_idx, wp_cmb, xp = arm_cmn_node_to_xp(dn)->logid;
 897
 898                 if (val.dtm_count[xp] == CMN_DTM_NUM_COUNTERS)
 899                         return -EINVAL;
 900
 901                 if (occupid && val.occupid[xp] && occupid != val.occupid[xp])
 902                         return -EINVAL;
 903
 904                 if (type != CMN_TYPE_WP)
 905                         continue;
 906
 907                 wp_idx = arm_cmn_wp_idx(event);
 908                 if (val.wp[xp][wp_idx])
 909                         return -EINVAL;
 910
 911                 wp_cmb = val.wp[xp][wp_idx ^ 1];
 912                 if (wp_cmb && wp_cmb != CMN_EVENT_WP_COMBINE(event) + 1)
 913                         return -EINVAL;
 914         }
 915
 916         return 0;
 917 }
 918
 919 static int arm_cmn_event_init(struct perf_event *event)
 920 {
 921         struct arm_cmn *cmn = to_cmn(event->pmu);
 922         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
 923         enum cmn_node_type type;
 924         unsigned int i;
 925         bool bynodeid;
 926         u16 nodeid, eventid;
 927
 928         if (event->attr.type != event->pmu->type)
 929                 return -ENOENT;
 930
 931         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 932                 return -EINVAL;
 933
 934         event->cpu = cmn->cpu;
 935         if (event->cpu < 0)
 936                 return -EINVAL;
 937
 938         type = CMN_EVENT_TYPE(event);
 939         /* DTC events (i.e. cycles) already have everything they need */
 940         if (type == CMN_TYPE_DTC)
 941                 return 0;
 942
 943         /* For watchpoints we need the actual XP node here */
 944         if (type == CMN_TYPE_WP) {
 945                 type = CMN_TYPE_XP;
 946                 /* ...and we need a "real" direction */
 947                 eventid = CMN_EVENT_EVENTID(event);
 948                 if (eventid != CMN_WP_UP && eventid != CMN_WP_DOWN)
 949                         return -EINVAL;
 950         }
 951
 952         bynodeid = CMN_EVENT_BYNODEID(event);
 953         nodeid = CMN_EVENT_NODEID(event);
 954
 955         hw->dn = arm_cmn_node(cmn, type);
 956         for (i = hw->dn - cmn->dns; i < cmn->num_dns && cmn->dns[i].type == type; i++) {
 957                 if (!bynodeid) {
 958                         hw->num_dns++;
 959                 } else if (cmn->dns[i].id != nodeid) {
 960                         hw->dn++;
 961                 } else {
 962                         hw->num_dns = 1;
 963                         break;
 964                 }
 965         }
 966
 967         if (!hw->num_dns) {
 968                 int bits = arm_cmn_xyidbits(cmn);
 969
 970                 dev_dbg(cmn->dev, "invalid node 0x%x (%d,%d,%d,%d) type 0x%x\n",
 971                         nodeid, CMN_NODEID_X(nodeid, bits), CMN_NODEID_Y(nodeid, bits),
 972                         CMN_NODEID_PID(nodeid), CMN_NODEID_DEVID(nodeid), type);
 973                 return -EINVAL;
 974         }
 975         /*
 976          * By assuming events count in all DTC domains, we cunningly avoid
 977          * needing to know anything about how XPs are assigned to domains.
 978          */
 979         hw->dtcs_used = (1U << cmn->num_dtcs) - 1;
 980
 981         return arm_cmn_validate_group(event);
 982 }
 983
 984 static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event,
 985                                 int i)
 986 {
 987         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
 988         enum cmn_node_type type = CMN_EVENT_TYPE(event);
 989
 990         while (i--) {
 991                 struct arm_cmn_node *xp = arm_cmn_node_to_xp(hw->dn + i);
 992                 unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
 993
 994                 if (type == CMN_TYPE_WP)
 995                         hw->dn[i].wp_event[arm_cmn_wp_idx(event)] = -1;
 996
 997                 if (arm_cmn_is_occup_event(type, CMN_EVENT_EVENTID(event)))
 998                         hw->dn[i].occupid_count--;
 999
1000                 xp->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx);
1001                 writel_relaxed(xp->pmu_config_low, xp->pmu_base + CMN_DTM_PMU_CONFIG);
1002         }
1003         memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx));
1004
1005         for (i = 0; hw->dtcs_used & (1U << i); i++)
1006                 cmn->dtc[i].counters[hw->dtc_idx] = NULL;
1007 }
1008
1009 static int arm_cmn_event_add(struct perf_event *event, int flags)
1010 {
1011         struct arm_cmn *cmn = to_cmn(event->pmu);
1012         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1013         struct arm_cmn_dtc *dtc = &cmn->dtc[0];
1014         struct arm_cmn_node *dn;
1015         enum cmn_node_type type = CMN_EVENT_TYPE(event);
1016         unsigned int i, dtc_idx, input_sel;
1017
1018         if (type == CMN_TYPE_DTC) {
1019                 i = 0;
1020                 while (cmn->dtc[i].cycles)
1021                         if (++i == cmn->num_dtcs)
1022                                 return -ENOSPC;
1023
1024                 cmn->dtc[i].cycles = event;
1025                 hw->dtc_idx = CMN_DT_NUM_COUNTERS;
1026                 hw->dtcs_used = 1U << i;
1027
1028                 if (flags & PERF_EF_START)
1029                         arm_cmn_event_start(event, 0);
1030                 return 0;
1031         }
1032
1033         /* Grab a free global counter first... */
1034         dtc_idx = 0;
1035         while (dtc->counters[dtc_idx])
1036                 if (++dtc_idx == CMN_DT_NUM_COUNTERS)
1037                         return -ENOSPC;
1038
1039         hw->dtc_idx = dtc_idx;
1040
1041         /* ...then the local counters to feed it. */
1042         for_each_hw_dn(hw, dn, i) {
1043                 struct arm_cmn_node *xp = arm_cmn_node_to_xp(dn);
1044                 unsigned int dtm_idx, shift;
1045                 u64 reg;
1046
1047                 dtm_idx = 0;
1048                 while (xp->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx))
1049                         if (++dtm_idx == CMN_DTM_NUM_COUNTERS)
1050                                 goto free_dtms;
1051
1052                 if (type == CMN_TYPE_XP) {
1053                         input_sel = CMN__PMEVCNT0_INPUT_SEL_XP + dtm_idx;
1054                 } else if (type == CMN_TYPE_WP) {
1055                         int tmp, wp_idx = arm_cmn_wp_idx(event);
1056                         u32 cfg = arm_cmn_wp_config(event);
1057
1058                         if (dn->wp_event[wp_idx] >= 0)
1059                                 goto free_dtms;
1060
1061                         tmp = dn->wp_event[wp_idx ^ 1];
1062                         if (tmp >= 0 && CMN_EVENT_WP_COMBINE(event) !=
1063                                         CMN_EVENT_WP_COMBINE(dtc->counters[tmp]))
1064                                 goto free_dtms;
1065
1066                         input_sel = CMN__PMEVCNT0_INPUT_SEL_WP + wp_idx;
1067                         dn->wp_event[wp_idx] = dtc_idx;
1068                         writel_relaxed(cfg, dn->pmu_base + CMN_DTM_WPn_CONFIG(wp_idx));
1069                 } else {
1070                         unsigned int port = CMN_NODEID_PID(dn->id);
1071                         unsigned int dev = CMN_NODEID_DEVID(dn->id);
1072
1073                         input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx +
1074                                     (port << 4) + (dev << 2);
1075
1076                         if (arm_cmn_is_occup_event(type, CMN_EVENT_EVENTID(event))) {
1077                                 int occupid = CMN_EVENT_OCCUPID(event);
1078
1079                                 if (dn->occupid_count == 0) {
1080                                         dn->occupid_val = occupid;
1081                                         writel_relaxed(occupid,
1082                                                        dn->pmu_base + CMN_PMU_EVENT_SEL + 4);
1083                                 } else if (dn->occupid_val != occupid) {
1084                                         goto free_dtms;
1085                                 }
1086                                 dn->occupid_count++;
1087                         }
1088                 }
1089
1090                 arm_cmn_set_index(hw->dtm_idx, i, dtm_idx);
1091
1092                 xp->input_sel[dtm_idx] = input_sel;
1093                 shift = CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(dtm_idx);
1094                 xp->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift);
1095                 xp->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, dtc_idx) << shift;
1096                 xp->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx);
1097                 reg = (u64)le32_to_cpu(xp->pmu_config_high) << 32 | xp->pmu_config_low;
1098                 writeq_relaxed(reg, xp->pmu_base + CMN_DTM_PMU_CONFIG);
1099         }
1100
1101         /* Go go go! */
1102         arm_cmn_init_counter(event);
1103
1104         if (flags & PERF_EF_START)
1105                 arm_cmn_event_start(event, 0);
1106
1107         return 0;
1108
1109 free_dtms:
1110         arm_cmn_event_clear(cmn, event, i);
1111         return -ENOSPC;
1112 }
1113
1114 static void arm_cmn_event_del(struct perf_event *event, int flags)
1115 {
1116         struct arm_cmn *cmn = to_cmn(event->pmu);
1117         struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1118         enum cmn_node_type type = CMN_EVENT_TYPE(event);
1119
1120         arm_cmn_event_stop(event, PERF_EF_UPDATE);
1121
1122         if (type == CMN_TYPE_DTC)
1123                 cmn->dtc[__ffs(hw->dtcs_used)].cycles = NULL;
1124         else
1125                 arm_cmn_event_clear(cmn, event, hw->num_dns);
1126 }
1127
1128 /*
1129  * We stop the PMU for both add and read, to avoid skew across DTM counters.
1130  * In theory we could use snapshots to read without stopping, but then it
1131  * becomes a lot trickier to deal with overlow and racing against interrupts,
1132  * plus it seems they don't work properly on some hardware anyway :(
1133  */
1134 static void arm_cmn_start_txn(struct pmu *pmu, unsigned int flags)
1135 {
1136         arm_cmn_set_state(to_cmn(pmu), CMN_STATE_TXN);
1137 }
1138
1139 static void arm_cmn_end_txn(struct pmu *pmu)
1140 {
1141         arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_TXN);
1142 }
1143
1144 static int arm_cmn_commit_txn(struct pmu *pmu)
1145 {
1146         arm_cmn_end_txn(pmu);
1147         return 0;
1148 }
1149
1150 static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
1151 {
1152         struct arm_cmn *cmn;
1153         unsigned int i, target;
1154
1155         cmn = hlist_entry_safe(node, struct arm_cmn, cpuhp_node);
1156         if (cpu != cmn->cpu)
1157                 return 0;
1158
1159         target = cpumask_any_but(cpu_online_mask, cpu);
1160         if (target >= nr_cpu_ids)
1161                 return 0;
1162
1163         perf_pmu_migrate_context(&cmn->pmu, cpu, target);
1164         for (i = 0; i < cmn->num_dtcs; i++)
1165                 irq_set_affinity_hint(cmn->dtc[i].irq, cpumask_of(target));
1166         cmn->cpu = target;
1167         return 0;
1168 }
1169
1170 static irqreturn_t arm_cmn_handle_irq(int irq, void *dev_id)
1171 {
1172         struct arm_cmn_dtc *dtc = dev_id;
1173         irqreturn_t ret = IRQ_NONE;
1174
1175         for (;;) {
1176                 u32 status = readl_relaxed(dtc->base + CMN_DT_PMOVSR);
1177                 u64 delta;
1178                 int i;
1179
1180                 for (i = 0; i < CMN_DTM_NUM_COUNTERS; i++) {
1181                         if (status & (1U << i)) {
1182                                 ret = IRQ_HANDLED;
1183                                 if (WARN_ON(!dtc->counters[i]))
1184                                         continue;
1185                                 delta = (u64)arm_cmn_read_counter(dtc, i) << 16;
1186                                 local64_add(delta, &dtc->counters[i]->count);
1187                         }
1188                 }
1189
1190                 if (status & (1U << CMN_DT_NUM_COUNTERS)) {
1191                         ret = IRQ_HANDLED;
1192                         if (dtc->cc_active && !WARN_ON(!dtc->cycles)) {
1193                                 delta = arm_cmn_read_cc(dtc);
1194                                 local64_add(delta, &dtc->cycles->count);
1195                         }
1196                 }
1197
1198                 writel_relaxed(status, dtc->base + CMN_DT_PMOVSR_CLR);
1199
1200                 if (!dtc->irq_friend)
1201                         return ret;
1202                 dtc += dtc->irq_friend;
1203         }
1204 }
1205
1206 /* We can reasonably accommodate DTCs of the same CMN sharing IRQs */
1207 static int arm_cmn_init_irqs(struct arm_cmn *cmn)
1208 {
1209         int i, j, irq, err;
1210
1211         for (i = 0; i < cmn->num_dtcs; i++) {
1212                 irq = cmn->dtc[i].irq;
1213                 for (j = i; j--; ) {
1214                         if (cmn->dtc[j].irq == irq) {
1215                                 cmn->dtc[j].irq_friend = j - i;
1216                                 goto next;
1217                         }
1218                 }
1219                 err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq,
1220                                        IRQF_NOBALANCING | IRQF_NO_THREAD,
1221                                        dev_name(cmn->dev), &cmn->dtc[i]);
1222                 if (err)
1223                         return err;
1224
1225                 err = irq_set_affinity_hint(irq, cpumask_of(cmn->cpu));
1226                 if (err)
1227                         return err;
1228         next:
1229                 ; /* isn't C great? */
1230         }
1231         return 0;
1232 }
1233
1234 static void arm_cmn_init_dtm(struct arm_cmn_node *xp)
1235 {
1236         int i;
1237
1238         for (i = 0; i < 4; i++) {
1239                 xp->wp_event[i] = -1;
1240                 writeq_relaxed(0, xp->pmu_base + CMN_DTM_WPn_MASK(i));
1241                 writeq_relaxed(~0ULL, xp->pmu_base + CMN_DTM_WPn_VAL(i));
1242         }
1243         xp->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN;
1244         xp->dtc = -1;
1245 }
1246
1247 static int arm_cmn_init_dtc(struct arm_cmn *cmn, struct arm_cmn_node *dn, int idx)
1248 {
1249         struct arm_cmn_dtc *dtc = cmn->dtc + idx;
1250         struct arm_cmn_node *xp;
1251
1252         dtc->base = dn->pmu_base - CMN_PMU_OFFSET;
1253         dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx);
1254         if (dtc->irq < 0)
1255                 return dtc->irq;
1256
1257         writel_relaxed(0, dtc->base + CMN_DT_PMCR);
1258         writel_relaxed(0x1ff, dtc->base + CMN_DT_PMOVSR_CLR);
1259         writel_relaxed(CMN_DT_PMCR_OVFL_INTR_EN, dtc->base + CMN_DT_PMCR);
1260
1261         /* We do at least know that a DTC's XP must be in that DTC's domain */
1262         xp = arm_cmn_node_to_xp(dn);
1263         xp->dtc = idx;
1264
1265         return 0;
1266 }
1267
1268 static int arm_cmn_node_cmp(const void *a, const void *b)
1269 {
1270         const struct arm_cmn_node *dna = a, *dnb = b;
1271         int cmp;
1272
1273         cmp = dna->type - dnb->type;
1274         if (!cmp)
1275                 cmp = dna->logid - dnb->logid;
1276         return cmp;
1277 }
1278
1279 static int arm_cmn_init_dtcs(struct arm_cmn *cmn)
1280 {
1281         struct arm_cmn_node *dn;
1282         int dtc_idx = 0;
1283
1284         cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL);
1285         if (!cmn->dtc)
1286                 return -ENOMEM;
1287
1288         sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL);
1289
1290         cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP);
1291
1292         for (dn = cmn->dns; dn < cmn->dns + cmn->num_dns; dn++) {
1293                 if (dn->type != CMN_TYPE_XP)
1294                         arm_cmn_init_node_to_xp(cmn, dn);
1295                 else if (cmn->num_dtcs == 1)
1296                         dn->dtc = 0;
1297
1298                 if (dn->type == CMN_TYPE_DTC)
1299                         arm_cmn_init_dtc(cmn, dn, dtc_idx++);
1300
1301                 /* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */
1302                 if (dn->type == CMN_TYPE_RND)
1303                         dn->type = CMN_TYPE_RNI;
1304         }
1305
1306         writel_relaxed(CMN_DT_DTC_CTL_DT_EN, cmn->dtc[0].base + CMN_DT_DTC_CTL);
1307
1308         return 0;
1309 }
1310
1311 static void arm_cmn_init_node_info(struct arm_cmn *cmn, u32 offset, struct arm_cmn_node *node)
1312 {
1313         int level;
1314         u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO);
1315
1316         node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg);
1317         node->id = FIELD_GET(CMN_NI_NODE_ID, reg);
1318         node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg);
1319
1320         node->pmu_base = cmn->base + offset + CMN_PMU_OFFSET;
1321
1322         if (node->type == CMN_TYPE_CFG)
1323                 level = 0;
1324         else if (node->type == CMN_TYPE_XP)
1325                 level = 1;
1326         else
1327                 level = 2;
1328
1329         dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n",
1330                         (level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ',
1331                         node->type, node->logid, offset);
1332 }
1333
1334 static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset)
1335 {
1336         void __iomem *cfg_region;
1337         struct arm_cmn_node cfg, *dn;
1338         u16 child_count, child_poff;
1339         u32 xp_offset[CMN_MAX_XPS];
1340         u64 reg;
1341         int i, j;
1342
1343         cfg_region = cmn->base + rgn_offset;
1344         reg = readl_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_2);
1345         cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg);
1346         dev_dbg(cmn->dev, "periph_id_2 revision: %d\n", cmn->rev);
1347
1348         arm_cmn_init_node_info(cmn, rgn_offset, &cfg);
1349         if (cfg.type != CMN_TYPE_CFG)
1350                 return -ENODEV;
1351
1352         reg = readq_relaxed(cfg_region + CMN_CHILD_INFO);
1353         child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg);
1354         child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg);
1355
1356         cmn->num_xps = child_count;
1357         cmn->num_dns = cmn->num_xps;
1358
1359         /* Pass 1: visit the XPs, enumerate their children */
1360         for (i = 0; i < cmn->num_xps; i++) {
1361                 reg = readq_relaxed(cfg_region + child_poff + i * 8);
1362                 xp_offset[i] = reg & CMN_CHILD_NODE_ADDR;
1363
1364                 reg = readq_relaxed(cmn->base + xp_offset[i] + CMN_CHILD_INFO);
1365                 cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg);
1366         }
1367
1368         /* Cheeky +1 to help terminate pointer-based iteration */
1369         cmn->dns = devm_kcalloc(cmn->dev, cmn->num_dns + 1,
1370                                 sizeof(*cmn->dns), GFP_KERNEL);
1371         if (!cmn->dns)
1372                 return -ENOMEM;
1373
1374         /* Pass 2: now we can actually populate the nodes */
1375         dn = cmn->dns;
1376         for (i = 0; i < cmn->num_xps; i++) {
1377                 void __iomem *xp_region = cmn->base + xp_offset[i];
1378                 struct arm_cmn_node *xp = dn++;
1379
1380                 arm_cmn_init_node_info(cmn, xp_offset[i], xp);
1381                 arm_cmn_init_dtm(xp);
1382                 /*
1383                  * Thanks to the order in which XP logical IDs seem to be
1384                  * assigned, we can handily infer the mesh X dimension by
1385                  * looking out for the XP at (0,1) without needing to know
1386                  * the exact node ID format, which we can later derive.
1387                  */
1388                 if (xp->id == (1 << 3))
1389                         cmn->mesh_x = xp->logid;
1390
1391                 reg = readq_relaxed(xp_region + CMN_CHILD_INFO);
1392                 child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg);
1393                 child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg);
1394
1395                 for (j = 0; j < child_count; j++) {
1396                         reg = readq_relaxed(xp_region + child_poff + j * 8);
1397                         /*
1398                          * Don't even try to touch anything external, since in general
1399                          * we haven't a clue how to power up arbitrary CHI requesters.
1400                          * As of CMN-600r1 these could only be RN-SAMs or CXLAs,
1401                          * neither of which have any PMU events anyway.
1402                          * (Actually, CXLAs do seem to have grown some events in r1p2,
1403                          * but they don't go to regular XP DTMs, and they depend on
1404                          * secure configuration which we can't easily deal with)
1405                          */
1406                         if (reg & CMN_CHILD_NODE_EXTERNAL) {
1407                                 dev_dbg(cmn->dev, "ignoring external node %llx\n", reg);
1408                                 continue;
1409                         }
1410
1411                         arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn);
1412
1413                         switch (dn->type) {
1414                         case CMN_TYPE_DTC:
1415                                 cmn->num_dtcs++;
1416                                 dn++;
1417                                 break;
1418                         /* These guys have PMU events */
1419                         case CMN_TYPE_DVM:
1420                         case CMN_TYPE_HNI:
1421                         case CMN_TYPE_HNF:
1422                         case CMN_TYPE_SBSX:
1423                         case CMN_TYPE_RNI:
1424                         case CMN_TYPE_RND:
1425                         case CMN_TYPE_CXRA:
1426                         case CMN_TYPE_CXHA:
1427                                 dn++;
1428                                 break;
1429                         /* Nothing to see here */
1430                         case CMN_TYPE_RNSAM:
1431                         case CMN_TYPE_CXLA:
1432                                 break;
1433                         /* Something has gone horribly wrong */
1434                         default:
1435                                 dev_err(cmn->dev, "invalid device node type: 0x%x\n", dn->type);
1436                                 return -ENODEV;
1437                         }
1438                 }
1439         }
1440
1441         /* Correct for any nodes we skipped */
1442         cmn->num_dns = dn - cmn->dns;
1443
1444         /*
1445          * If mesh_x wasn't set during discovery then we never saw
1446          * an XP at (0,1), thus we must have an Nx1 configuration.
1447          */
1448         if (!cmn->mesh_x)
1449                 cmn->mesh_x = cmn->num_xps;
1450         cmn->mesh_y = cmn->num_xps / cmn->mesh_x;
1451
1452         dev_dbg(cmn->dev, "mesh %dx%d, ID width %d\n",
1453                 cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn));
1454
1455         return 0;
1456 }
1457
1458 static int arm_cmn_acpi_probe(struct platform_device *pdev, struct arm_cmn *cmn)
1459 {
1460         struct resource *cfg, *root;
1461
1462         cfg = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1463         if (!cfg)
1464                 return -EINVAL;
1465
1466         root = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1467         if (!root)
1468                 return -EINVAL;
1469
1470         if (!resource_contains(cfg, root))
1471                 swap(cfg, root);
1472         /*
1473          * Note that devm_ioremap_resource() is dumb and won't let the platform
1474          * device claim cfg when the ACPI companion device has already claimed
1475          * root within it. But since they *are* already both claimed in the
1476          * appropriate name, we don't really need to do it again here anyway.
1477          */
1478         cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg));
1479         if (!cmn->base)
1480                 return -ENOMEM;
1481
1482         return root->start - cfg->start;
1483 }
1484
1485 static int arm_cmn_of_probe(struct platform_device *pdev, struct arm_cmn *cmn)
1486 {
1487         struct device_node *np = pdev->dev.of_node;
1488         u32 rootnode;
1489         int ret;
1490
1491         cmn->base = devm_platform_ioremap_resource(pdev, 0);
1492         if (IS_ERR(cmn->base))
1493                 return PTR_ERR(cmn->base);
1494
1495         ret = of_property_read_u32(np, "arm,root-node", &rootnode);
1496         if (ret)
1497                 return ret;
1498
1499         return rootnode;
1500 }
1501
1502 static int arm_cmn_probe(struct platform_device *pdev)
1503 {
1504         struct arm_cmn *cmn;
1505         const char *name;
1506         static atomic_t id;
1507         int err, rootnode;
1508
1509         cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL);
1510         if (!cmn)
1511                 return -ENOMEM;
1512
1513         cmn->dev = &pdev->dev;
1514         platform_set_drvdata(pdev, cmn);
1515
1516         if (has_acpi_companion(cmn->dev))
1517                 rootnode = arm_cmn_acpi_probe(pdev, cmn);
1518         else
1519                 rootnode = arm_cmn_of_probe(pdev, cmn);
1520         if (rootnode < 0)
1521                 return rootnode;
1522
1523         err = arm_cmn_discover(cmn, rootnode);
1524         if (err)
1525                 return err;
1526
1527         err = arm_cmn_init_dtcs(cmn);
1528         if (err)
1529                 return err;
1530
1531         err = arm_cmn_init_irqs(cmn);
1532         if (err)
1533                 return err;
1534
1535         cmn->cpu = raw_smp_processor_id();
1536         cmn->pmu = (struct pmu) {
1537                 .module = THIS_MODULE,
1538                 .attr_groups = arm_cmn_attr_groups,
1539                 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1540                 .task_ctx_nr = perf_invalid_context,
1541                 .pmu_enable = arm_cmn_pmu_enable,
1542                 .pmu_disable = arm_cmn_pmu_disable,
1543                 .event_init = arm_cmn_event_init,
1544                 .add = arm_cmn_event_add,
1545                 .del = arm_cmn_event_del,
1546                 .start = arm_cmn_event_start,
1547                 .stop = arm_cmn_event_stop,
1548                 .read = arm_cmn_event_read,
1549                 .start_txn = arm_cmn_start_txn,
1550                 .commit_txn = arm_cmn_commit_txn,
1551                 .cancel_txn = arm_cmn_end_txn,
1552         };
1553
1554         name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", atomic_fetch_inc(&id));
1555         if (!name)
1556                 return -ENOMEM;
1557
1558         err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
1559         if (err)
1560                 return err;
1561
1562         err = perf_pmu_register(&cmn->pmu, name, -1);
1563         if (err)
1564                 cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
1565         return err;
1566 }
1567
1568 static int arm_cmn_remove(struct platform_device *pdev)
1569 {
1570         struct arm_cmn *cmn = platform_get_drvdata(pdev);
1571         int i;
1572
1573         writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL);
1574
1575         perf_pmu_unregister(&cmn->pmu);
1576         cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
1577
1578         for (i = 0; i < cmn->num_dtcs; i++)
1579                 irq_set_affinity_hint(cmn->dtc[i].irq, NULL);
1580
1581         return 0;
1582 }
1583
1584 #ifdef CONFIG_OF
1585 static const struct of_device_id arm_cmn_of_match[] = {
1586         { .compatible = "arm,cmn-600", },
1587         {}
1588 };
1589 MODULE_DEVICE_TABLE(of, arm_cmn_of_match);
1590 #endif
1591
1592 #ifdef CONFIG_ACPI
1593 static const struct acpi_device_id arm_cmn_acpi_match[] = {
1594         { "ARMHC600", },
1595         {}
1596 };
1597 MODULE_DEVICE_TABLE(acpi, arm_cmn_acpi_match);
1598 #endif
1599
1600 static struct platform_driver arm_cmn_driver = {
1601         .driver = {
1602                 .name = "arm-cmn",
1603                 .of_match_table = of_match_ptr(arm_cmn_of_match),
1604                 .acpi_match_table = ACPI_PTR(arm_cmn_acpi_match),
1605         },
1606         .probe = arm_cmn_probe,
1607         .remove = arm_cmn_remove,
1608 };
1609
1610 static int __init arm_cmn_init(void)
1611 {
1612         int ret;
1613
1614         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1615                                       "perf/arm/cmn:online", NULL,
1616                                       arm_cmn_pmu_offline_cpu);
1617         if (ret < 0)
1618                 return ret;
1619
1620         arm_cmn_hp_state = ret;
1621         ret = platform_driver_register(&arm_cmn_driver);
1622         if (ret)
1623                 cpuhp_remove_multi_state(arm_cmn_hp_state);
1624         return ret;
1625 }
1626
1627 static void __exit arm_cmn_exit(void)
1628 {
1629         platform_driver_unregister(&arm_cmn_driver);
1630         cpuhp_remove_multi_state(arm_cmn_hp_state);
1631 }
1632
1633 module_init(arm_cmn_init);
1634 module_exit(arm_cmn_exit);
1635
1636 MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
1637 MODULE_DESCRIPTION("Arm CMN-600 PMU driver");
1638 MODULE_LICENSE("GPL v2");