arch/x86/events/intel/uncore_snb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Nehalem/SandBridge/Haswell/Broadwell/Skylake uncore support */
   3 #include "uncore.h"
   4
   5 /* Uncore IMC PCI IDs */
   6 #define PCI_DEVICE_ID_INTEL_SNB_IMC             0x0100
   7 #define PCI_DEVICE_ID_INTEL_IVB_IMC             0x0154
   8 #define PCI_DEVICE_ID_INTEL_IVB_E3_IMC          0x0150
   9 #define PCI_DEVICE_ID_INTEL_HSW_IMC             0x0c00
  10 #define PCI_DEVICE_ID_INTEL_HSW_U_IMC           0x0a04
  11 #define PCI_DEVICE_ID_INTEL_BDW_IMC             0x1604
  12 #define PCI_DEVICE_ID_INTEL_SKL_U_IMC           0x1904
  13 #define PCI_DEVICE_ID_INTEL_SKL_Y_IMC           0x190c
  14 #define PCI_DEVICE_ID_INTEL_SKL_HD_IMC          0x1900
  15 #define PCI_DEVICE_ID_INTEL_SKL_HQ_IMC          0x1910
  16 #define PCI_DEVICE_ID_INTEL_SKL_SD_IMC          0x190f
  17 #define PCI_DEVICE_ID_INTEL_SKL_SQ_IMC          0x191f
  18 #define PCI_DEVICE_ID_INTEL_SKL_E3_IMC          0x1918
  19 #define PCI_DEVICE_ID_INTEL_KBL_Y_IMC           0x590c
  20 #define PCI_DEVICE_ID_INTEL_KBL_U_IMC           0x5904
  21 #define PCI_DEVICE_ID_INTEL_KBL_UQ_IMC          0x5914
  22 #define PCI_DEVICE_ID_INTEL_KBL_SD_IMC          0x590f
  23 #define PCI_DEVICE_ID_INTEL_KBL_SQ_IMC          0x591f
  24 #define PCI_DEVICE_ID_INTEL_KBL_HQ_IMC          0x5910
  25 #define PCI_DEVICE_ID_INTEL_KBL_WQ_IMC          0x5918
  26 #define PCI_DEVICE_ID_INTEL_CFL_2U_IMC          0x3ecc
  27 #define PCI_DEVICE_ID_INTEL_CFL_4U_IMC          0x3ed0
  28 #define PCI_DEVICE_ID_INTEL_CFL_4H_IMC          0x3e10
  29 #define PCI_DEVICE_ID_INTEL_CFL_6H_IMC          0x3ec4
  30 #define PCI_DEVICE_ID_INTEL_CFL_2S_D_IMC        0x3e0f
  31 #define PCI_DEVICE_ID_INTEL_CFL_4S_D_IMC        0x3e1f
  32 #define PCI_DEVICE_ID_INTEL_CFL_6S_D_IMC        0x3ec2
  33 #define PCI_DEVICE_ID_INTEL_CFL_8S_D_IMC        0x3e30
  34 #define PCI_DEVICE_ID_INTEL_CFL_4S_W_IMC        0x3e18
  35 #define PCI_DEVICE_ID_INTEL_CFL_6S_W_IMC        0x3ec6
  36 #define PCI_DEVICE_ID_INTEL_CFL_8S_W_IMC        0x3e31
  37 #define PCI_DEVICE_ID_INTEL_CFL_4S_S_IMC        0x3e33
  38 #define PCI_DEVICE_ID_INTEL_CFL_6S_S_IMC        0x3eca
  39 #define PCI_DEVICE_ID_INTEL_CFL_8S_S_IMC        0x3e32
  40 #define PCI_DEVICE_ID_INTEL_AML_YD_IMC          0x590c
  41 #define PCI_DEVICE_ID_INTEL_AML_YQ_IMC          0x590d
  42 #define PCI_DEVICE_ID_INTEL_WHL_UQ_IMC          0x3ed0
  43 #define PCI_DEVICE_ID_INTEL_WHL_4_UQ_IMC        0x3e34
  44 #define PCI_DEVICE_ID_INTEL_WHL_UD_IMC          0x3e35
  45 #define PCI_DEVICE_ID_INTEL_CML_H1_IMC          0x9b44
  46 #define PCI_DEVICE_ID_INTEL_CML_H2_IMC          0x9b54
  47 #define PCI_DEVICE_ID_INTEL_CML_H3_IMC          0x9b64
  48 #define PCI_DEVICE_ID_INTEL_CML_U1_IMC          0x9b51
  49 #define PCI_DEVICE_ID_INTEL_CML_U2_IMC          0x9b61
  50 #define PCI_DEVICE_ID_INTEL_CML_U3_IMC          0x9b71
  51 #define PCI_DEVICE_ID_INTEL_CML_S1_IMC          0x9b33
  52 #define PCI_DEVICE_ID_INTEL_CML_S2_IMC          0x9b43
  53 #define PCI_DEVICE_ID_INTEL_CML_S3_IMC          0x9b53
  54 #define PCI_DEVICE_ID_INTEL_CML_S4_IMC          0x9b63
  55 #define PCI_DEVICE_ID_INTEL_CML_S5_IMC          0x9b73
  56 #define PCI_DEVICE_ID_INTEL_ICL_U_IMC           0x8a02
  57 #define PCI_DEVICE_ID_INTEL_ICL_U2_IMC          0x8a12
  58 #define PCI_DEVICE_ID_INTEL_TGL_U1_IMC          0x9a02
  59 #define PCI_DEVICE_ID_INTEL_TGL_U2_IMC          0x9a04
  60 #define PCI_DEVICE_ID_INTEL_TGL_U3_IMC          0x9a12
  61 #define PCI_DEVICE_ID_INTEL_TGL_U4_IMC          0x9a14
  62 #define PCI_DEVICE_ID_INTEL_TGL_H_IMC           0x9a36
  63
  64
  65 /* SNB event control */
  66 #define SNB_UNC_CTL_EV_SEL_MASK                 0x000000ff
  67 #define SNB_UNC_CTL_UMASK_MASK                  0x0000ff00
  68 #define SNB_UNC_CTL_EDGE_DET                    (1 << 18)
  69 #define SNB_UNC_CTL_EN                          (1 << 22)
  70 #define SNB_UNC_CTL_INVERT                      (1 << 23)
  71 #define SNB_UNC_CTL_CMASK_MASK                  0x1f000000
  72 #define NHM_UNC_CTL_CMASK_MASK                  0xff000000
  73 #define NHM_UNC_FIXED_CTR_CTL_EN                (1 << 0)
  74
  75 #define SNB_UNC_RAW_EVENT_MASK                  (SNB_UNC_CTL_EV_SEL_MASK | \
  76                                                  SNB_UNC_CTL_UMASK_MASK | \
  77                                                  SNB_UNC_CTL_EDGE_DET | \
  78                                                  SNB_UNC_CTL_INVERT | \
  79                                                  SNB_UNC_CTL_CMASK_MASK)
  80
  81 #define NHM_UNC_RAW_EVENT_MASK                  (SNB_UNC_CTL_EV_SEL_MASK | \
  82                                                  SNB_UNC_CTL_UMASK_MASK | \
  83                                                  SNB_UNC_CTL_EDGE_DET | \
  84                                                  SNB_UNC_CTL_INVERT | \
  85                                                  NHM_UNC_CTL_CMASK_MASK)
  86
  87 /* SNB global control register */
  88 #define SNB_UNC_PERF_GLOBAL_CTL                 0x391
  89 #define SNB_UNC_FIXED_CTR_CTRL                  0x394
  90 #define SNB_UNC_FIXED_CTR                       0x395
  91
  92 /* SNB uncore global control */
  93 #define SNB_UNC_GLOBAL_CTL_CORE_ALL             ((1 << 4) - 1)
  94 #define SNB_UNC_GLOBAL_CTL_EN                   (1 << 29)
  95
  96 /* SNB Cbo register */
  97 #define SNB_UNC_CBO_0_PERFEVTSEL0               0x700
  98 #define SNB_UNC_CBO_0_PER_CTR0                  0x706
  99 #define SNB_UNC_CBO_MSR_OFFSET                  0x10
 100
 101 /* SNB ARB register */
 102 #define SNB_UNC_ARB_PER_CTR0                    0x3b0
 103 #define SNB_UNC_ARB_PERFEVTSEL0                 0x3b2
 104 #define SNB_UNC_ARB_MSR_OFFSET                  0x10
 105
 106 /* NHM global control register */
 107 #define NHM_UNC_PERF_GLOBAL_CTL                 0x391
 108 #define NHM_UNC_FIXED_CTR                       0x394
 109 #define NHM_UNC_FIXED_CTR_CTRL                  0x395
 110
 111 /* NHM uncore global control */
 112 #define NHM_UNC_GLOBAL_CTL_EN_PC_ALL            ((1ULL << 8) - 1)
 113 #define NHM_UNC_GLOBAL_CTL_EN_FC                (1ULL << 32)
 114
 115 /* NHM uncore register */
 116 #define NHM_UNC_PERFEVTSEL0                     0x3c0
 117 #define NHM_UNC_UNCORE_PMC0                     0x3b0
 118
 119 /* SKL uncore global control */
 120 #define SKL_UNC_PERF_GLOBAL_CTL                 0xe01
 121 #define SKL_UNC_GLOBAL_CTL_CORE_ALL             ((1 << 5) - 1)
 122
 123 /* ICL Cbo register */
 124 #define ICL_UNC_CBO_CONFIG                      0x396
 125 #define ICL_UNC_NUM_CBO_MASK                    0xf
 126 #define ICL_UNC_CBO_0_PER_CTR0                  0x702
 127 #define ICL_UNC_CBO_MSR_OFFSET                  0x8
 128
 129 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
 130 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
 131 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
 132 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
 133 DEFINE_UNCORE_FORMAT_ATTR(cmask5, cmask, "config:24-28");
 134 DEFINE_UNCORE_FORMAT_ATTR(cmask8, cmask, "config:24-31");
 135
 136 /* Sandy Bridge uncore support */
 137 static void snb_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 138 {
 139         struct hw_perf_event *hwc = &event->hw;
 140
 141         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
 142                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 143         else
 144                 wrmsrl(hwc->config_base, SNB_UNC_CTL_EN);
 145 }
 146
 147 static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 148 {
 149         wrmsrl(event->hw.config_base, 0);
 150 }
 151
 152 static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
 153 {
 154         if (box->pmu->pmu_idx == 0) {
 155                 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
 156                         SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 157         }
 158 }
 159
 160 static void snb_uncore_msr_enable_box(struct intel_uncore_box *box)
 161 {
 162         wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
 163                 SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 164 }
 165
 166 static void snb_uncore_msr_exit_box(struct intel_uncore_box *box)
 167 {
 168         if (box->pmu->pmu_idx == 0)
 169                 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL, 0);
 170 }
 171
 172 static struct uncore_event_desc snb_uncore_events[] = {
 173         INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0x00"),
 174         { /* end: all zeroes */ },
 175 };
 176
 177 static struct attribute *snb_uncore_formats_attr[] = {
 178         &format_attr_event.attr,
 179         &format_attr_umask.attr,
 180         &format_attr_edge.attr,
 181         &format_attr_inv.attr,
 182         &format_attr_cmask5.attr,
 183         NULL,
 184 };
 185
 186 static const struct attribute_group snb_uncore_format_group = {
 187         .name           = "format",
 188         .attrs          = snb_uncore_formats_attr,
 189 };
 190
 191 static struct intel_uncore_ops snb_uncore_msr_ops = {
 192         .init_box       = snb_uncore_msr_init_box,
 193         .enable_box     = snb_uncore_msr_enable_box,
 194         .exit_box       = snb_uncore_msr_exit_box,
 195         .disable_event  = snb_uncore_msr_disable_event,
 196         .enable_event   = snb_uncore_msr_enable_event,
 197         .read_counter   = uncore_msr_read_counter,
 198 };
 199
 200 static struct event_constraint snb_uncore_arb_constraints[] = {
 201         UNCORE_EVENT_CONSTRAINT(0x80, 0x1),
 202         UNCORE_EVENT_CONSTRAINT(0x83, 0x1),
 203         EVENT_CONSTRAINT_END
 204 };
 205
 206 static struct intel_uncore_type snb_uncore_cbox = {
 207         .name           = "cbox",
 208         .num_counters   = 2,
 209         .num_boxes      = 4,
 210         .perf_ctr_bits  = 44,
 211         .fixed_ctr_bits = 48,
 212         .perf_ctr       = SNB_UNC_CBO_0_PER_CTR0,
 213         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
 214         .fixed_ctr      = SNB_UNC_FIXED_CTR,
 215         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
 216         .single_fixed   = 1,
 217         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 218         .msr_offset     = SNB_UNC_CBO_MSR_OFFSET,
 219         .ops            = &snb_uncore_msr_ops,
 220         .format_group   = &snb_uncore_format_group,
 221         .event_descs    = snb_uncore_events,
 222 };
 223
 224 static struct intel_uncore_type snb_uncore_arb = {
 225         .name           = "arb",
 226         .num_counters   = 2,
 227         .num_boxes      = 1,
 228         .perf_ctr_bits  = 44,
 229         .perf_ctr       = SNB_UNC_ARB_PER_CTR0,
 230         .event_ctl      = SNB_UNC_ARB_PERFEVTSEL0,
 231         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 232         .msr_offset     = SNB_UNC_ARB_MSR_OFFSET,
 233         .constraints    = snb_uncore_arb_constraints,
 234         .ops            = &snb_uncore_msr_ops,
 235         .format_group   = &snb_uncore_format_group,
 236 };
 237
 238 static struct intel_uncore_type *snb_msr_uncores[] = {
 239         &snb_uncore_cbox,
 240         &snb_uncore_arb,
 241         NULL,
 242 };
 243
 244 void snb_uncore_cpu_init(void)
 245 {
 246         uncore_msr_uncores = snb_msr_uncores;
 247         if (snb_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
 248                 snb_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
 249 }
 250
 251 static void skl_uncore_msr_init_box(struct intel_uncore_box *box)
 252 {
 253         if (box->pmu->pmu_idx == 0) {
 254                 wrmsrl(SKL_UNC_PERF_GLOBAL_CTL,
 255                         SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL);
 256         }
 257
 258         /* The 8th CBOX has different MSR space */
 259         if (box->pmu->pmu_idx == 7)
 260                 __set_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags);
 261 }
 262
 263 static void skl_uncore_msr_enable_box(struct intel_uncore_box *box)
 264 {
 265         wrmsrl(SKL_UNC_PERF_GLOBAL_CTL,
 266                 SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL);
 267 }
 268
 269 static void skl_uncore_msr_exit_box(struct intel_uncore_box *box)
 270 {
 271         if (box->pmu->pmu_idx == 0)
 272                 wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, 0);
 273 }
 274
 275 static struct intel_uncore_ops skl_uncore_msr_ops = {
 276         .init_box       = skl_uncore_msr_init_box,
 277         .enable_box     = skl_uncore_msr_enable_box,
 278         .exit_box       = skl_uncore_msr_exit_box,
 279         .disable_event  = snb_uncore_msr_disable_event,
 280         .enable_event   = snb_uncore_msr_enable_event,
 281         .read_counter   = uncore_msr_read_counter,
 282 };
 283
 284 static struct intel_uncore_type skl_uncore_cbox = {
 285         .name           = "cbox",
 286         .num_counters   = 4,
 287         .num_boxes      = 8,
 288         .perf_ctr_bits  = 44,
 289         .fixed_ctr_bits = 48,
 290         .perf_ctr       = SNB_UNC_CBO_0_PER_CTR0,
 291         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
 292         .fixed_ctr      = SNB_UNC_FIXED_CTR,
 293         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
 294         .single_fixed   = 1,
 295         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 296         .msr_offset     = SNB_UNC_CBO_MSR_OFFSET,
 297         .ops            = &skl_uncore_msr_ops,
 298         .format_group   = &snb_uncore_format_group,
 299         .event_descs    = snb_uncore_events,
 300 };
 301
 302 static struct intel_uncore_type *skl_msr_uncores[] = {
 303         &skl_uncore_cbox,
 304         &snb_uncore_arb,
 305         NULL,
 306 };
 307
 308 void skl_uncore_cpu_init(void)
 309 {
 310         uncore_msr_uncores = skl_msr_uncores;
 311         if (skl_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
 312                 skl_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
 313         snb_uncore_arb.ops = &skl_uncore_msr_ops;
 314 }
 315
 316 static struct intel_uncore_type icl_uncore_cbox = {
 317         .name           = "cbox",
 318         .num_counters   = 4,
 319         .perf_ctr_bits  = 44,
 320         .perf_ctr       = ICL_UNC_CBO_0_PER_CTR0,
 321         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
 322         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 323         .msr_offset     = ICL_UNC_CBO_MSR_OFFSET,
 324         .ops            = &skl_uncore_msr_ops,
 325         .format_group   = &snb_uncore_format_group,
 326 };
 327
 328 static struct uncore_event_desc icl_uncore_events[] = {
 329         INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff"),
 330         { /* end: all zeroes */ },
 331 };
 332
 333 static struct attribute *icl_uncore_clock_formats_attr[] = {
 334         &format_attr_event.attr,
 335         NULL,
 336 };
 337
 338 static struct attribute_group icl_uncore_clock_format_group = {
 339         .name = "format",
 340         .attrs = icl_uncore_clock_formats_attr,
 341 };
 342
 343 static struct intel_uncore_type icl_uncore_clockbox = {
 344         .name           = "clock",
 345         .num_counters   = 1,
 346         .num_boxes      = 1,
 347         .fixed_ctr_bits = 48,
 348         .fixed_ctr      = SNB_UNC_FIXED_CTR,
 349         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
 350         .single_fixed   = 1,
 351         .event_mask     = SNB_UNC_CTL_EV_SEL_MASK,
 352         .format_group   = &icl_uncore_clock_format_group,
 353         .ops            = &skl_uncore_msr_ops,
 354         .event_descs    = icl_uncore_events,
 355 };
 356
 357 static struct intel_uncore_type *icl_msr_uncores[] = {
 358         &icl_uncore_cbox,
 359         &snb_uncore_arb,
 360         &icl_uncore_clockbox,
 361         NULL,
 362 };
 363
 364 static int icl_get_cbox_num(void)
 365 {
 366         u64 num_boxes;
 367
 368         rdmsrl(ICL_UNC_CBO_CONFIG, num_boxes);
 369
 370         return num_boxes & ICL_UNC_NUM_CBO_MASK;
 371 }
 372
 373 void icl_uncore_cpu_init(void)
 374 {
 375         uncore_msr_uncores = icl_msr_uncores;
 376         icl_uncore_cbox.num_boxes = icl_get_cbox_num();
 377         snb_uncore_arb.ops = &skl_uncore_msr_ops;
 378 }
 379
 380 enum {
 381         SNB_PCI_UNCORE_IMC,
 382 };
 383
 384 static struct uncore_event_desc snb_uncore_imc_events[] = {
 385         INTEL_UNCORE_EVENT_DESC(data_reads,  "event=0x01"),
 386         INTEL_UNCORE_EVENT_DESC(data_reads.scale, "6.103515625e-5"),
 387         INTEL_UNCORE_EVENT_DESC(data_reads.unit, "MiB"),
 388
 389         INTEL_UNCORE_EVENT_DESC(data_writes, "event=0x02"),
 390         INTEL_UNCORE_EVENT_DESC(data_writes.scale, "6.103515625e-5"),
 391         INTEL_UNCORE_EVENT_DESC(data_writes.unit, "MiB"),
 392
 393         { /* end: all zeroes */ },
 394 };
 395
 396 #define SNB_UNCORE_PCI_IMC_EVENT_MASK           0xff
 397 #define SNB_UNCORE_PCI_IMC_BAR_OFFSET           0x48
 398
 399 /* page size multiple covering all config regs */
 400 #define SNB_UNCORE_PCI_IMC_MAP_SIZE             0x6000
 401
 402 #define SNB_UNCORE_PCI_IMC_DATA_READS           0x1
 403 #define SNB_UNCORE_PCI_IMC_DATA_READS_BASE      0x5050
 404 #define SNB_UNCORE_PCI_IMC_DATA_WRITES          0x2
 405 #define SNB_UNCORE_PCI_IMC_DATA_WRITES_BASE     0x5054
 406 #define SNB_UNCORE_PCI_IMC_CTR_BASE             SNB_UNCORE_PCI_IMC_DATA_READS_BASE
 407
 408 enum perf_snb_uncore_imc_freerunning_types {
 409         SNB_PCI_UNCORE_IMC_DATA         = 0,
 410         SNB_PCI_UNCORE_IMC_FREERUNNING_TYPE_MAX,
 411 };
 412
 413 static struct freerunning_counters snb_uncore_imc_freerunning[] = {
 414         [SNB_PCI_UNCORE_IMC_DATA]     = { SNB_UNCORE_PCI_IMC_DATA_READS_BASE, 0x4, 0x0, 2, 32 },
 415 };
 416
 417 static struct attribute *snb_uncore_imc_formats_attr[] = {
 418         &format_attr_event.attr,
 419         NULL,
 420 };
 421
 422 static const struct attribute_group snb_uncore_imc_format_group = {
 423         .name = "format",
 424         .attrs = snb_uncore_imc_formats_attr,
 425 };
 426
 427 static void snb_uncore_imc_init_box(struct intel_uncore_box *box)
 428 {
 429         struct intel_uncore_type *type = box->pmu->type;
 430         struct pci_dev *pdev = box->pci_dev;
 431         int where = SNB_UNCORE_PCI_IMC_BAR_OFFSET;
 432         resource_size_t addr;
 433         u32 pci_dword;
 434
 435         pci_read_config_dword(pdev, where, &pci_dword);
 436         addr = pci_dword;
 437
 438 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 439         pci_read_config_dword(pdev, where + 4, &pci_dword);
 440         addr |= ((resource_size_t)pci_dword << 32);
 441 #endif
 442
 443         addr &= ~(PAGE_SIZE - 1);
 444
 445         box->io_addr = ioremap(addr, type->mmio_map_size);
 446         if (!box->io_addr)
 447                 pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name);
 448
 449         box->hrtimer_duration = UNCORE_SNB_IMC_HRTIMER_INTERVAL;
 450 }
 451
 452 static void snb_uncore_imc_enable_box(struct intel_uncore_box *box)
 453 {}
 454
 455 static void snb_uncore_imc_disable_box(struct intel_uncore_box *box)
 456 {}
 457
 458 static void snb_uncore_imc_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 459 {}
 460
 461 static void snb_uncore_imc_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 462 {}
 463
 464 /*
 465  * Keep the custom event_init() function compatible with old event
 466  * encoding for free running counters.
 467  */
 468 static int snb_uncore_imc_event_init(struct perf_event *event)
 469 {
 470         struct intel_uncore_pmu *pmu;
 471         struct intel_uncore_box *box;
 472         struct hw_perf_event *hwc = &event->hw;
 473         u64 cfg = event->attr.config & SNB_UNCORE_PCI_IMC_EVENT_MASK;
 474         int idx, base;
 475
 476         if (event->attr.type != event->pmu->type)
 477                 return -ENOENT;
 478
 479         pmu = uncore_event_to_pmu(event);
 480         /* no device found for this pmu */
 481         if (pmu->func_id < 0)
 482                 return -ENOENT;
 483
 484         /* Sampling not supported yet */
 485         if (hwc->sample_period)
 486                 return -EINVAL;
 487
 488         /* unsupported modes and filters */
 489         if (event->attr.sample_period) /* no sampling */
 490                 return -EINVAL;
 491
 492         /*
 493          * Place all uncore events for a particular physical package
 494          * onto a single cpu
 495          */
 496         if (event->cpu < 0)
 497                 return -EINVAL;
 498
 499         /* check only supported bits are set */
 500         if (event->attr.config & ~SNB_UNCORE_PCI_IMC_EVENT_MASK)
 501                 return -EINVAL;
 502
 503         box = uncore_pmu_to_box(pmu, event->cpu);
 504         if (!box || box->cpu < 0)
 505                 return -EINVAL;
 506
 507         event->cpu = box->cpu;
 508         event->pmu_private = box;
 509
 510         event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
 511
 512         event->hw.idx = -1;
 513         event->hw.last_tag = ~0ULL;
 514         event->hw.extra_reg.idx = EXTRA_REG_NONE;
 515         event->hw.branch_reg.idx = EXTRA_REG_NONE;
 516         /*
 517          * check event is known (whitelist, determines counter)
 518          */
 519         switch (cfg) {
 520         case SNB_UNCORE_PCI_IMC_DATA_READS:
 521                 base = SNB_UNCORE_PCI_IMC_DATA_READS_BASE;
 522                 idx = UNCORE_PMC_IDX_FREERUNNING;
 523                 break;
 524         case SNB_UNCORE_PCI_IMC_DATA_WRITES:
 525                 base = SNB_UNCORE_PCI_IMC_DATA_WRITES_BASE;
 526                 idx = UNCORE_PMC_IDX_FREERUNNING;
 527                 break;
 528         default:
 529                 return -EINVAL;
 530         }
 531
 532         /* must be done before validate_group */
 533         event->hw.event_base = base;
 534         event->hw.idx = idx;
 535
 536         /* Convert to standard encoding format for freerunning counters */
 537         event->hw.config = ((cfg - 1) << 8) | 0x10ff;
 538
 539         /* no group validation needed, we have free running counters */
 540
 541         return 0;
 542 }
 543
 544 static int snb_uncore_imc_hw_config(struct intel_uncore_box *box, struct perf_event *event)
 545 {
 546         return 0;
 547 }
 548
 549 int snb_pci2phy_map_init(int devid)
 550 {
 551         struct pci_dev *dev = NULL;
 552         struct pci2phy_map *map;
 553         int bus, segment;
 554
 555         dev = pci_get_device(PCI_VENDOR_ID_INTEL, devid, dev);
 556         if (!dev)
 557                 return -ENOTTY;
 558
 559         bus = dev->bus->number;
 560         segment = pci_domain_nr(dev->bus);
 561
 562         raw_spin_lock(&pci2phy_map_lock);
 563         map = __find_pci2phy_map(segment);
 564         if (!map) {
 565                 raw_spin_unlock(&pci2phy_map_lock);
 566                 pci_dev_put(dev);
 567                 return -ENOMEM;
 568         }
 569         map->pbus_to_physid[bus] = 0;
 570         raw_spin_unlock(&pci2phy_map_lock);
 571
 572         pci_dev_put(dev);
 573
 574         return 0;
 575 }
 576
 577 static struct pmu snb_uncore_imc_pmu = {
 578         .task_ctx_nr    = perf_invalid_context,
 579         .event_init     = snb_uncore_imc_event_init,
 580         .add            = uncore_pmu_event_add,
 581         .del            = uncore_pmu_event_del,
 582         .start          = uncore_pmu_event_start,
 583         .stop           = uncore_pmu_event_stop,
 584         .read           = uncore_pmu_event_read,
 585         .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
 586 };
 587
 588 static struct intel_uncore_ops snb_uncore_imc_ops = {
 589         .init_box       = snb_uncore_imc_init_box,
 590         .exit_box       = uncore_mmio_exit_box,
 591         .enable_box     = snb_uncore_imc_enable_box,
 592         .disable_box    = snb_uncore_imc_disable_box,
 593         .disable_event  = snb_uncore_imc_disable_event,
 594         .enable_event   = snb_uncore_imc_enable_event,
 595         .hw_config      = snb_uncore_imc_hw_config,
 596         .read_counter   = uncore_mmio_read_counter,
 597 };
 598
 599 static struct intel_uncore_type snb_uncore_imc = {
 600         .name           = "imc",
 601         .num_counters   = 2,
 602         .num_boxes      = 1,
 603         .num_freerunning_types  = SNB_PCI_UNCORE_IMC_FREERUNNING_TYPE_MAX,
 604         .mmio_map_size  = SNB_UNCORE_PCI_IMC_MAP_SIZE,
 605         .freerunning    = snb_uncore_imc_freerunning,
 606         .event_descs    = snb_uncore_imc_events,
 607         .format_group   = &snb_uncore_imc_format_group,
 608         .ops            = &snb_uncore_imc_ops,
 609         .pmu            = &snb_uncore_imc_pmu,
 610 };
 611
 612 static struct intel_uncore_type *snb_pci_uncores[] = {
 613         [SNB_PCI_UNCORE_IMC]    = &snb_uncore_imc,
 614         NULL,
 615 };
 616
 617 static const struct pci_device_id snb_uncore_pci_ids[] = {
 618         { /* IMC */
 619                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SNB_IMC),
 620                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 621         },
 622         { /* end: all zeroes */ },
 623 };
 624
 625 static const struct pci_device_id ivb_uncore_pci_ids[] = {
 626         { /* IMC */
 627                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IVB_IMC),
 628                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 629         },
 630         { /* IMC */
 631                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IVB_E3_IMC),
 632                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 633         },
 634         { /* end: all zeroes */ },
 635 };
 636
 637 static const struct pci_device_id hsw_uncore_pci_ids[] = {
 638         { /* IMC */
 639                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_HSW_IMC),
 640                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 641         },
 642         { /* IMC */
 643                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_HSW_U_IMC),
 644                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 645         },
 646         { /* end: all zeroes */ },
 647 };
 648
 649 static const struct pci_device_id bdw_uncore_pci_ids[] = {
 650         { /* IMC */
 651                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BDW_IMC),
 652                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 653         },
 654         { /* end: all zeroes */ },
 655 };
 656
 657 static const struct pci_device_id skl_uncore_pci_ids[] = {
 658         { /* IMC */
 659                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_Y_IMC),
 660                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 661         },
 662         { /* IMC */
 663                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_U_IMC),
 664                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 665         },
 666         { /* IMC */
 667                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_HD_IMC),
 668                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 669         },
 670         { /* IMC */
 671                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_HQ_IMC),
 672                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 673         },
 674         { /* IMC */
 675                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_SD_IMC),
 676                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 677         },
 678         { /* IMC */
 679                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_SQ_IMC),
 680                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 681         },
 682         { /* IMC */
 683                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_E3_IMC),
 684                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 685         },
 686         { /* IMC */
 687                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_Y_IMC),
 688                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 689         },
 690         { /* IMC */
 691                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_U_IMC),
 692                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 693         },
 694         { /* IMC */
 695                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_UQ_IMC),
 696                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 697         },
 698         { /* IMC */
 699                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_SD_IMC),
 700                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 701         },
 702         { /* IMC */
 703                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_SQ_IMC),
 704                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 705         },
 706         { /* IMC */
 707                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_HQ_IMC),
 708                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 709         },
 710         { /* IMC */
 711                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_WQ_IMC),
 712                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 713         },
 714         { /* IMC */
 715                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_2U_IMC),
 716                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 717         },
 718         { /* IMC */
 719                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4U_IMC),
 720                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 721         },
 722         { /* IMC */
 723                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4H_IMC),
 724                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 725         },
 726         { /* IMC */
 727                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6H_IMC),
 728                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 729         },
 730         { /* IMC */
 731                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_2S_D_IMC),
 732                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 733         },
 734         { /* IMC */
 735                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_D_IMC),
 736                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 737         },
 738         { /* IMC */
 739                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_D_IMC),
 740                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 741         },
 742         { /* IMC */
 743                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_D_IMC),
 744                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 745         },
 746         { /* IMC */
 747                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_W_IMC),
 748                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 749         },
 750         { /* IMC */
 751                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_W_IMC),
 752                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 753         },
 754         { /* IMC */
 755                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_W_IMC),
 756                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 757         },
 758         { /* IMC */
 759                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_S_IMC),
 760                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 761         },
 762         { /* IMC */
 763                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_S_IMC),
 764                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 765         },
 766         { /* IMC */
 767                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_S_IMC),
 768                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 769         },
 770         { /* IMC */
 771                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_AML_YD_IMC),
 772                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 773         },
 774         { /* IMC */
 775                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_AML_YQ_IMC),
 776                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 777         },
 778         { /* IMC */
 779                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_UQ_IMC),
 780                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 781         },
 782         { /* IMC */
 783                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_4_UQ_IMC),
 784                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 785         },
 786         { /* IMC */
 787                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_UD_IMC),
 788                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 789         },
 790         { /* IMC */
 791                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_H1_IMC),
 792                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 793         },
 794         { /* IMC */
 795                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_H2_IMC),
 796                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 797         },
 798         { /* IMC */
 799                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_H3_IMC),
 800                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 801         },
 802         { /* IMC */
 803                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_U1_IMC),
 804                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 805         },
 806         { /* IMC */
 807                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_U2_IMC),
 808                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 809         },
 810         { /* IMC */
 811                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_U3_IMC),
 812                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 813         },
 814         { /* IMC */
 815                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S1_IMC),
 816                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 817         },
 818         { /* IMC */
 819                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S2_IMC),
 820                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 821         },
 822         { /* IMC */
 823                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S3_IMC),
 824                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 825         },
 826         { /* IMC */
 827                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S4_IMC),
 828                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 829         },
 830         { /* IMC */
 831                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CML_S5_IMC),
 832                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 833         },
 834         { /* end: all zeroes */ },
 835 };
 836
 837 static const struct pci_device_id icl_uncore_pci_ids[] = {
 838         { /* IMC */
 839                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICL_U_IMC),
 840                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 841         },
 842         { /* IMC */
 843                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICL_U2_IMC),
 844                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 845         },
 846         { /* end: all zeroes */ },
 847 };
 848
 849 static struct pci_driver snb_uncore_pci_driver = {
 850         .name           = "snb_uncore",
 851         .id_table       = snb_uncore_pci_ids,
 852 };
 853
 854 static struct pci_driver ivb_uncore_pci_driver = {
 855         .name           = "ivb_uncore",
 856         .id_table       = ivb_uncore_pci_ids,
 857 };
 858
 859 static struct pci_driver hsw_uncore_pci_driver = {
 860         .name           = "hsw_uncore",
 861         .id_table       = hsw_uncore_pci_ids,
 862 };
 863
 864 static struct pci_driver bdw_uncore_pci_driver = {
 865         .name           = "bdw_uncore",
 866         .id_table       = bdw_uncore_pci_ids,
 867 };
 868
 869 static struct pci_driver skl_uncore_pci_driver = {
 870         .name           = "skl_uncore",
 871         .id_table       = skl_uncore_pci_ids,
 872 };
 873
 874 static struct pci_driver icl_uncore_pci_driver = {
 875         .name           = "icl_uncore",
 876         .id_table       = icl_uncore_pci_ids,
 877 };
 878
 879 struct imc_uncore_pci_dev {
 880         __u32 pci_id;
 881         struct pci_driver *driver;
 882 };
 883 #define IMC_DEV(a, d) \
 884         { .pci_id = PCI_DEVICE_ID_INTEL_##a, .driver = (d) }
 885
 886 static const struct imc_uncore_pci_dev desktop_imc_pci_ids[] = {
 887         IMC_DEV(SNB_IMC, &snb_uncore_pci_driver),
 888         IMC_DEV(IVB_IMC, &ivb_uncore_pci_driver),    /* 3rd Gen Core processor */
 889         IMC_DEV(IVB_E3_IMC, &ivb_uncore_pci_driver), /* Xeon E3-1200 v2/3rd Gen Core processor */
 890         IMC_DEV(HSW_IMC, &hsw_uncore_pci_driver),    /* 4th Gen Core Processor */
 891         IMC_DEV(HSW_U_IMC, &hsw_uncore_pci_driver),  /* 4th Gen Core ULT Mobile Processor */
 892         IMC_DEV(BDW_IMC, &bdw_uncore_pci_driver),    /* 5th Gen Core U */
 893         IMC_DEV(SKL_Y_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core Y */
 894         IMC_DEV(SKL_U_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core U */
 895         IMC_DEV(SKL_HD_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core H Dual Core */
 896         IMC_DEV(SKL_HQ_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core H Quad Core */
 897         IMC_DEV(SKL_SD_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core S Dual Core */
 898         IMC_DEV(SKL_SQ_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core S Quad Core */
 899         IMC_DEV(SKL_E3_IMC, &skl_uncore_pci_driver),  /* Xeon E3 V5 Gen Core processor */
 900         IMC_DEV(KBL_Y_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core Y */
 901         IMC_DEV(KBL_U_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core U */
 902         IMC_DEV(KBL_UQ_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core U Quad Core */
 903         IMC_DEV(KBL_SD_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core S Dual Core */
 904         IMC_DEV(KBL_SQ_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core S Quad Core */
 905         IMC_DEV(KBL_HQ_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core H Quad Core */
 906         IMC_DEV(KBL_WQ_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core S 4 cores Work Station */
 907         IMC_DEV(CFL_2U_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core U 2 Cores */
 908         IMC_DEV(CFL_4U_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core U 4 Cores */
 909         IMC_DEV(CFL_4H_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core H 4 Cores */
 910         IMC_DEV(CFL_6H_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core H 6 Cores */
 911         IMC_DEV(CFL_2S_D_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 2 Cores Desktop */
 912         IMC_DEV(CFL_4S_D_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 4 Cores Desktop */
 913         IMC_DEV(CFL_6S_D_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 6 Cores Desktop */
 914         IMC_DEV(CFL_8S_D_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 8 Cores Desktop */
 915         IMC_DEV(CFL_4S_W_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 4 Cores Work Station */
 916         IMC_DEV(CFL_6S_W_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 6 Cores Work Station */
 917         IMC_DEV(CFL_8S_W_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 8 Cores Work Station */
 918         IMC_DEV(CFL_4S_S_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 4 Cores Server */
 919         IMC_DEV(CFL_6S_S_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 6 Cores Server */
 920         IMC_DEV(CFL_8S_S_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 8 Cores Server */
 921         IMC_DEV(AML_YD_IMC, &skl_uncore_pci_driver),    /* 8th Gen Core Y Mobile Dual Core */
 922         IMC_DEV(AML_YQ_IMC, &skl_uncore_pci_driver),    /* 8th Gen Core Y Mobile Quad Core */
 923         IMC_DEV(WHL_UQ_IMC, &skl_uncore_pci_driver),    /* 8th Gen Core U Mobile Quad Core */
 924         IMC_DEV(WHL_4_UQ_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core U Mobile Quad Core */
 925         IMC_DEV(WHL_UD_IMC, &skl_uncore_pci_driver),    /* 8th Gen Core U Mobile Dual Core */
 926         IMC_DEV(CML_H1_IMC, &skl_uncore_pci_driver),
 927         IMC_DEV(CML_H2_IMC, &skl_uncore_pci_driver),
 928         IMC_DEV(CML_H3_IMC, &skl_uncore_pci_driver),
 929         IMC_DEV(CML_U1_IMC, &skl_uncore_pci_driver),
 930         IMC_DEV(CML_U2_IMC, &skl_uncore_pci_driver),
 931         IMC_DEV(CML_U3_IMC, &skl_uncore_pci_driver),
 932         IMC_DEV(CML_S1_IMC, &skl_uncore_pci_driver),
 933         IMC_DEV(CML_S2_IMC, &skl_uncore_pci_driver),
 934         IMC_DEV(CML_S3_IMC, &skl_uncore_pci_driver),
 935         IMC_DEV(CML_S4_IMC, &skl_uncore_pci_driver),
 936         IMC_DEV(CML_S5_IMC, &skl_uncore_pci_driver),
 937         IMC_DEV(ICL_U_IMC, &icl_uncore_pci_driver),     /* 10th Gen Core Mobile */
 938         IMC_DEV(ICL_U2_IMC, &icl_uncore_pci_driver),    /* 10th Gen Core Mobile */
 939         {  /* end marker */ }
 940 };
 941
 942
 943 #define for_each_imc_pci_id(x, t) \
 944         for (x = (t); (x)->pci_id; x++)
 945
 946 static struct pci_driver *imc_uncore_find_dev(void)
 947 {
 948         const struct imc_uncore_pci_dev *p;
 949         int ret;
 950
 951         for_each_imc_pci_id(p, desktop_imc_pci_ids) {
 952                 ret = snb_pci2phy_map_init(p->pci_id);
 953                 if (ret == 0)
 954                         return p->driver;
 955         }
 956         return NULL;
 957 }
 958
 959 static int imc_uncore_pci_init(void)
 960 {
 961         struct pci_driver *imc_drv = imc_uncore_find_dev();
 962
 963         if (!imc_drv)
 964                 return -ENODEV;
 965
 966         uncore_pci_uncores = snb_pci_uncores;
 967         uncore_pci_driver = imc_drv;
 968
 969         return 0;
 970 }
 971
 972 int snb_uncore_pci_init(void)
 973 {
 974         return imc_uncore_pci_init();
 975 }
 976
 977 int ivb_uncore_pci_init(void)
 978 {
 979         return imc_uncore_pci_init();
 980 }
 981 int hsw_uncore_pci_init(void)
 982 {
 983         return imc_uncore_pci_init();
 984 }
 985
 986 int bdw_uncore_pci_init(void)
 987 {
 988         return imc_uncore_pci_init();
 989 }
 990
 991 int skl_uncore_pci_init(void)
 992 {
 993         return imc_uncore_pci_init();
 994 }
 995
 996 /* end of Sandy Bridge uncore support */
 997
 998 /* Nehalem uncore support */
 999 static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box)
1000 {
1001         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0);
1002 }
1003
1004 static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box)
1005 {
1006         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
1007 }
1008
1009 static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1010 {
1011         struct hw_perf_event *hwc = &event->hw;
1012
1013         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
1014                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
1015         else
1016                 wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
1017 }
1018
1019 static struct attribute *nhm_uncore_formats_attr[] = {
1020         &format_attr_event.attr,
1021         &format_attr_umask.attr,
1022         &format_attr_edge.attr,
1023         &format_attr_inv.attr,
1024         &format_attr_cmask8.attr,
1025         NULL,
1026 };
1027
1028 static const struct attribute_group nhm_uncore_format_group = {
1029         .name = "format",
1030         .attrs = nhm_uncore_formats_attr,
1031 };
1032
1033 static struct uncore_event_desc nhm_uncore_events[] = {
1034         INTEL_UNCORE_EVENT_DESC(clockticks,                "event=0xff,umask=0x00"),
1035         INTEL_UNCORE_EVENT_DESC(qmc_writes_full_any,       "event=0x2f,umask=0x0f"),
1036         INTEL_UNCORE_EVENT_DESC(qmc_normal_reads_any,      "event=0x2c,umask=0x0f"),
1037         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_reads,     "event=0x20,umask=0x01"),
1038         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_writes,    "event=0x20,umask=0x02"),
1039         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_reads,  "event=0x20,umask=0x04"),
1040         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_writes, "event=0x20,umask=0x08"),
1041         INTEL_UNCORE_EVENT_DESC(qhl_request_local_reads,   "event=0x20,umask=0x10"),
1042         INTEL_UNCORE_EVENT_DESC(qhl_request_local_writes,  "event=0x20,umask=0x20"),
1043         { /* end: all zeroes */ },
1044 };
1045
1046 static struct intel_uncore_ops nhm_uncore_msr_ops = {
1047         .disable_box    = nhm_uncore_msr_disable_box,
1048         .enable_box     = nhm_uncore_msr_enable_box,
1049         .disable_event  = snb_uncore_msr_disable_event,
1050         .enable_event   = nhm_uncore_msr_enable_event,
1051         .read_counter   = uncore_msr_read_counter,
1052 };
1053
1054 static struct intel_uncore_type nhm_uncore = {
1055         .name           = "",
1056         .num_counters   = 8,
1057         .num_boxes      = 1,
1058         .perf_ctr_bits  = 48,
1059         .fixed_ctr_bits = 48,
1060         .event_ctl      = NHM_UNC_PERFEVTSEL0,
1061         .perf_ctr       = NHM_UNC_UNCORE_PMC0,
1062         .fixed_ctr      = NHM_UNC_FIXED_CTR,
1063         .fixed_ctl      = NHM_UNC_FIXED_CTR_CTRL,
1064         .event_mask     = NHM_UNC_RAW_EVENT_MASK,
1065         .event_descs    = nhm_uncore_events,
1066         .ops            = &nhm_uncore_msr_ops,
1067         .format_group   = &nhm_uncore_format_group,
1068 };
1069
1070 static struct intel_uncore_type *nhm_msr_uncores[] = {
1071         &nhm_uncore,
1072         NULL,
1073 };
1074
1075 void nhm_uncore_cpu_init(void)
1076 {
1077         uncore_msr_uncores = nhm_msr_uncores;
1078 }
1079
1080 /* end of Nehalem uncore support */
1081
1082 /* Tiger Lake MMIO uncore support */
1083
1084 static const struct pci_device_id tgl_uncore_pci_ids[] = {
1085         { /* IMC */
1086                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U1_IMC),
1087                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1088         },
1089         { /* IMC */
1090                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U2_IMC),
1091                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1092         },
1093         { /* IMC */
1094                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U3_IMC),
1095                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1096         },
1097         { /* IMC */
1098                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U4_IMC),
1099                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1100         },
1101         { /* IMC */
1102                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_H_IMC),
1103                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1104         },
1105         { /* end: all zeroes */ }
1106 };
1107
1108 enum perf_tgl_uncore_imc_freerunning_types {
1109         TGL_MMIO_UNCORE_IMC_DATA_TOTAL,
1110         TGL_MMIO_UNCORE_IMC_DATA_READ,
1111         TGL_MMIO_UNCORE_IMC_DATA_WRITE,
1112         TGL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX
1113 };
1114
1115 static struct freerunning_counters tgl_l_uncore_imc_freerunning[] = {
1116         [TGL_MMIO_UNCORE_IMC_DATA_TOTAL]        = { 0x5040, 0x0, 0x0, 1, 64 },
1117         [TGL_MMIO_UNCORE_IMC_DATA_READ]         = { 0x5058, 0x0, 0x0, 1, 64 },
1118         [TGL_MMIO_UNCORE_IMC_DATA_WRITE]        = { 0x50A0, 0x0, 0x0, 1, 64 },
1119 };
1120
1121 static struct freerunning_counters tgl_uncore_imc_freerunning[] = {
1122         [TGL_MMIO_UNCORE_IMC_DATA_TOTAL]        = { 0xd840, 0x0, 0x0, 1, 64 },
1123         [TGL_MMIO_UNCORE_IMC_DATA_READ]         = { 0xd858, 0x0, 0x0, 1, 64 },
1124         [TGL_MMIO_UNCORE_IMC_DATA_WRITE]        = { 0xd8A0, 0x0, 0x0, 1, 64 },
1125 };
1126
1127 static struct uncore_event_desc tgl_uncore_imc_events[] = {
1128         INTEL_UNCORE_EVENT_DESC(data_total,         "event=0xff,umask=0x10"),
1129         INTEL_UNCORE_EVENT_DESC(data_total.scale,   "6.103515625e-5"),
1130         INTEL_UNCORE_EVENT_DESC(data_total.unit,    "MiB"),
1131
1132         INTEL_UNCORE_EVENT_DESC(data_read,         "event=0xff,umask=0x20"),
1133         INTEL_UNCORE_EVENT_DESC(data_read.scale,   "6.103515625e-5"),
1134         INTEL_UNCORE_EVENT_DESC(data_read.unit,    "MiB"),
1135
1136         INTEL_UNCORE_EVENT_DESC(data_write,        "event=0xff,umask=0x30"),
1137         INTEL_UNCORE_EVENT_DESC(data_write.scale,  "6.103515625e-5"),
1138         INTEL_UNCORE_EVENT_DESC(data_write.unit,   "MiB"),
1139
1140         { /* end: all zeroes */ }
1141 };
1142
1143 static struct pci_dev *tgl_uncore_get_mc_dev(void)
1144 {
1145         const struct pci_device_id *ids = tgl_uncore_pci_ids;
1146         struct pci_dev *mc_dev = NULL;
1147
1148         while (ids && ids->vendor) {
1149                 mc_dev = pci_get_device(PCI_VENDOR_ID_INTEL, ids->device, NULL);
1150                 if (mc_dev)
1151                         return mc_dev;
1152                 ids++;
1153         }
1154
1155         return mc_dev;
1156 }
1157
1158 #define TGL_UNCORE_MMIO_IMC_MEM_OFFSET          0x10000
1159 #define TGL_UNCORE_PCI_IMC_MAP_SIZE             0xe000
1160
1161 static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
1162 {
1163         struct pci_dev *pdev = tgl_uncore_get_mc_dev();
1164         struct intel_uncore_pmu *pmu = box->pmu;
1165         struct intel_uncore_type *type = pmu->type;
1166         resource_size_t addr;
1167         u32 mch_bar;
1168
1169         if (!pdev) {
1170                 pr_warn("perf uncore: Cannot find matched IMC device.\n");
1171                 return;
1172         }
1173
1174         pci_read_config_dword(pdev, SNB_UNCORE_PCI_IMC_BAR_OFFSET, &mch_bar);
1175         /* MCHBAR is disabled */
1176         if (!(mch_bar & BIT(0))) {
1177                 pr_warn("perf uncore: MCHBAR is disabled. Failed to map IMC free-running counters.\n");
1178                 return;
1179         }
1180         mch_bar &= ~BIT(0);
1181         addr = (resource_size_t)(mch_bar + TGL_UNCORE_MMIO_IMC_MEM_OFFSET * pmu->pmu_idx);
1182
1183 #ifdef CONFIG_PHYS_ADDR_T_64BIT
1184         pci_read_config_dword(pdev, SNB_UNCORE_PCI_IMC_BAR_OFFSET + 4, &mch_bar);
1185         addr |= ((resource_size_t)mch_bar << 32);
1186 #endif
1187
1188         box->io_addr = ioremap(addr, type->mmio_map_size);
1189         if (!box->io_addr)
1190                 pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name);
1191 }
1192
1193 static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = {
1194         .init_box       = tgl_uncore_imc_freerunning_init_box,
1195         .exit_box       = uncore_mmio_exit_box,
1196         .read_counter   = uncore_mmio_read_counter,
1197         .hw_config      = uncore_freerunning_hw_config,
1198 };
1199
1200 static struct attribute *tgl_uncore_imc_formats_attr[] = {
1201         &format_attr_event.attr,
1202         &format_attr_umask.attr,
1203         NULL
1204 };
1205
1206 static const struct attribute_group tgl_uncore_imc_format_group = {
1207         .name = "format",
1208         .attrs = tgl_uncore_imc_formats_attr,
1209 };
1210
1211 static struct intel_uncore_type tgl_uncore_imc_free_running = {
1212         .name                   = "imc_free_running",
1213         .num_counters           = 3,
1214         .num_boxes              = 2,
1215         .num_freerunning_types  = TGL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX,
1216         .mmio_map_size          = TGL_UNCORE_PCI_IMC_MAP_SIZE,
1217         .freerunning            = tgl_uncore_imc_freerunning,
1218         .ops                    = &tgl_uncore_imc_freerunning_ops,
1219         .event_descs            = tgl_uncore_imc_events,
1220         .format_group           = &tgl_uncore_imc_format_group,
1221 };
1222
1223 static struct intel_uncore_type *tgl_mmio_uncores[] = {
1224         &tgl_uncore_imc_free_running,
1225         NULL
1226 };
1227
1228 void tgl_l_uncore_mmio_init(void)
1229 {
1230         tgl_uncore_imc_free_running.freerunning = tgl_l_uncore_imc_freerunning;
1231         uncore_mmio_uncores = tgl_mmio_uncores;
1232 }
1233
1234 void tgl_uncore_mmio_init(void)
1235 {
1236         uncore_mmio_uncores = tgl_mmio_uncores;
1237 }
1238
1239 /* end of Tiger Lake MMIO uncore support */