arch/x86/events/intel/uncore_snb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Nehalem/SandBridge/Haswell/Broadwell/Skylake uncore support */
   3 #include "uncore.h"
   4
   5 /* Uncore IMC PCI IDs */
   6 #define PCI_DEVICE_ID_INTEL_SNB_IMC             0x0100
   7 #define PCI_DEVICE_ID_INTEL_IVB_IMC             0x0154
   8 #define PCI_DEVICE_ID_INTEL_IVB_E3_IMC          0x0150
   9 #define PCI_DEVICE_ID_INTEL_HSW_IMC             0x0c00
  10 #define PCI_DEVICE_ID_INTEL_HSW_U_IMC           0x0a04
  11 #define PCI_DEVICE_ID_INTEL_BDW_IMC             0x1604
  12 #define PCI_DEVICE_ID_INTEL_SKL_U_IMC           0x1904
  13 #define PCI_DEVICE_ID_INTEL_SKL_Y_IMC           0x190c
  14 #define PCI_DEVICE_ID_INTEL_SKL_HD_IMC          0x1900
  15 #define PCI_DEVICE_ID_INTEL_SKL_HQ_IMC          0x1910
  16 #define PCI_DEVICE_ID_INTEL_SKL_SD_IMC          0x190f
  17 #define PCI_DEVICE_ID_INTEL_SKL_SQ_IMC          0x191f
  18 #define PCI_DEVICE_ID_INTEL_SKL_E3_IMC          0x1918
  19 #define PCI_DEVICE_ID_INTEL_KBL_Y_IMC           0x590c
  20 #define PCI_DEVICE_ID_INTEL_KBL_U_IMC           0x5904
  21 #define PCI_DEVICE_ID_INTEL_KBL_UQ_IMC          0x5914
  22 #define PCI_DEVICE_ID_INTEL_KBL_SD_IMC          0x590f
  23 #define PCI_DEVICE_ID_INTEL_KBL_SQ_IMC          0x591f
  24 #define PCI_DEVICE_ID_INTEL_KBL_HQ_IMC          0x5910
  25 #define PCI_DEVICE_ID_INTEL_KBL_WQ_IMC          0x5918
  26 #define PCI_DEVICE_ID_INTEL_CFL_2U_IMC          0x3ecc
  27 #define PCI_DEVICE_ID_INTEL_CFL_4U_IMC          0x3ed0
  28 #define PCI_DEVICE_ID_INTEL_CFL_4H_IMC          0x3e10
  29 #define PCI_DEVICE_ID_INTEL_CFL_6H_IMC          0x3ec4
  30 #define PCI_DEVICE_ID_INTEL_CFL_2S_D_IMC        0x3e0f
  31 #define PCI_DEVICE_ID_INTEL_CFL_4S_D_IMC        0x3e1f
  32 #define PCI_DEVICE_ID_INTEL_CFL_6S_D_IMC        0x3ec2
  33 #define PCI_DEVICE_ID_INTEL_CFL_8S_D_IMC        0x3e30
  34 #define PCI_DEVICE_ID_INTEL_CFL_4S_W_IMC        0x3e18
  35 #define PCI_DEVICE_ID_INTEL_CFL_6S_W_IMC        0x3ec6
  36 #define PCI_DEVICE_ID_INTEL_CFL_8S_W_IMC        0x3e31
  37 #define PCI_DEVICE_ID_INTEL_CFL_4S_S_IMC        0x3e33
  38 #define PCI_DEVICE_ID_INTEL_CFL_6S_S_IMC        0x3eca
  39 #define PCI_DEVICE_ID_INTEL_CFL_8S_S_IMC        0x3e32
  40 #define PCI_DEVICE_ID_INTEL_AML_YD_IMC          0x590c
  41 #define PCI_DEVICE_ID_INTEL_AML_YQ_IMC          0x590d
  42 #define PCI_DEVICE_ID_INTEL_WHL_UQ_IMC          0x3ed0
  43 #define PCI_DEVICE_ID_INTEL_WHL_4_UQ_IMC        0x3e34
  44 #define PCI_DEVICE_ID_INTEL_WHL_UD_IMC          0x3e35
  45 #define PCI_DEVICE_ID_INTEL_ICL_U_IMC           0x8a02
  46 #define PCI_DEVICE_ID_INTEL_ICL_U2_IMC          0x8a12
  47 #define PCI_DEVICE_ID_INTEL_TGL_U1_IMC          0x9a02
  48 #define PCI_DEVICE_ID_INTEL_TGL_U2_IMC          0x9a04
  49 #define PCI_DEVICE_ID_INTEL_TGL_U3_IMC          0x9a12
  50 #define PCI_DEVICE_ID_INTEL_TGL_U4_IMC          0x9a14
  51 #define PCI_DEVICE_ID_INTEL_TGL_H_IMC           0x9a36
  52
  53
  54 /* SNB event control */
  55 #define SNB_UNC_CTL_EV_SEL_MASK                 0x000000ff
  56 #define SNB_UNC_CTL_UMASK_MASK                  0x0000ff00
  57 #define SNB_UNC_CTL_EDGE_DET                    (1 << 18)
  58 #define SNB_UNC_CTL_EN                          (1 << 22)
  59 #define SNB_UNC_CTL_INVERT                      (1 << 23)
  60 #define SNB_UNC_CTL_CMASK_MASK                  0x1f000000
  61 #define NHM_UNC_CTL_CMASK_MASK                  0xff000000
  62 #define NHM_UNC_FIXED_CTR_CTL_EN                (1 << 0)
  63
  64 #define SNB_UNC_RAW_EVENT_MASK                  (SNB_UNC_CTL_EV_SEL_MASK | \
  65                                                  SNB_UNC_CTL_UMASK_MASK | \
  66                                                  SNB_UNC_CTL_EDGE_DET | \
  67                                                  SNB_UNC_CTL_INVERT | \
  68                                                  SNB_UNC_CTL_CMASK_MASK)
  69
  70 #define NHM_UNC_RAW_EVENT_MASK                  (SNB_UNC_CTL_EV_SEL_MASK | \
  71                                                  SNB_UNC_CTL_UMASK_MASK | \
  72                                                  SNB_UNC_CTL_EDGE_DET | \
  73                                                  SNB_UNC_CTL_INVERT | \
  74                                                  NHM_UNC_CTL_CMASK_MASK)
  75
  76 /* SNB global control register */
  77 #define SNB_UNC_PERF_GLOBAL_CTL                 0x391
  78 #define SNB_UNC_FIXED_CTR_CTRL                  0x394
  79 #define SNB_UNC_FIXED_CTR                       0x395
  80
  81 /* SNB uncore global control */
  82 #define SNB_UNC_GLOBAL_CTL_CORE_ALL             ((1 << 4) - 1)
  83 #define SNB_UNC_GLOBAL_CTL_EN                   (1 << 29)
  84
  85 /* SNB Cbo register */
  86 #define SNB_UNC_CBO_0_PERFEVTSEL0               0x700
  87 #define SNB_UNC_CBO_0_PER_CTR0                  0x706
  88 #define SNB_UNC_CBO_MSR_OFFSET                  0x10
  89
  90 /* SNB ARB register */
  91 #define SNB_UNC_ARB_PER_CTR0                    0x3b0
  92 #define SNB_UNC_ARB_PERFEVTSEL0                 0x3b2
  93 #define SNB_UNC_ARB_MSR_OFFSET                  0x10
  94
  95 /* NHM global control register */
  96 #define NHM_UNC_PERF_GLOBAL_CTL                 0x391
  97 #define NHM_UNC_FIXED_CTR                       0x394
  98 #define NHM_UNC_FIXED_CTR_CTRL                  0x395
  99
 100 /* NHM uncore global control */
 101 #define NHM_UNC_GLOBAL_CTL_EN_PC_ALL            ((1ULL << 8) - 1)
 102 #define NHM_UNC_GLOBAL_CTL_EN_FC                (1ULL << 32)
 103
 104 /* NHM uncore register */
 105 #define NHM_UNC_PERFEVTSEL0                     0x3c0
 106 #define NHM_UNC_UNCORE_PMC0                     0x3b0
 107
 108 /* SKL uncore global control */
 109 #define SKL_UNC_PERF_GLOBAL_CTL                 0xe01
 110 #define SKL_UNC_GLOBAL_CTL_CORE_ALL             ((1 << 5) - 1)
 111
 112 /* ICL Cbo register */
 113 #define ICL_UNC_CBO_CONFIG                      0x396
 114 #define ICL_UNC_NUM_CBO_MASK                    0xf
 115 #define ICL_UNC_CBO_0_PER_CTR0                  0x702
 116 #define ICL_UNC_CBO_MSR_OFFSET                  0x8
 117
 118 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
 119 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
 120 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
 121 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
 122 DEFINE_UNCORE_FORMAT_ATTR(cmask5, cmask, "config:24-28");
 123 DEFINE_UNCORE_FORMAT_ATTR(cmask8, cmask, "config:24-31");
 124
 125 /* Sandy Bridge uncore support */
 126 static void snb_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 127 {
 128         struct hw_perf_event *hwc = &event->hw;
 129
 130         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
 131                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 132         else
 133                 wrmsrl(hwc->config_base, SNB_UNC_CTL_EN);
 134 }
 135
 136 static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 137 {
 138         wrmsrl(event->hw.config_base, 0);
 139 }
 140
 141 static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
 142 {
 143         if (box->pmu->pmu_idx == 0) {
 144                 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
 145                         SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 146         }
 147 }
 148
 149 static void snb_uncore_msr_enable_box(struct intel_uncore_box *box)
 150 {
 151         wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
 152                 SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
 153 }
 154
 155 static void snb_uncore_msr_exit_box(struct intel_uncore_box *box)
 156 {
 157         if (box->pmu->pmu_idx == 0)
 158                 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL, 0);
 159 }
 160
 161 static struct uncore_event_desc snb_uncore_events[] = {
 162         INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0x00"),
 163         { /* end: all zeroes */ },
 164 };
 165
 166 static struct attribute *snb_uncore_formats_attr[] = {
 167         &format_attr_event.attr,
 168         &format_attr_umask.attr,
 169         &format_attr_edge.attr,
 170         &format_attr_inv.attr,
 171         &format_attr_cmask5.attr,
 172         NULL,
 173 };
 174
 175 static const struct attribute_group snb_uncore_format_group = {
 176         .name           = "format",
 177         .attrs          = snb_uncore_formats_attr,
 178 };
 179
 180 static struct intel_uncore_ops snb_uncore_msr_ops = {
 181         .init_box       = snb_uncore_msr_init_box,
 182         .enable_box     = snb_uncore_msr_enable_box,
 183         .exit_box       = snb_uncore_msr_exit_box,
 184         .disable_event  = snb_uncore_msr_disable_event,
 185         .enable_event   = snb_uncore_msr_enable_event,
 186         .read_counter   = uncore_msr_read_counter,
 187 };
 188
 189 static struct event_constraint snb_uncore_arb_constraints[] = {
 190         UNCORE_EVENT_CONSTRAINT(0x80, 0x1),
 191         UNCORE_EVENT_CONSTRAINT(0x83, 0x1),
 192         EVENT_CONSTRAINT_END
 193 };
 194
 195 static struct intel_uncore_type snb_uncore_cbox = {
 196         .name           = "cbox",
 197         .num_counters   = 2,
 198         .num_boxes      = 4,
 199         .perf_ctr_bits  = 44,
 200         .fixed_ctr_bits = 48,
 201         .perf_ctr       = SNB_UNC_CBO_0_PER_CTR0,
 202         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
 203         .fixed_ctr      = SNB_UNC_FIXED_CTR,
 204         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
 205         .single_fixed   = 1,
 206         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 207         .msr_offset     = SNB_UNC_CBO_MSR_OFFSET,
 208         .ops            = &snb_uncore_msr_ops,
 209         .format_group   = &snb_uncore_format_group,
 210         .event_descs    = snb_uncore_events,
 211 };
 212
 213 static struct intel_uncore_type snb_uncore_arb = {
 214         .name           = "arb",
 215         .num_counters   = 2,
 216         .num_boxes      = 1,
 217         .perf_ctr_bits  = 44,
 218         .perf_ctr       = SNB_UNC_ARB_PER_CTR0,
 219         .event_ctl      = SNB_UNC_ARB_PERFEVTSEL0,
 220         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 221         .msr_offset     = SNB_UNC_ARB_MSR_OFFSET,
 222         .constraints    = snb_uncore_arb_constraints,
 223         .ops            = &snb_uncore_msr_ops,
 224         .format_group   = &snb_uncore_format_group,
 225 };
 226
 227 static struct intel_uncore_type *snb_msr_uncores[] = {
 228         &snb_uncore_cbox,
 229         &snb_uncore_arb,
 230         NULL,
 231 };
 232
 233 void snb_uncore_cpu_init(void)
 234 {
 235         uncore_msr_uncores = snb_msr_uncores;
 236         if (snb_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
 237                 snb_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
 238 }
 239
 240 static void skl_uncore_msr_init_box(struct intel_uncore_box *box)
 241 {
 242         if (box->pmu->pmu_idx == 0) {
 243                 wrmsrl(SKL_UNC_PERF_GLOBAL_CTL,
 244                         SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL);
 245         }
 246
 247         /* The 8th CBOX has different MSR space */
 248         if (box->pmu->pmu_idx == 7)
 249                 __set_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags);
 250 }
 251
 252 static void skl_uncore_msr_enable_box(struct intel_uncore_box *box)
 253 {
 254         wrmsrl(SKL_UNC_PERF_GLOBAL_CTL,
 255                 SNB_UNC_GLOBAL_CTL_EN | SKL_UNC_GLOBAL_CTL_CORE_ALL);
 256 }
 257
 258 static void skl_uncore_msr_exit_box(struct intel_uncore_box *box)
 259 {
 260         if (box->pmu->pmu_idx == 0)
 261                 wrmsrl(SKL_UNC_PERF_GLOBAL_CTL, 0);
 262 }
 263
 264 static struct intel_uncore_ops skl_uncore_msr_ops = {
 265         .init_box       = skl_uncore_msr_init_box,
 266         .enable_box     = skl_uncore_msr_enable_box,
 267         .exit_box       = skl_uncore_msr_exit_box,
 268         .disable_event  = snb_uncore_msr_disable_event,
 269         .enable_event   = snb_uncore_msr_enable_event,
 270         .read_counter   = uncore_msr_read_counter,
 271 };
 272
 273 static struct intel_uncore_type skl_uncore_cbox = {
 274         .name           = "cbox",
 275         .num_counters   = 4,
 276         .num_boxes      = 8,
 277         .perf_ctr_bits  = 44,
 278         .fixed_ctr_bits = 48,
 279         .perf_ctr       = SNB_UNC_CBO_0_PER_CTR0,
 280         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
 281         .fixed_ctr      = SNB_UNC_FIXED_CTR,
 282         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
 283         .single_fixed   = 1,
 284         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 285         .msr_offset     = SNB_UNC_CBO_MSR_OFFSET,
 286         .ops            = &skl_uncore_msr_ops,
 287         .format_group   = &snb_uncore_format_group,
 288         .event_descs    = snb_uncore_events,
 289 };
 290
 291 static struct intel_uncore_type *skl_msr_uncores[] = {
 292         &skl_uncore_cbox,
 293         &snb_uncore_arb,
 294         NULL,
 295 };
 296
 297 void skl_uncore_cpu_init(void)
 298 {
 299         uncore_msr_uncores = skl_msr_uncores;
 300         if (skl_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
 301                 skl_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
 302         snb_uncore_arb.ops = &skl_uncore_msr_ops;
 303 }
 304
 305 static struct intel_uncore_type icl_uncore_cbox = {
 306         .name           = "cbox",
 307         .num_counters   = 4,
 308         .perf_ctr_bits  = 44,
 309         .perf_ctr       = ICL_UNC_CBO_0_PER_CTR0,
 310         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
 311         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
 312         .msr_offset     = ICL_UNC_CBO_MSR_OFFSET,
 313         .ops            = &skl_uncore_msr_ops,
 314         .format_group   = &snb_uncore_format_group,
 315 };
 316
 317 static struct uncore_event_desc icl_uncore_events[] = {
 318         INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff"),
 319         { /* end: all zeroes */ },
 320 };
 321
 322 static struct attribute *icl_uncore_clock_formats_attr[] = {
 323         &format_attr_event.attr,
 324         NULL,
 325 };
 326
 327 static struct attribute_group icl_uncore_clock_format_group = {
 328         .name = "format",
 329         .attrs = icl_uncore_clock_formats_attr,
 330 };
 331
 332 static struct intel_uncore_type icl_uncore_clockbox = {
 333         .name           = "clock",
 334         .num_counters   = 1,
 335         .num_boxes      = 1,
 336         .fixed_ctr_bits = 48,
 337         .fixed_ctr      = SNB_UNC_FIXED_CTR,
 338         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
 339         .single_fixed   = 1,
 340         .event_mask     = SNB_UNC_CTL_EV_SEL_MASK,
 341         .format_group   = &icl_uncore_clock_format_group,
 342         .ops            = &skl_uncore_msr_ops,
 343         .event_descs    = icl_uncore_events,
 344 };
 345
 346 static struct intel_uncore_type *icl_msr_uncores[] = {
 347         &icl_uncore_cbox,
 348         &snb_uncore_arb,
 349         &icl_uncore_clockbox,
 350         NULL,
 351 };
 352
 353 static int icl_get_cbox_num(void)
 354 {
 355         u64 num_boxes;
 356
 357         rdmsrl(ICL_UNC_CBO_CONFIG, num_boxes);
 358
 359         return num_boxes & ICL_UNC_NUM_CBO_MASK;
 360 }
 361
 362 void icl_uncore_cpu_init(void)
 363 {
 364         uncore_msr_uncores = icl_msr_uncores;
 365         icl_uncore_cbox.num_boxes = icl_get_cbox_num();
 366         snb_uncore_arb.ops = &skl_uncore_msr_ops;
 367 }
 368
 369 enum {
 370         SNB_PCI_UNCORE_IMC,
 371 };
 372
 373 static struct uncore_event_desc snb_uncore_imc_events[] = {
 374         INTEL_UNCORE_EVENT_DESC(data_reads,  "event=0x01"),
 375         INTEL_UNCORE_EVENT_DESC(data_reads.scale, "6.103515625e-5"),
 376         INTEL_UNCORE_EVENT_DESC(data_reads.unit, "MiB"),
 377
 378         INTEL_UNCORE_EVENT_DESC(data_writes, "event=0x02"),
 379         INTEL_UNCORE_EVENT_DESC(data_writes.scale, "6.103515625e-5"),
 380         INTEL_UNCORE_EVENT_DESC(data_writes.unit, "MiB"),
 381
 382         { /* end: all zeroes */ },
 383 };
 384
 385 #define SNB_UNCORE_PCI_IMC_EVENT_MASK           0xff
 386 #define SNB_UNCORE_PCI_IMC_BAR_OFFSET           0x48
 387
 388 /* page size multiple covering all config regs */
 389 #define SNB_UNCORE_PCI_IMC_MAP_SIZE             0x6000
 390
 391 #define SNB_UNCORE_PCI_IMC_DATA_READS           0x1
 392 #define SNB_UNCORE_PCI_IMC_DATA_READS_BASE      0x5050
 393 #define SNB_UNCORE_PCI_IMC_DATA_WRITES          0x2
 394 #define SNB_UNCORE_PCI_IMC_DATA_WRITES_BASE     0x5054
 395 #define SNB_UNCORE_PCI_IMC_CTR_BASE             SNB_UNCORE_PCI_IMC_DATA_READS_BASE
 396
 397 enum perf_snb_uncore_imc_freerunning_types {
 398         SNB_PCI_UNCORE_IMC_DATA         = 0,
 399         SNB_PCI_UNCORE_IMC_FREERUNNING_TYPE_MAX,
 400 };
 401
 402 static struct freerunning_counters snb_uncore_imc_freerunning[] = {
 403         [SNB_PCI_UNCORE_IMC_DATA]     = { SNB_UNCORE_PCI_IMC_DATA_READS_BASE, 0x4, 0x0, 2, 32 },
 404 };
 405
 406 static struct attribute *snb_uncore_imc_formats_attr[] = {
 407         &format_attr_event.attr,
 408         NULL,
 409 };
 410
 411 static const struct attribute_group snb_uncore_imc_format_group = {
 412         .name = "format",
 413         .attrs = snb_uncore_imc_formats_attr,
 414 };
 415
 416 static void snb_uncore_imc_init_box(struct intel_uncore_box *box)
 417 {
 418         struct pci_dev *pdev = box->pci_dev;
 419         int where = SNB_UNCORE_PCI_IMC_BAR_OFFSET;
 420         resource_size_t addr;
 421         u32 pci_dword;
 422
 423         pci_read_config_dword(pdev, where, &pci_dword);
 424         addr = pci_dword;
 425
 426 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 427         pci_read_config_dword(pdev, where + 4, &pci_dword);
 428         addr |= ((resource_size_t)pci_dword << 32);
 429 #endif
 430
 431         addr &= ~(PAGE_SIZE - 1);
 432
 433         box->io_addr = ioremap(addr, SNB_UNCORE_PCI_IMC_MAP_SIZE);
 434         box->hrtimer_duration = UNCORE_SNB_IMC_HRTIMER_INTERVAL;
 435 }
 436
 437 static void snb_uncore_imc_enable_box(struct intel_uncore_box *box)
 438 {}
 439
 440 static void snb_uncore_imc_disable_box(struct intel_uncore_box *box)
 441 {}
 442
 443 static void snb_uncore_imc_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 444 {}
 445
 446 static void snb_uncore_imc_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 447 {}
 448
 449 /*
 450  * Keep the custom event_init() function compatible with old event
 451  * encoding for free running counters.
 452  */
 453 static int snb_uncore_imc_event_init(struct perf_event *event)
 454 {
 455         struct intel_uncore_pmu *pmu;
 456         struct intel_uncore_box *box;
 457         struct hw_perf_event *hwc = &event->hw;
 458         u64 cfg = event->attr.config & SNB_UNCORE_PCI_IMC_EVENT_MASK;
 459         int idx, base;
 460
 461         if (event->attr.type != event->pmu->type)
 462                 return -ENOENT;
 463
 464         pmu = uncore_event_to_pmu(event);
 465         /* no device found for this pmu */
 466         if (pmu->func_id < 0)
 467                 return -ENOENT;
 468
 469         /* Sampling not supported yet */
 470         if (hwc->sample_period)
 471                 return -EINVAL;
 472
 473         /* unsupported modes and filters */
 474         if (event->attr.sample_period) /* no sampling */
 475                 return -EINVAL;
 476
 477         /*
 478          * Place all uncore events for a particular physical package
 479          * onto a single cpu
 480          */
 481         if (event->cpu < 0)
 482                 return -EINVAL;
 483
 484         /* check only supported bits are set */
 485         if (event->attr.config & ~SNB_UNCORE_PCI_IMC_EVENT_MASK)
 486                 return -EINVAL;
 487
 488         box = uncore_pmu_to_box(pmu, event->cpu);
 489         if (!box || box->cpu < 0)
 490                 return -EINVAL;
 491
 492         event->cpu = box->cpu;
 493         event->pmu_private = box;
 494
 495         event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
 496
 497         event->hw.idx = -1;
 498         event->hw.last_tag = ~0ULL;
 499         event->hw.extra_reg.idx = EXTRA_REG_NONE;
 500         event->hw.branch_reg.idx = EXTRA_REG_NONE;
 501         /*
 502          * check event is known (whitelist, determines counter)
 503          */
 504         switch (cfg) {
 505         case SNB_UNCORE_PCI_IMC_DATA_READS:
 506                 base = SNB_UNCORE_PCI_IMC_DATA_READS_BASE;
 507                 idx = UNCORE_PMC_IDX_FREERUNNING;
 508                 break;
 509         case SNB_UNCORE_PCI_IMC_DATA_WRITES:
 510                 base = SNB_UNCORE_PCI_IMC_DATA_WRITES_BASE;
 511                 idx = UNCORE_PMC_IDX_FREERUNNING;
 512                 break;
 513         default:
 514                 return -EINVAL;
 515         }
 516
 517         /* must be done before validate_group */
 518         event->hw.event_base = base;
 519         event->hw.idx = idx;
 520
 521         /* Convert to standard encoding format for freerunning counters */
 522         event->hw.config = ((cfg - 1) << 8) | 0x10ff;
 523
 524         /* no group validation needed, we have free running counters */
 525
 526         return 0;
 527 }
 528
 529 static int snb_uncore_imc_hw_config(struct intel_uncore_box *box, struct perf_event *event)
 530 {
 531         return 0;
 532 }
 533
 534 int snb_pci2phy_map_init(int devid)
 535 {
 536         struct pci_dev *dev = NULL;
 537         struct pci2phy_map *map;
 538         int bus, segment;
 539
 540         dev = pci_get_device(PCI_VENDOR_ID_INTEL, devid, dev);
 541         if (!dev)
 542                 return -ENOTTY;
 543
 544         bus = dev->bus->number;
 545         segment = pci_domain_nr(dev->bus);
 546
 547         raw_spin_lock(&pci2phy_map_lock);
 548         map = __find_pci2phy_map(segment);
 549         if (!map) {
 550                 raw_spin_unlock(&pci2phy_map_lock);
 551                 pci_dev_put(dev);
 552                 return -ENOMEM;
 553         }
 554         map->pbus_to_physid[bus] = 0;
 555         raw_spin_unlock(&pci2phy_map_lock);
 556
 557         pci_dev_put(dev);
 558
 559         return 0;
 560 }
 561
 562 static struct pmu snb_uncore_imc_pmu = {
 563         .task_ctx_nr    = perf_invalid_context,
 564         .event_init     = snb_uncore_imc_event_init,
 565         .add            = uncore_pmu_event_add,
 566         .del            = uncore_pmu_event_del,
 567         .start          = uncore_pmu_event_start,
 568         .stop           = uncore_pmu_event_stop,
 569         .read           = uncore_pmu_event_read,
 570         .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
 571 };
 572
 573 static struct intel_uncore_ops snb_uncore_imc_ops = {
 574         .init_box       = snb_uncore_imc_init_box,
 575         .exit_box       = uncore_mmio_exit_box,
 576         .enable_box     = snb_uncore_imc_enable_box,
 577         .disable_box    = snb_uncore_imc_disable_box,
 578         .disable_event  = snb_uncore_imc_disable_event,
 579         .enable_event   = snb_uncore_imc_enable_event,
 580         .hw_config      = snb_uncore_imc_hw_config,
 581         .read_counter   = uncore_mmio_read_counter,
 582 };
 583
 584 static struct intel_uncore_type snb_uncore_imc = {
 585         .name           = "imc",
 586         .num_counters   = 2,
 587         .num_boxes      = 1,
 588         .num_freerunning_types  = SNB_PCI_UNCORE_IMC_FREERUNNING_TYPE_MAX,
 589         .freerunning    = snb_uncore_imc_freerunning,
 590         .event_descs    = snb_uncore_imc_events,
 591         .format_group   = &snb_uncore_imc_format_group,
 592         .ops            = &snb_uncore_imc_ops,
 593         .pmu            = &snb_uncore_imc_pmu,
 594 };
 595
 596 static struct intel_uncore_type *snb_pci_uncores[] = {
 597         [SNB_PCI_UNCORE_IMC]    = &snb_uncore_imc,
 598         NULL,
 599 };
 600
 601 static const struct pci_device_id snb_uncore_pci_ids[] = {
 602         { /* IMC */
 603                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SNB_IMC),
 604                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 605         },
 606         { /* end: all zeroes */ },
 607 };
 608
 609 static const struct pci_device_id ivb_uncore_pci_ids[] = {
 610         { /* IMC */
 611                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IVB_IMC),
 612                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 613         },
 614         { /* IMC */
 615                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IVB_E3_IMC),
 616                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 617         },
 618         { /* end: all zeroes */ },
 619 };
 620
 621 static const struct pci_device_id hsw_uncore_pci_ids[] = {
 622         { /* IMC */
 623                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_HSW_IMC),
 624                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 625         },
 626         { /* IMC */
 627                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_HSW_U_IMC),
 628                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 629         },
 630         { /* end: all zeroes */ },
 631 };
 632
 633 static const struct pci_device_id bdw_uncore_pci_ids[] = {
 634         { /* IMC */
 635                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_BDW_IMC),
 636                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 637         },
 638         { /* end: all zeroes */ },
 639 };
 640
 641 static const struct pci_device_id skl_uncore_pci_ids[] = {
 642         { /* IMC */
 643                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_Y_IMC),
 644                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 645         },
 646         { /* IMC */
 647                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_U_IMC),
 648                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 649         },
 650         { /* IMC */
 651                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_HD_IMC),
 652                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 653         },
 654         { /* IMC */
 655                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_HQ_IMC),
 656                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 657         },
 658         { /* IMC */
 659                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_SD_IMC),
 660                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 661         },
 662         { /* IMC */
 663                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_SQ_IMC),
 664                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 665         },
 666         { /* IMC */
 667                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SKL_E3_IMC),
 668                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 669         },
 670         { /* IMC */
 671                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_Y_IMC),
 672                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 673         },
 674         { /* IMC */
 675                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_U_IMC),
 676                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 677         },
 678         { /* IMC */
 679                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_UQ_IMC),
 680                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 681         },
 682         { /* IMC */
 683                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_SD_IMC),
 684                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 685         },
 686         { /* IMC */
 687                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_SQ_IMC),
 688                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 689         },
 690         { /* IMC */
 691                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_HQ_IMC),
 692                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 693         },
 694         { /* IMC */
 695                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_KBL_WQ_IMC),
 696                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 697         },
 698         { /* IMC */
 699                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_2U_IMC),
 700                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 701         },
 702         { /* IMC */
 703                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4U_IMC),
 704                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 705         },
 706         { /* IMC */
 707                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4H_IMC),
 708                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 709         },
 710         { /* IMC */
 711                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6H_IMC),
 712                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 713         },
 714         { /* IMC */
 715                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_2S_D_IMC),
 716                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 717         },
 718         { /* IMC */
 719                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_D_IMC),
 720                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 721         },
 722         { /* IMC */
 723                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_D_IMC),
 724                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 725         },
 726         { /* IMC */
 727                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_D_IMC),
 728                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 729         },
 730         { /* IMC */
 731                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_W_IMC),
 732                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 733         },
 734         { /* IMC */
 735                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_W_IMC),
 736                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 737         },
 738         { /* IMC */
 739                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_W_IMC),
 740                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 741         },
 742         { /* IMC */
 743                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_4S_S_IMC),
 744                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 745         },
 746         { /* IMC */
 747                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_6S_S_IMC),
 748                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 749         },
 750         { /* IMC */
 751                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CFL_8S_S_IMC),
 752                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 753         },
 754         { /* IMC */
 755                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_AML_YD_IMC),
 756                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 757         },
 758         { /* IMC */
 759                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_AML_YQ_IMC),
 760                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 761         },
 762         { /* IMC */
 763                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_UQ_IMC),
 764                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 765         },
 766         { /* IMC */
 767                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_4_UQ_IMC),
 768                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 769         },
 770         { /* IMC */
 771                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_WHL_UD_IMC),
 772                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 773         },
 774         { /* end: all zeroes */ },
 775 };
 776
 777 static const struct pci_device_id icl_uncore_pci_ids[] = {
 778         { /* IMC */
 779                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICL_U_IMC),
 780                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 781         },
 782         { /* IMC */
 783                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICL_U2_IMC),
 784                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
 785         },
 786         { /* end: all zeroes */ },
 787 };
 788
 789 static struct pci_driver snb_uncore_pci_driver = {
 790         .name           = "snb_uncore",
 791         .id_table       = snb_uncore_pci_ids,
 792 };
 793
 794 static struct pci_driver ivb_uncore_pci_driver = {
 795         .name           = "ivb_uncore",
 796         .id_table       = ivb_uncore_pci_ids,
 797 };
 798
 799 static struct pci_driver hsw_uncore_pci_driver = {
 800         .name           = "hsw_uncore",
 801         .id_table       = hsw_uncore_pci_ids,
 802 };
 803
 804 static struct pci_driver bdw_uncore_pci_driver = {
 805         .name           = "bdw_uncore",
 806         .id_table       = bdw_uncore_pci_ids,
 807 };
 808
 809 static struct pci_driver skl_uncore_pci_driver = {
 810         .name           = "skl_uncore",
 811         .id_table       = skl_uncore_pci_ids,
 812 };
 813
 814 static struct pci_driver icl_uncore_pci_driver = {
 815         .name           = "icl_uncore",
 816         .id_table       = icl_uncore_pci_ids,
 817 };
 818
 819 struct imc_uncore_pci_dev {
 820         __u32 pci_id;
 821         struct pci_driver *driver;
 822 };
 823 #define IMC_DEV(a, d) \
 824         { .pci_id = PCI_DEVICE_ID_INTEL_##a, .driver = (d) }
 825
 826 static const struct imc_uncore_pci_dev desktop_imc_pci_ids[] = {
 827         IMC_DEV(SNB_IMC, &snb_uncore_pci_driver),
 828         IMC_DEV(IVB_IMC, &ivb_uncore_pci_driver),    /* 3rd Gen Core processor */
 829         IMC_DEV(IVB_E3_IMC, &ivb_uncore_pci_driver), /* Xeon E3-1200 v2/3rd Gen Core processor */
 830         IMC_DEV(HSW_IMC, &hsw_uncore_pci_driver),    /* 4th Gen Core Processor */
 831         IMC_DEV(HSW_U_IMC, &hsw_uncore_pci_driver),  /* 4th Gen Core ULT Mobile Processor */
 832         IMC_DEV(BDW_IMC, &bdw_uncore_pci_driver),    /* 5th Gen Core U */
 833         IMC_DEV(SKL_Y_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core Y */
 834         IMC_DEV(SKL_U_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core U */
 835         IMC_DEV(SKL_HD_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core H Dual Core */
 836         IMC_DEV(SKL_HQ_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core H Quad Core */
 837         IMC_DEV(SKL_SD_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core S Dual Core */
 838         IMC_DEV(SKL_SQ_IMC, &skl_uncore_pci_driver),  /* 6th Gen Core S Quad Core */
 839         IMC_DEV(SKL_E3_IMC, &skl_uncore_pci_driver),  /* Xeon E3 V5 Gen Core processor */
 840         IMC_DEV(KBL_Y_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core Y */
 841         IMC_DEV(KBL_U_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core U */
 842         IMC_DEV(KBL_UQ_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core U Quad Core */
 843         IMC_DEV(KBL_SD_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core S Dual Core */
 844         IMC_DEV(KBL_SQ_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core S Quad Core */
 845         IMC_DEV(KBL_HQ_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core H Quad Core */
 846         IMC_DEV(KBL_WQ_IMC, &skl_uncore_pci_driver),  /* 7th Gen Core S 4 cores Work Station */
 847         IMC_DEV(CFL_2U_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core U 2 Cores */
 848         IMC_DEV(CFL_4U_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core U 4 Cores */
 849         IMC_DEV(CFL_4H_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core H 4 Cores */
 850         IMC_DEV(CFL_6H_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core H 6 Cores */
 851         IMC_DEV(CFL_2S_D_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 2 Cores Desktop */
 852         IMC_DEV(CFL_4S_D_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 4 Cores Desktop */
 853         IMC_DEV(CFL_6S_D_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 6 Cores Desktop */
 854         IMC_DEV(CFL_8S_D_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 8 Cores Desktop */
 855         IMC_DEV(CFL_4S_W_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 4 Cores Work Station */
 856         IMC_DEV(CFL_6S_W_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 6 Cores Work Station */
 857         IMC_DEV(CFL_8S_W_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 8 Cores Work Station */
 858         IMC_DEV(CFL_4S_S_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 4 Cores Server */
 859         IMC_DEV(CFL_6S_S_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 6 Cores Server */
 860         IMC_DEV(CFL_8S_S_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core S 8 Cores Server */
 861         IMC_DEV(AML_YD_IMC, &skl_uncore_pci_driver),    /* 8th Gen Core Y Mobile Dual Core */
 862         IMC_DEV(AML_YQ_IMC, &skl_uncore_pci_driver),    /* 8th Gen Core Y Mobile Quad Core */
 863         IMC_DEV(WHL_UQ_IMC, &skl_uncore_pci_driver),    /* 8th Gen Core U Mobile Quad Core */
 864         IMC_DEV(WHL_4_UQ_IMC, &skl_uncore_pci_driver),  /* 8th Gen Core U Mobile Quad Core */
 865         IMC_DEV(WHL_UD_IMC, &skl_uncore_pci_driver),    /* 8th Gen Core U Mobile Dual Core */
 866         IMC_DEV(ICL_U_IMC, &icl_uncore_pci_driver),     /* 10th Gen Core Mobile */
 867         IMC_DEV(ICL_U2_IMC, &icl_uncore_pci_driver),    /* 10th Gen Core Mobile */
 868         {  /* end marker */ }
 869 };
 870
 871
 872 #define for_each_imc_pci_id(x, t) \
 873         for (x = (t); (x)->pci_id; x++)
 874
 875 static struct pci_driver *imc_uncore_find_dev(void)
 876 {
 877         const struct imc_uncore_pci_dev *p;
 878         int ret;
 879
 880         for_each_imc_pci_id(p, desktop_imc_pci_ids) {
 881                 ret = snb_pci2phy_map_init(p->pci_id);
 882                 if (ret == 0)
 883                         return p->driver;
 884         }
 885         return NULL;
 886 }
 887
 888 static int imc_uncore_pci_init(void)
 889 {
 890         struct pci_driver *imc_drv = imc_uncore_find_dev();
 891
 892         if (!imc_drv)
 893                 return -ENODEV;
 894
 895         uncore_pci_uncores = snb_pci_uncores;
 896         uncore_pci_driver = imc_drv;
 897
 898         return 0;
 899 }
 900
 901 int snb_uncore_pci_init(void)
 902 {
 903         return imc_uncore_pci_init();
 904 }
 905
 906 int ivb_uncore_pci_init(void)
 907 {
 908         return imc_uncore_pci_init();
 909 }
 910 int hsw_uncore_pci_init(void)
 911 {
 912         return imc_uncore_pci_init();
 913 }
 914
 915 int bdw_uncore_pci_init(void)
 916 {
 917         return imc_uncore_pci_init();
 918 }
 919
 920 int skl_uncore_pci_init(void)
 921 {
 922         return imc_uncore_pci_init();
 923 }
 924
 925 /* end of Sandy Bridge uncore support */
 926
 927 /* Nehalem uncore support */
 928 static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box)
 929 {
 930         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0);
 931 }
 932
 933 static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box)
 934 {
 935         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
 936 }
 937
 938 static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 939 {
 940         struct hw_perf_event *hwc = &event->hw;
 941
 942         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
 943                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
 944         else
 945                 wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
 946 }
 947
 948 static struct attribute *nhm_uncore_formats_attr[] = {
 949         &format_attr_event.attr,
 950         &format_attr_umask.attr,
 951         &format_attr_edge.attr,
 952         &format_attr_inv.attr,
 953         &format_attr_cmask8.attr,
 954         NULL,
 955 };
 956
 957 static const struct attribute_group nhm_uncore_format_group = {
 958         .name = "format",
 959         .attrs = nhm_uncore_formats_attr,
 960 };
 961
 962 static struct uncore_event_desc nhm_uncore_events[] = {
 963         INTEL_UNCORE_EVENT_DESC(clockticks,                "event=0xff,umask=0x00"),
 964         INTEL_UNCORE_EVENT_DESC(qmc_writes_full_any,       "event=0x2f,umask=0x0f"),
 965         INTEL_UNCORE_EVENT_DESC(qmc_normal_reads_any,      "event=0x2c,umask=0x0f"),
 966         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_reads,     "event=0x20,umask=0x01"),
 967         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_writes,    "event=0x20,umask=0x02"),
 968         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_reads,  "event=0x20,umask=0x04"),
 969         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_writes, "event=0x20,umask=0x08"),
 970         INTEL_UNCORE_EVENT_DESC(qhl_request_local_reads,   "event=0x20,umask=0x10"),
 971         INTEL_UNCORE_EVENT_DESC(qhl_request_local_writes,  "event=0x20,umask=0x20"),
 972         { /* end: all zeroes */ },
 973 };
 974
 975 static struct intel_uncore_ops nhm_uncore_msr_ops = {
 976         .disable_box    = nhm_uncore_msr_disable_box,
 977         .enable_box     = nhm_uncore_msr_enable_box,
 978         .disable_event  = snb_uncore_msr_disable_event,
 979         .enable_event   = nhm_uncore_msr_enable_event,
 980         .read_counter   = uncore_msr_read_counter,
 981 };
 982
 983 static struct intel_uncore_type nhm_uncore = {
 984         .name           = "",
 985         .num_counters   = 8,
 986         .num_boxes      = 1,
 987         .perf_ctr_bits  = 48,
 988         .fixed_ctr_bits = 48,
 989         .event_ctl      = NHM_UNC_PERFEVTSEL0,
 990         .perf_ctr       = NHM_UNC_UNCORE_PMC0,
 991         .fixed_ctr      = NHM_UNC_FIXED_CTR,
 992         .fixed_ctl      = NHM_UNC_FIXED_CTR_CTRL,
 993         .event_mask     = NHM_UNC_RAW_EVENT_MASK,
 994         .event_descs    = nhm_uncore_events,
 995         .ops            = &nhm_uncore_msr_ops,
 996         .format_group   = &nhm_uncore_format_group,
 997 };
 998
 999 static struct intel_uncore_type *nhm_msr_uncores[] = {
1000         &nhm_uncore,
1001         NULL,
1002 };
1003
1004 void nhm_uncore_cpu_init(void)
1005 {
1006         uncore_msr_uncores = nhm_msr_uncores;
1007 }
1008
1009 /* end of Nehalem uncore support */
1010
1011 /* Tiger Lake MMIO uncore support */
1012
1013 static const struct pci_device_id tgl_uncore_pci_ids[] = {
1014         { /* IMC */
1015                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U1_IMC),
1016                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1017         },
1018         { /* IMC */
1019                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U2_IMC),
1020                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1021         },
1022         { /* IMC */
1023                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U3_IMC),
1024                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1025         },
1026         { /* IMC */
1027                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_U4_IMC),
1028                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1029         },
1030         { /* IMC */
1031                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_TGL_H_IMC),
1032                 .driver_data = UNCORE_PCI_DEV_DATA(SNB_PCI_UNCORE_IMC, 0),
1033         },
1034         { /* end: all zeroes */ }
1035 };
1036
1037 enum perf_tgl_uncore_imc_freerunning_types {
1038         TGL_MMIO_UNCORE_IMC_DATA_TOTAL,
1039         TGL_MMIO_UNCORE_IMC_DATA_READ,
1040         TGL_MMIO_UNCORE_IMC_DATA_WRITE,
1041         TGL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX
1042 };
1043
1044 static struct freerunning_counters tgl_l_uncore_imc_freerunning[] = {
1045         [TGL_MMIO_UNCORE_IMC_DATA_TOTAL]        = { 0x5040, 0x0, 0x0, 1, 64 },
1046         [TGL_MMIO_UNCORE_IMC_DATA_READ]         = { 0x5058, 0x0, 0x0, 1, 64 },
1047         [TGL_MMIO_UNCORE_IMC_DATA_WRITE]        = { 0x50A0, 0x0, 0x0, 1, 64 },
1048 };
1049
1050 static struct freerunning_counters tgl_uncore_imc_freerunning[] = {
1051         [TGL_MMIO_UNCORE_IMC_DATA_TOTAL]        = { 0xd840, 0x0, 0x0, 1, 64 },
1052         [TGL_MMIO_UNCORE_IMC_DATA_READ]         = { 0xd858, 0x0, 0x0, 1, 64 },
1053         [TGL_MMIO_UNCORE_IMC_DATA_WRITE]        = { 0xd8A0, 0x0, 0x0, 1, 64 },
1054 };
1055
1056 static struct uncore_event_desc tgl_uncore_imc_events[] = {
1057         INTEL_UNCORE_EVENT_DESC(data_total,         "event=0xff,umask=0x10"),
1058         INTEL_UNCORE_EVENT_DESC(data_total.scale,   "6.103515625e-5"),
1059         INTEL_UNCORE_EVENT_DESC(data_total.unit,    "MiB"),
1060
1061         INTEL_UNCORE_EVENT_DESC(data_read,         "event=0xff,umask=0x20"),
1062         INTEL_UNCORE_EVENT_DESC(data_read.scale,   "6.103515625e-5"),
1063         INTEL_UNCORE_EVENT_DESC(data_read.unit,    "MiB"),
1064
1065         INTEL_UNCORE_EVENT_DESC(data_write,        "event=0xff,umask=0x30"),
1066         INTEL_UNCORE_EVENT_DESC(data_write.scale,  "6.103515625e-5"),
1067         INTEL_UNCORE_EVENT_DESC(data_write.unit,   "MiB"),
1068
1069         { /* end: all zeroes */ }
1070 };
1071
1072 static struct pci_dev *tgl_uncore_get_mc_dev(void)
1073 {
1074         const struct pci_device_id *ids = tgl_uncore_pci_ids;
1075         struct pci_dev *mc_dev = NULL;
1076
1077         while (ids && ids->vendor) {
1078                 mc_dev = pci_get_device(PCI_VENDOR_ID_INTEL, ids->device, NULL);
1079                 if (mc_dev)
1080                         return mc_dev;
1081                 ids++;
1082         }
1083
1084         return mc_dev;
1085 }
1086
1087 #define TGL_UNCORE_MMIO_IMC_MEM_OFFSET          0x10000
1088
1089 static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
1090 {
1091         struct pci_dev *pdev = tgl_uncore_get_mc_dev();
1092         struct intel_uncore_pmu *pmu = box->pmu;
1093         resource_size_t addr;
1094         u32 mch_bar;
1095
1096         if (!pdev) {
1097                 pr_warn("perf uncore: Cannot find matched IMC device.\n");
1098                 return;
1099         }
1100
1101         pci_read_config_dword(pdev, SNB_UNCORE_PCI_IMC_BAR_OFFSET, &mch_bar);
1102         /* MCHBAR is disabled */
1103         if (!(mch_bar & BIT(0))) {
1104                 pr_warn("perf uncore: MCHBAR is disabled. Failed to map IMC free-running counters.\n");
1105                 return;
1106         }
1107         mch_bar &= ~BIT(0);
1108         addr = (resource_size_t)(mch_bar + TGL_UNCORE_MMIO_IMC_MEM_OFFSET * pmu->pmu_idx);
1109
1110 #ifdef CONFIG_PHYS_ADDR_T_64BIT
1111         pci_read_config_dword(pdev, SNB_UNCORE_PCI_IMC_BAR_OFFSET + 4, &mch_bar);
1112         addr |= ((resource_size_t)mch_bar << 32);
1113 #endif
1114
1115         box->io_addr = ioremap(addr, SNB_UNCORE_PCI_IMC_MAP_SIZE);
1116 }
1117
1118 static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = {
1119         .init_box       = tgl_uncore_imc_freerunning_init_box,
1120         .exit_box       = uncore_mmio_exit_box,
1121         .read_counter   = uncore_mmio_read_counter,
1122         .hw_config      = uncore_freerunning_hw_config,
1123 };
1124
1125 static struct attribute *tgl_uncore_imc_formats_attr[] = {
1126         &format_attr_event.attr,
1127         &format_attr_umask.attr,
1128         NULL
1129 };
1130
1131 static const struct attribute_group tgl_uncore_imc_format_group = {
1132         .name = "format",
1133         .attrs = tgl_uncore_imc_formats_attr,
1134 };
1135
1136 static struct intel_uncore_type tgl_uncore_imc_free_running = {
1137         .name                   = "imc_free_running",
1138         .num_counters           = 3,
1139         .num_boxes              = 2,
1140         .num_freerunning_types  = TGL_MMIO_UNCORE_IMC_FREERUNNING_TYPE_MAX,
1141         .freerunning            = tgl_uncore_imc_freerunning,
1142         .ops                    = &tgl_uncore_imc_freerunning_ops,
1143         .event_descs            = tgl_uncore_imc_events,
1144         .format_group           = &tgl_uncore_imc_format_group,
1145 };
1146
1147 static struct intel_uncore_type *tgl_mmio_uncores[] = {
1148         &tgl_uncore_imc_free_running,
1149         NULL
1150 };
1151
1152 void tgl_l_uncore_mmio_init(void)
1153 {
1154         tgl_uncore_imc_free_running.freerunning = tgl_l_uncore_imc_freerunning;
1155         uncore_mmio_uncores = tgl_mmio_uncores;
1156 }
1157
1158 void tgl_uncore_mmio_init(void)
1159 {
1160         uncore_mmio_uncores = tgl_mmio_uncores;
1161 }
1162
1163 /* end of Tiger Lake MMIO uncore support */