arch/x86/events/intel/core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Per core/cpu state
   4  *
   5  * Used to coordinate shared registers between HT threads or
   6  * among events on a single PMU.
   7  */
   8
   9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10
  11 #include <linux/stddef.h>
  12 #include <linux/types.h>
  13 #include <linux/init.h>
  14 #include <linux/slab.h>
  15 #include <linux/export.h>
  16 #include <linux/nmi.h>
  17
  18 #include <asm/cpufeature.h>
  19 #include <asm/hardirq.h>
  20 #include <asm/intel-family.h>
  21 #include <asm/intel_pt.h>
  22 #include <asm/apic.h>
  23 #include <asm/cpu_device_id.h>
  24
  25 #include "../perf_event.h"
  26
  27 /*
  28  * Intel PerfMon, used on Core and later.
  29  */
  30 static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =
  31 {
  32         [PERF_COUNT_HW_CPU_CYCLES]              = 0x003c,
  33         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
  34         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0x4f2e,
  35         [PERF_COUNT_HW_CACHE_MISSES]            = 0x412e,
  36         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c4,
  37         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c5,
  38         [PERF_COUNT_HW_BUS_CYCLES]              = 0x013c,
  39         [PERF_COUNT_HW_REF_CPU_CYCLES]          = 0x0300, /* pseudo-encoding */
  40 };
  41
  42 static struct event_constraint intel_core_event_constraints[] __read_mostly =
  43 {
  44         INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  45         INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  46         INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  47         INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  48         INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  49         INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FP_COMP_INSTR_RET */
  50         EVENT_CONSTRAINT_END
  51 };
  52
  53 static struct event_constraint intel_core2_event_constraints[] __read_mostly =
  54 {
  55         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  56         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  57         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  58         INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
  59         INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
  60         INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
  61         INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
  62         INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
  63         INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
  64         INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
  65         INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
  66         INTEL_EVENT_CONSTRAINT(0xc9, 0x1), /* ITLB_MISS_RETIRED (T30-9) */
  67         INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
  68         EVENT_CONSTRAINT_END
  69 };
  70
  71 static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
  72 {
  73         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  74         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  75         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
  76         INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
  77         INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
  78         INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
  79         INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
  80         INTEL_EVENT_CONSTRAINT(0x48, 0x3), /* L1D_PEND_MISS */
  81         INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
  82         INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
  83         INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
  84         EVENT_CONSTRAINT_END
  85 };
  86
  87 static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
  88 {
  89         /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
  90         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
  91         INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
  92         EVENT_EXTRA_END
  93 };
  94
  95 static struct event_constraint intel_westmere_event_constraints[] __read_mostly =
  96 {
  97         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
  98         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
  99         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
 100         INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
 101         INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */
 102         INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
 103         INTEL_EVENT_CONSTRAINT(0xb3, 0x1), /* SNOOPQ_REQUEST_OUTSTANDING */
 104         EVENT_CONSTRAINT_END
 105 };
 106
 107 static struct event_constraint intel_snb_event_constraints[] __read_mostly =
 108 {
 109         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
 110         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
 111         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
 112         INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
 113         INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
 114         INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
 115         INTEL_UEVENT_CONSTRAINT(0x06a3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
 116         INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
 117         INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
 118         INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
 119         INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
 120         INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
 121
 122         /*
 123          * When HT is off these events can only run on the bottom 4 counters
 124          * When HT is on, they are impacted by the HT bug and require EXCL access
 125          */
 126         INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */
 127         INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
 128         INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
 129         INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
 130
 131         EVENT_CONSTRAINT_END
 132 };
 133
 134 static struct event_constraint intel_ivb_event_constraints[] __read_mostly =
 135 {
 136         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
 137         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
 138         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
 139         INTEL_UEVENT_CONSTRAINT(0x0148, 0x4), /* L1D_PEND_MISS.PENDING */
 140         INTEL_UEVENT_CONSTRAINT(0x0279, 0xf), /* IDQ.EMTPY */
 141         INTEL_UEVENT_CONSTRAINT(0x019c, 0xf), /* IDQ_UOPS_NOT_DELIVERED.CORE */
 142         INTEL_UEVENT_CONSTRAINT(0x02a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_LDM_PENDING */
 143         INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
 144         INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
 145         INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf), /* CYCLE_ACTIVITY.STALLS_LDM_PENDING */
 146         INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
 147         INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
 148         INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
 149
 150         /*
 151          * When HT is off these events can only run on the bottom 4 counters
 152          * When HT is on, they are impacted by the HT bug and require EXCL access
 153          */
 154         INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */
 155         INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
 156         INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
 157         INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
 158
 159         EVENT_CONSTRAINT_END
 160 };
 161
 162 static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
 163 {
 164         /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
 165         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
 166         INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1),
 167         INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
 168         EVENT_EXTRA_END
 169 };
 170
 171 static struct event_constraint intel_v1_event_constraints[] __read_mostly =
 172 {
 173         EVENT_CONSTRAINT_END
 174 };
 175
 176 static struct event_constraint intel_gen_event_constraints[] __read_mostly =
 177 {
 178         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
 179         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
 180         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
 181         EVENT_CONSTRAINT_END
 182 };
 183
 184 static struct event_constraint intel_slm_event_constraints[] __read_mostly =
 185 {
 186         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
 187         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
 188         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
 189         EVENT_CONSTRAINT_END
 190 };
 191
 192 static struct event_constraint intel_skl_event_constraints[] = {
 193         FIXED_EVENT_CONSTRAINT(0x00c0, 0),      /* INST_RETIRED.ANY */
 194         FIXED_EVENT_CONSTRAINT(0x003c, 1),      /* CPU_CLK_UNHALTED.CORE */
 195         FIXED_EVENT_CONSTRAINT(0x0300, 2),      /* CPU_CLK_UNHALTED.REF */
 196         INTEL_UEVENT_CONSTRAINT(0x1c0, 0x2),    /* INST_RETIRED.PREC_DIST */
 197
 198         /*
 199          * when HT is off, these can only run on the bottom 4 counters
 200          */
 201         INTEL_EVENT_CONSTRAINT(0xd0, 0xf),      /* MEM_INST_RETIRED.* */
 202         INTEL_EVENT_CONSTRAINT(0xd1, 0xf),      /* MEM_LOAD_RETIRED.* */
 203         INTEL_EVENT_CONSTRAINT(0xd2, 0xf),      /* MEM_LOAD_L3_HIT_RETIRED.* */
 204         INTEL_EVENT_CONSTRAINT(0xcd, 0xf),      /* MEM_TRANS_RETIRED.* */
 205         INTEL_EVENT_CONSTRAINT(0xc6, 0xf),      /* FRONTEND_RETIRED.* */
 206
 207         EVENT_CONSTRAINT_END
 208 };
 209
 210 static struct extra_reg intel_knl_extra_regs[] __read_mostly = {
 211         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x799ffbb6e7ull, RSP_0),
 212         INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x399ffbffe7ull, RSP_1),
 213         EVENT_EXTRA_END
 214 };
 215
 216 static struct extra_reg intel_snb_extra_regs[] __read_mostly = {
 217         /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
 218         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0),
 219         INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1),
 220         INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
 221         EVENT_EXTRA_END
 222 };
 223
 224 static struct extra_reg intel_snbep_extra_regs[] __read_mostly = {
 225         /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
 226         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0),
 227         INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1),
 228         INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
 229         EVENT_EXTRA_END
 230 };
 231
 232 static struct extra_reg intel_skl_extra_regs[] __read_mostly = {
 233         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0),
 234         INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1),
 235         INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
 236         /*
 237          * Note the low 8 bits eventsel code is not a continuous field, containing
 238          * some #GPing bits. These are masked out.
 239          */
 240         INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff17, FE),
 241         EVENT_EXTRA_END
 242 };
 243
 244 static struct event_constraint intel_icl_event_constraints[] = {
 245         FIXED_EVENT_CONSTRAINT(0x00c0, 0),      /* INST_RETIRED.ANY */
 246         INTEL_UEVENT_CONSTRAINT(0x1c0, 0),      /* INST_RETIRED.PREC_DIST */
 247         FIXED_EVENT_CONSTRAINT(0x003c, 1),      /* CPU_CLK_UNHALTED.CORE */
 248         FIXED_EVENT_CONSTRAINT(0x0300, 2),      /* CPU_CLK_UNHALTED.REF */
 249         FIXED_EVENT_CONSTRAINT(0x0400, 3),      /* SLOTS */
 250         INTEL_EVENT_CONSTRAINT_RANGE(0x03, 0x0a, 0xf),
 251         INTEL_EVENT_CONSTRAINT_RANGE(0x1f, 0x28, 0xf),
 252         INTEL_EVENT_CONSTRAINT(0x32, 0xf),      /* SW_PREFETCH_ACCESS.* */
 253         INTEL_EVENT_CONSTRAINT_RANGE(0x48, 0x54, 0xf),
 254         INTEL_EVENT_CONSTRAINT_RANGE(0x60, 0x8b, 0xf),
 255         INTEL_UEVENT_CONSTRAINT(0x04a3, 0xff),  /* CYCLE_ACTIVITY.STALLS_TOTAL */
 256         INTEL_UEVENT_CONSTRAINT(0x10a3, 0xff),  /* CYCLE_ACTIVITY.STALLS_MEM_ANY */
 257         INTEL_EVENT_CONSTRAINT(0xa3, 0xf),      /* CYCLE_ACTIVITY.* */
 258         INTEL_EVENT_CONSTRAINT_RANGE(0xa8, 0xb0, 0xf),
 259         INTEL_EVENT_CONSTRAINT_RANGE(0xb7, 0xbd, 0xf),
 260         INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xe6, 0xf),
 261         INTEL_EVENT_CONSTRAINT_RANGE(0xf0, 0xf4, 0xf),
 262         EVENT_CONSTRAINT_END
 263 };
 264
 265 static struct extra_reg intel_icl_extra_regs[] __read_mostly = {
 266         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffffbfffull, RSP_0),
 267         INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffffbfffull, RSP_1),
 268         INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
 269         INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff17, FE),
 270         EVENT_EXTRA_END
 271 };
 272
 273 EVENT_ATTR_STR(mem-loads,       mem_ld_nhm,     "event=0x0b,umask=0x10,ldlat=3");
 274 EVENT_ATTR_STR(mem-loads,       mem_ld_snb,     "event=0xcd,umask=0x1,ldlat=3");
 275 EVENT_ATTR_STR(mem-stores,      mem_st_snb,     "event=0xcd,umask=0x2");
 276
 277 static struct attribute *nhm_mem_events_attrs[] = {
 278         EVENT_PTR(mem_ld_nhm),
 279         NULL,
 280 };
 281
 282 /*
 283  * topdown events for Intel Core CPUs.
 284  *
 285  * The events are all in slots, which is a free slot in a 4 wide
 286  * pipeline. Some events are already reported in slots, for cycle
 287  * events we multiply by the pipeline width (4).
 288  *
 289  * With Hyper Threading on, topdown metrics are either summed or averaged
 290  * between the threads of a core: (count_t0 + count_t1).
 291  *
 292  * For the average case the metric is always scaled to pipeline width,
 293  * so we use factor 2 ((count_t0 + count_t1) / 2 * 4)
 294  */
 295
 296 EVENT_ATTR_STR_HT(topdown-total-slots, td_total_slots,
 297         "event=0x3c,umask=0x0",                 /* cpu_clk_unhalted.thread */
 298         "event=0x3c,umask=0x0,any=1");          /* cpu_clk_unhalted.thread_any */
 299 EVENT_ATTR_STR_HT(topdown-total-slots.scale, td_total_slots_scale, "4", "2");
 300 EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued,
 301         "event=0xe,umask=0x1");                 /* uops_issued.any */
 302 EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired,
 303         "event=0xc2,umask=0x2");                /* uops_retired.retire_slots */
 304 EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles,
 305         "event=0x9c,umask=0x1");                /* idq_uops_not_delivered_core */
 306 EVENT_ATTR_STR_HT(topdown-recovery-bubbles, td_recovery_bubbles,
 307         "event=0xd,umask=0x3,cmask=1",          /* int_misc.recovery_cycles */
 308         "event=0xd,umask=0x3,cmask=1,any=1");   /* int_misc.recovery_cycles_any */
 309 EVENT_ATTR_STR_HT(topdown-recovery-bubbles.scale, td_recovery_bubbles_scale,
 310         "4", "2");
 311
 312 static struct attribute *snb_events_attrs[] = {
 313         EVENT_PTR(td_slots_issued),
 314         EVENT_PTR(td_slots_retired),
 315         EVENT_PTR(td_fetch_bubbles),
 316         EVENT_PTR(td_total_slots),
 317         EVENT_PTR(td_total_slots_scale),
 318         EVENT_PTR(td_recovery_bubbles),
 319         EVENT_PTR(td_recovery_bubbles_scale),
 320         NULL,
 321 };
 322
 323 static struct attribute *snb_mem_events_attrs[] = {
 324         EVENT_PTR(mem_ld_snb),
 325         EVENT_PTR(mem_st_snb),
 326         NULL,
 327 };
 328
 329 static struct event_constraint intel_hsw_event_constraints[] = {
 330         FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
 331         FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
 332         FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
 333         INTEL_UEVENT_CONSTRAINT(0x148, 0x4),    /* L1D_PEND_MISS.PENDING */
 334         INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
 335         INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
 336         /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
 337         INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4),
 338         /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
 339         INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4),
 340         /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
 341         INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf),
 342
 343         /*
 344          * When HT is off these events can only run on the bottom 4 counters
 345          * When HT is on, they are impacted by the HT bug and require EXCL access
 346          */
 347         INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */
 348         INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
 349         INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
 350         INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
 351
 352         EVENT_CONSTRAINT_END
 353 };
 354
 355 static struct event_constraint intel_bdw_event_constraints[] = {
 356         FIXED_EVENT_CONSTRAINT(0x00c0, 0),      /* INST_RETIRED.ANY */
 357         FIXED_EVENT_CONSTRAINT(0x003c, 1),      /* CPU_CLK_UNHALTED.CORE */
 358         FIXED_EVENT_CONSTRAINT(0x0300, 2),      /* CPU_CLK_UNHALTED.REF */
 359         INTEL_UEVENT_CONSTRAINT(0x148, 0x4),    /* L1D_PEND_MISS.PENDING */
 360         INTEL_UBIT_EVENT_CONSTRAINT(0x8a3, 0x4),        /* CYCLE_ACTIVITY.CYCLES_L1D_MISS */
 361         /*
 362          * when HT is off, these can only run on the bottom 4 counters
 363          */
 364         INTEL_EVENT_CONSTRAINT(0xd0, 0xf),      /* MEM_INST_RETIRED.* */
 365         INTEL_EVENT_CONSTRAINT(0xd1, 0xf),      /* MEM_LOAD_RETIRED.* */
 366         INTEL_EVENT_CONSTRAINT(0xd2, 0xf),      /* MEM_LOAD_L3_HIT_RETIRED.* */
 367         INTEL_EVENT_CONSTRAINT(0xcd, 0xf),      /* MEM_TRANS_RETIRED.* */
 368         EVENT_CONSTRAINT_END
 369 };
 370
 371 static u64 intel_pmu_event_map(int hw_event)
 372 {
 373         return intel_perfmon_event_map[hw_event];
 374 }
 375
 376 /*
 377  * Notes on the events:
 378  * - data reads do not include code reads (comparable to earlier tables)
 379  * - data counts include speculative execution (except L1 write, dtlb, bpu)
 380  * - remote node access includes remote memory, remote cache, remote mmio.
 381  * - prefetches are not included in the counts.
 382  * - icache miss does not include decoded icache
 383  */
 384
 385 #define SKL_DEMAND_DATA_RD              BIT_ULL(0)
 386 #define SKL_DEMAND_RFO                  BIT_ULL(1)
 387 #define SKL_ANY_RESPONSE                BIT_ULL(16)
 388 #define SKL_SUPPLIER_NONE               BIT_ULL(17)
 389 #define SKL_L3_MISS_LOCAL_DRAM          BIT_ULL(26)
 390 #define SKL_L3_MISS_REMOTE_HOP0_DRAM    BIT_ULL(27)
 391 #define SKL_L3_MISS_REMOTE_HOP1_DRAM    BIT_ULL(28)
 392 #define SKL_L3_MISS_REMOTE_HOP2P_DRAM   BIT_ULL(29)
 393 #define SKL_L3_MISS                     (SKL_L3_MISS_LOCAL_DRAM| \
 394                                          SKL_L3_MISS_REMOTE_HOP0_DRAM| \
 395                                          SKL_L3_MISS_REMOTE_HOP1_DRAM| \
 396                                          SKL_L3_MISS_REMOTE_HOP2P_DRAM)
 397 #define SKL_SPL_HIT                     BIT_ULL(30)
 398 #define SKL_SNOOP_NONE                  BIT_ULL(31)
 399 #define SKL_SNOOP_NOT_NEEDED            BIT_ULL(32)
 400 #define SKL_SNOOP_MISS                  BIT_ULL(33)
 401 #define SKL_SNOOP_HIT_NO_FWD            BIT_ULL(34)
 402 #define SKL_SNOOP_HIT_WITH_FWD          BIT_ULL(35)
 403 #define SKL_SNOOP_HITM                  BIT_ULL(36)
 404 #define SKL_SNOOP_NON_DRAM              BIT_ULL(37)
 405 #define SKL_ANY_SNOOP                   (SKL_SPL_HIT|SKL_SNOOP_NONE| \
 406                                          SKL_SNOOP_NOT_NEEDED|SKL_SNOOP_MISS| \
 407                                          SKL_SNOOP_HIT_NO_FWD|SKL_SNOOP_HIT_WITH_FWD| \
 408                                          SKL_SNOOP_HITM|SKL_SNOOP_NON_DRAM)
 409 #define SKL_DEMAND_READ                 SKL_DEMAND_DATA_RD
 410 #define SKL_SNOOP_DRAM                  (SKL_SNOOP_NONE| \
 411                                          SKL_SNOOP_NOT_NEEDED|SKL_SNOOP_MISS| \
 412                                          SKL_SNOOP_HIT_NO_FWD|SKL_SNOOP_HIT_WITH_FWD| \
 413                                          SKL_SNOOP_HITM|SKL_SPL_HIT)
 414 #define SKL_DEMAND_WRITE                SKL_DEMAND_RFO
 415 #define SKL_LLC_ACCESS                  SKL_ANY_RESPONSE
 416 #define SKL_L3_MISS_REMOTE              (SKL_L3_MISS_REMOTE_HOP0_DRAM| \
 417                                          SKL_L3_MISS_REMOTE_HOP1_DRAM| \
 418                                          SKL_L3_MISS_REMOTE_HOP2P_DRAM)
 419
 420 static __initconst const u64 skl_hw_cache_event_ids
 421                                 [PERF_COUNT_HW_CACHE_MAX]
 422                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 423                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 424 {
 425  [ C(L1D ) ] = {
 426         [ C(OP_READ) ] = {
 427                 [ C(RESULT_ACCESS) ] = 0x81d0,  /* MEM_INST_RETIRED.ALL_LOADS */
 428                 [ C(RESULT_MISS)   ] = 0x151,   /* L1D.REPLACEMENT */
 429         },
 430         [ C(OP_WRITE) ] = {
 431                 [ C(RESULT_ACCESS) ] = 0x82d0,  /* MEM_INST_RETIRED.ALL_STORES */
 432                 [ C(RESULT_MISS)   ] = 0x0,
 433         },
 434         [ C(OP_PREFETCH) ] = {
 435                 [ C(RESULT_ACCESS) ] = 0x0,
 436                 [ C(RESULT_MISS)   ] = 0x0,
 437         },
 438  },
 439  [ C(L1I ) ] = {
 440         [ C(OP_READ) ] = {
 441                 [ C(RESULT_ACCESS) ] = 0x0,
 442                 [ C(RESULT_MISS)   ] = 0x283,   /* ICACHE_64B.MISS */
 443         },
 444         [ C(OP_WRITE) ] = {
 445                 [ C(RESULT_ACCESS) ] = -1,
 446                 [ C(RESULT_MISS)   ] = -1,
 447         },
 448         [ C(OP_PREFETCH) ] = {
 449                 [ C(RESULT_ACCESS) ] = 0x0,
 450                 [ C(RESULT_MISS)   ] = 0x0,
 451         },
 452  },
 453  [ C(LL  ) ] = {
 454         [ C(OP_READ) ] = {
 455                 [ C(RESULT_ACCESS) ] = 0x1b7,   /* OFFCORE_RESPONSE */
 456                 [ C(RESULT_MISS)   ] = 0x1b7,   /* OFFCORE_RESPONSE */
 457         },
 458         [ C(OP_WRITE) ] = {
 459                 [ C(RESULT_ACCESS) ] = 0x1b7,   /* OFFCORE_RESPONSE */
 460                 [ C(RESULT_MISS)   ] = 0x1b7,   /* OFFCORE_RESPONSE */
 461         },
 462         [ C(OP_PREFETCH) ] = {
 463                 [ C(RESULT_ACCESS) ] = 0x0,
 464                 [ C(RESULT_MISS)   ] = 0x0,
 465         },
 466  },
 467  [ C(DTLB) ] = {
 468         [ C(OP_READ) ] = {
 469                 [ C(RESULT_ACCESS) ] = 0x81d0,  /* MEM_INST_RETIRED.ALL_LOADS */
 470                 [ C(RESULT_MISS)   ] = 0xe08,   /* DTLB_LOAD_MISSES.WALK_COMPLETED */
 471         },
 472         [ C(OP_WRITE) ] = {
 473                 [ C(RESULT_ACCESS) ] = 0x82d0,  /* MEM_INST_RETIRED.ALL_STORES */
 474                 [ C(RESULT_MISS)   ] = 0xe49,   /* DTLB_STORE_MISSES.WALK_COMPLETED */
 475         },
 476         [ C(OP_PREFETCH) ] = {
 477                 [ C(RESULT_ACCESS) ] = 0x0,
 478                 [ C(RESULT_MISS)   ] = 0x0,
 479         },
 480  },
 481  [ C(ITLB) ] = {
 482         [ C(OP_READ) ] = {
 483                 [ C(RESULT_ACCESS) ] = 0x2085,  /* ITLB_MISSES.STLB_HIT */
 484                 [ C(RESULT_MISS)   ] = 0xe85,   /* ITLB_MISSES.WALK_COMPLETED */
 485         },
 486         [ C(OP_WRITE) ] = {
 487                 [ C(RESULT_ACCESS) ] = -1,
 488                 [ C(RESULT_MISS)   ] = -1,
 489         },
 490         [ C(OP_PREFETCH) ] = {
 491                 [ C(RESULT_ACCESS) ] = -1,
 492                 [ C(RESULT_MISS)   ] = -1,
 493         },
 494  },
 495  [ C(BPU ) ] = {
 496         [ C(OP_READ) ] = {
 497                 [ C(RESULT_ACCESS) ] = 0xc4,    /* BR_INST_RETIRED.ALL_BRANCHES */
 498                 [ C(RESULT_MISS)   ] = 0xc5,    /* BR_MISP_RETIRED.ALL_BRANCHES */
 499         },
 500         [ C(OP_WRITE) ] = {
 501                 [ C(RESULT_ACCESS) ] = -1,
 502                 [ C(RESULT_MISS)   ] = -1,
 503         },
 504         [ C(OP_PREFETCH) ] = {
 505                 [ C(RESULT_ACCESS) ] = -1,
 506                 [ C(RESULT_MISS)   ] = -1,
 507         },
 508  },
 509  [ C(NODE) ] = {
 510         [ C(OP_READ) ] = {
 511                 [ C(RESULT_ACCESS) ] = 0x1b7,   /* OFFCORE_RESPONSE */
 512                 [ C(RESULT_MISS)   ] = 0x1b7,   /* OFFCORE_RESPONSE */
 513         },
 514         [ C(OP_WRITE) ] = {
 515                 [ C(RESULT_ACCESS) ] = 0x1b7,   /* OFFCORE_RESPONSE */
 516                 [ C(RESULT_MISS)   ] = 0x1b7,   /* OFFCORE_RESPONSE */
 517         },
 518         [ C(OP_PREFETCH) ] = {
 519                 [ C(RESULT_ACCESS) ] = 0x0,
 520                 [ C(RESULT_MISS)   ] = 0x0,
 521         },
 522  },
 523 };
 524
 525 static __initconst const u64 skl_hw_cache_extra_regs
 526                                 [PERF_COUNT_HW_CACHE_MAX]
 527                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 528                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 529 {
 530  [ C(LL  ) ] = {
 531         [ C(OP_READ) ] = {
 532                 [ C(RESULT_ACCESS) ] = SKL_DEMAND_READ|
 533                                        SKL_LLC_ACCESS|SKL_ANY_SNOOP,
 534                 [ C(RESULT_MISS)   ] = SKL_DEMAND_READ|
 535                                        SKL_L3_MISS|SKL_ANY_SNOOP|
 536                                        SKL_SUPPLIER_NONE,
 537         },
 538         [ C(OP_WRITE) ] = {
 539                 [ C(RESULT_ACCESS) ] = SKL_DEMAND_WRITE|
 540                                        SKL_LLC_ACCESS|SKL_ANY_SNOOP,
 541                 [ C(RESULT_MISS)   ] = SKL_DEMAND_WRITE|
 542                                        SKL_L3_MISS|SKL_ANY_SNOOP|
 543                                        SKL_SUPPLIER_NONE,
 544         },
 545         [ C(OP_PREFETCH) ] = {
 546                 [ C(RESULT_ACCESS) ] = 0x0,
 547                 [ C(RESULT_MISS)   ] = 0x0,
 548         },
 549  },
 550  [ C(NODE) ] = {
 551         [ C(OP_READ) ] = {
 552                 [ C(RESULT_ACCESS) ] = SKL_DEMAND_READ|
 553                                        SKL_L3_MISS_LOCAL_DRAM|SKL_SNOOP_DRAM,
 554                 [ C(RESULT_MISS)   ] = SKL_DEMAND_READ|
 555                                        SKL_L3_MISS_REMOTE|SKL_SNOOP_DRAM,
 556         },
 557         [ C(OP_WRITE) ] = {
 558                 [ C(RESULT_ACCESS) ] = SKL_DEMAND_WRITE|
 559                                        SKL_L3_MISS_LOCAL_DRAM|SKL_SNOOP_DRAM,
 560                 [ C(RESULT_MISS)   ] = SKL_DEMAND_WRITE|
 561                                        SKL_L3_MISS_REMOTE|SKL_SNOOP_DRAM,
 562         },
 563         [ C(OP_PREFETCH) ] = {
 564                 [ C(RESULT_ACCESS) ] = 0x0,
 565                 [ C(RESULT_MISS)   ] = 0x0,
 566         },
 567  },
 568 };
 569
 570 #define SNB_DMND_DATA_RD        (1ULL << 0)
 571 #define SNB_DMND_RFO            (1ULL << 1)
 572 #define SNB_DMND_IFETCH         (1ULL << 2)
 573 #define SNB_DMND_WB             (1ULL << 3)
 574 #define SNB_PF_DATA_RD          (1ULL << 4)
 575 #define SNB_PF_RFO              (1ULL << 5)
 576 #define SNB_PF_IFETCH           (1ULL << 6)
 577 #define SNB_LLC_DATA_RD         (1ULL << 7)
 578 #define SNB_LLC_RFO             (1ULL << 8)
 579 #define SNB_LLC_IFETCH          (1ULL << 9)
 580 #define SNB_BUS_LOCKS           (1ULL << 10)
 581 #define SNB_STRM_ST             (1ULL << 11)
 582 #define SNB_OTHER               (1ULL << 15)
 583 #define SNB_RESP_ANY            (1ULL << 16)
 584 #define SNB_NO_SUPP             (1ULL << 17)
 585 #define SNB_LLC_HITM            (1ULL << 18)
 586 #define SNB_LLC_HITE            (1ULL << 19)
 587 #define SNB_LLC_HITS            (1ULL << 20)
 588 #define SNB_LLC_HITF            (1ULL << 21)
 589 #define SNB_LOCAL               (1ULL << 22)
 590 #define SNB_REMOTE              (0xffULL << 23)
 591 #define SNB_SNP_NONE            (1ULL << 31)
 592 #define SNB_SNP_NOT_NEEDED      (1ULL << 32)
 593 #define SNB_SNP_MISS            (1ULL << 33)
 594 #define SNB_NO_FWD              (1ULL << 34)
 595 #define SNB_SNP_FWD             (1ULL << 35)
 596 #define SNB_HITM                (1ULL << 36)
 597 #define SNB_NON_DRAM            (1ULL << 37)
 598
 599 #define SNB_DMND_READ           (SNB_DMND_DATA_RD|SNB_LLC_DATA_RD)
 600 #define SNB_DMND_WRITE          (SNB_DMND_RFO|SNB_LLC_RFO)
 601 #define SNB_DMND_PREFETCH       (SNB_PF_DATA_RD|SNB_PF_RFO)
 602
 603 #define SNB_SNP_ANY             (SNB_SNP_NONE|SNB_SNP_NOT_NEEDED| \
 604                                  SNB_SNP_MISS|SNB_NO_FWD|SNB_SNP_FWD| \
 605                                  SNB_HITM)
 606
 607 #define SNB_DRAM_ANY            (SNB_LOCAL|SNB_REMOTE|SNB_SNP_ANY)
 608 #define SNB_DRAM_REMOTE         (SNB_REMOTE|SNB_SNP_ANY)
 609
 610 #define SNB_L3_ACCESS           SNB_RESP_ANY
 611 #define SNB_L3_MISS             (SNB_DRAM_ANY|SNB_NON_DRAM)
 612
 613 static __initconst const u64 snb_hw_cache_extra_regs
 614                                 [PERF_COUNT_HW_CACHE_MAX]
 615                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 616                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 617 {
 618  [ C(LL  ) ] = {
 619         [ C(OP_READ) ] = {
 620                 [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_L3_ACCESS,
 621                 [ C(RESULT_MISS)   ] = SNB_DMND_READ|SNB_L3_MISS,
 622         },
 623         [ C(OP_WRITE) ] = {
 624                 [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_L3_ACCESS,
 625                 [ C(RESULT_MISS)   ] = SNB_DMND_WRITE|SNB_L3_MISS,
 626         },
 627         [ C(OP_PREFETCH) ] = {
 628                 [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_L3_ACCESS,
 629                 [ C(RESULT_MISS)   ] = SNB_DMND_PREFETCH|SNB_L3_MISS,
 630         },
 631  },
 632  [ C(NODE) ] = {
 633         [ C(OP_READ) ] = {
 634                 [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_DRAM_ANY,
 635                 [ C(RESULT_MISS)   ] = SNB_DMND_READ|SNB_DRAM_REMOTE,
 636         },
 637         [ C(OP_WRITE) ] = {
 638                 [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_DRAM_ANY,
 639                 [ C(RESULT_MISS)   ] = SNB_DMND_WRITE|SNB_DRAM_REMOTE,
 640         },
 641         [ C(OP_PREFETCH) ] = {
 642                 [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_DRAM_ANY,
 643                 [ C(RESULT_MISS)   ] = SNB_DMND_PREFETCH|SNB_DRAM_REMOTE,
 644         },
 645  },
 646 };
 647
 648 static __initconst const u64 snb_hw_cache_event_ids
 649                                 [PERF_COUNT_HW_CACHE_MAX]
 650                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 651                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 652 {
 653  [ C(L1D) ] = {
 654         [ C(OP_READ) ] = {
 655                 [ C(RESULT_ACCESS) ] = 0xf1d0, /* MEM_UOP_RETIRED.LOADS        */
 656                 [ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPLACEMENT              */
 657         },
 658         [ C(OP_WRITE) ] = {
 659                 [ C(RESULT_ACCESS) ] = 0xf2d0, /* MEM_UOP_RETIRED.STORES       */
 660                 [ C(RESULT_MISS)   ] = 0x0851, /* L1D.ALL_M_REPLACEMENT        */
 661         },
 662         [ C(OP_PREFETCH) ] = {
 663                 [ C(RESULT_ACCESS) ] = 0x0,
 664                 [ C(RESULT_MISS)   ] = 0x024e, /* HW_PRE_REQ.DL1_MISS          */
 665         },
 666  },
 667  [ C(L1I ) ] = {
 668         [ C(OP_READ) ] = {
 669                 [ C(RESULT_ACCESS) ] = 0x0,
 670                 [ C(RESULT_MISS)   ] = 0x0280, /* ICACHE.MISSES */
 671         },
 672         [ C(OP_WRITE) ] = {
 673                 [ C(RESULT_ACCESS) ] = -1,
 674                 [ C(RESULT_MISS)   ] = -1,
 675         },
 676         [ C(OP_PREFETCH) ] = {
 677                 [ C(RESULT_ACCESS) ] = 0x0,
 678                 [ C(RESULT_MISS)   ] = 0x0,
 679         },
 680  },
 681  [ C(LL  ) ] = {
 682         [ C(OP_READ) ] = {
 683                 /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
 684                 [ C(RESULT_ACCESS) ] = 0x01b7,
 685                 /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
 686                 [ C(RESULT_MISS)   ] = 0x01b7,
 687         },
 688         [ C(OP_WRITE) ] = {
 689                 /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
 690                 [ C(RESULT_ACCESS) ] = 0x01b7,
 691                 /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
 692                 [ C(RESULT_MISS)   ] = 0x01b7,
 693         },
 694         [ C(OP_PREFETCH) ] = {
 695                 /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
 696                 [ C(RESULT_ACCESS) ] = 0x01b7,
 697                 /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
 698                 [ C(RESULT_MISS)   ] = 0x01b7,
 699         },
 700  },
 701  [ C(DTLB) ] = {
 702         [ C(OP_READ) ] = {
 703                 [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */
 704                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */
 705         },
 706         [ C(OP_WRITE) ] = {
 707                 [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOP_RETIRED.ALL_STORES */
 708                 [ C(RESULT_MISS)   ] = 0x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
 709         },
 710         [ C(OP_PREFETCH) ] = {
 711                 [ C(RESULT_ACCESS) ] = 0x0,
 712                 [ C(RESULT_MISS)   ] = 0x0,
 713         },
 714  },
 715  [ C(ITLB) ] = {
 716         [ C(OP_READ) ] = {
 717                 [ C(RESULT_ACCESS) ] = 0x1085, /* ITLB_MISSES.STLB_HIT         */
 718                 [ C(RESULT_MISS)   ] = 0x0185, /* ITLB_MISSES.CAUSES_A_WALK    */
 719         },
 720         [ C(OP_WRITE) ] = {
 721                 [ C(RESULT_ACCESS) ] = -1,
 722                 [ C(RESULT_MISS)   ] = -1,
 723         },
 724         [ C(OP_PREFETCH) ] = {
 725                 [ C(RESULT_ACCESS) ] = -1,
 726                 [ C(RESULT_MISS)   ] = -1,
 727         },
 728  },
 729  [ C(BPU ) ] = {
 730         [ C(OP_READ) ] = {
 731                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 732                 [ C(RESULT_MISS)   ] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */
 733         },
 734         [ C(OP_WRITE) ] = {
 735                 [ C(RESULT_ACCESS) ] = -1,
 736                 [ C(RESULT_MISS)   ] = -1,
 737         },
 738         [ C(OP_PREFETCH) ] = {
 739                 [ C(RESULT_ACCESS) ] = -1,
 740                 [ C(RESULT_MISS)   ] = -1,
 741         },
 742  },
 743  [ C(NODE) ] = {
 744         [ C(OP_READ) ] = {
 745                 [ C(RESULT_ACCESS) ] = 0x01b7,
 746                 [ C(RESULT_MISS)   ] = 0x01b7,
 747         },
 748         [ C(OP_WRITE) ] = {
 749                 [ C(RESULT_ACCESS) ] = 0x01b7,
 750                 [ C(RESULT_MISS)   ] = 0x01b7,
 751         },
 752         [ C(OP_PREFETCH) ] = {
 753                 [ C(RESULT_ACCESS) ] = 0x01b7,
 754                 [ C(RESULT_MISS)   ] = 0x01b7,
 755         },
 756  },
 757
 758 };
 759
 760 /*
 761  * Notes on the events:
 762  * - data reads do not include code reads (comparable to earlier tables)
 763  * - data counts include speculative execution (except L1 write, dtlb, bpu)
 764  * - remote node access includes remote memory, remote cache, remote mmio.
 765  * - prefetches are not included in the counts because they are not
 766  *   reliably counted.
 767  */
 768
 769 #define HSW_DEMAND_DATA_RD              BIT_ULL(0)
 770 #define HSW_DEMAND_RFO                  BIT_ULL(1)
 771 #define HSW_ANY_RESPONSE                BIT_ULL(16)
 772 #define HSW_SUPPLIER_NONE               BIT_ULL(17)
 773 #define HSW_L3_MISS_LOCAL_DRAM          BIT_ULL(22)
 774 #define HSW_L3_MISS_REMOTE_HOP0         BIT_ULL(27)
 775 #define HSW_L3_MISS_REMOTE_HOP1         BIT_ULL(28)
 776 #define HSW_L3_MISS_REMOTE_HOP2P        BIT_ULL(29)
 777 #define HSW_L3_MISS                     (HSW_L3_MISS_LOCAL_DRAM| \
 778                                          HSW_L3_MISS_REMOTE_HOP0|HSW_L3_MISS_REMOTE_HOP1| \
 779                                          HSW_L3_MISS_REMOTE_HOP2P)
 780 #define HSW_SNOOP_NONE                  BIT_ULL(31)
 781 #define HSW_SNOOP_NOT_NEEDED            BIT_ULL(32)
 782 #define HSW_SNOOP_MISS                  BIT_ULL(33)
 783 #define HSW_SNOOP_HIT_NO_FWD            BIT_ULL(34)
 784 #define HSW_SNOOP_HIT_WITH_FWD          BIT_ULL(35)
 785 #define HSW_SNOOP_HITM                  BIT_ULL(36)
 786 #define HSW_SNOOP_NON_DRAM              BIT_ULL(37)
 787 #define HSW_ANY_SNOOP                   (HSW_SNOOP_NONE| \
 788                                          HSW_SNOOP_NOT_NEEDED|HSW_SNOOP_MISS| \
 789                                          HSW_SNOOP_HIT_NO_FWD|HSW_SNOOP_HIT_WITH_FWD| \
 790                                          HSW_SNOOP_HITM|HSW_SNOOP_NON_DRAM)
 791 #define HSW_SNOOP_DRAM                  (HSW_ANY_SNOOP & ~HSW_SNOOP_NON_DRAM)
 792 #define HSW_DEMAND_READ                 HSW_DEMAND_DATA_RD
 793 #define HSW_DEMAND_WRITE                HSW_DEMAND_RFO
 794 #define HSW_L3_MISS_REMOTE              (HSW_L3_MISS_REMOTE_HOP0|\
 795                                          HSW_L3_MISS_REMOTE_HOP1|HSW_L3_MISS_REMOTE_HOP2P)
 796 #define HSW_LLC_ACCESS                  HSW_ANY_RESPONSE
 797
 798 #define BDW_L3_MISS_LOCAL               BIT(26)
 799 #define BDW_L3_MISS                     (BDW_L3_MISS_LOCAL| \
 800                                          HSW_L3_MISS_REMOTE_HOP0|HSW_L3_MISS_REMOTE_HOP1| \
 801                                          HSW_L3_MISS_REMOTE_HOP2P)
 802
 803
 804 static __initconst const u64 hsw_hw_cache_event_ids
 805                                 [PERF_COUNT_HW_CACHE_MAX]
 806                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 807                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 808 {
 809  [ C(L1D ) ] = {
 810         [ C(OP_READ) ] = {
 811                 [ C(RESULT_ACCESS) ] = 0x81d0,  /* MEM_UOPS_RETIRED.ALL_LOADS */
 812                 [ C(RESULT_MISS)   ] = 0x151,   /* L1D.REPLACEMENT */
 813         },
 814         [ C(OP_WRITE) ] = {
 815                 [ C(RESULT_ACCESS) ] = 0x82d0,  /* MEM_UOPS_RETIRED.ALL_STORES */
 816                 [ C(RESULT_MISS)   ] = 0x0,
 817         },
 818         [ C(OP_PREFETCH) ] = {
 819                 [ C(RESULT_ACCESS) ] = 0x0,
 820                 [ C(RESULT_MISS)   ] = 0x0,
 821         },
 822  },
 823  [ C(L1I ) ] = {
 824         [ C(OP_READ) ] = {
 825                 [ C(RESULT_ACCESS) ] = 0x0,
 826                 [ C(RESULT_MISS)   ] = 0x280,   /* ICACHE.MISSES */
 827         },
 828         [ C(OP_WRITE) ] = {
 829                 [ C(RESULT_ACCESS) ] = -1,
 830                 [ C(RESULT_MISS)   ] = -1,
 831         },
 832         [ C(OP_PREFETCH) ] = {
 833                 [ C(RESULT_ACCESS) ] = 0x0,
 834                 [ C(RESULT_MISS)   ] = 0x0,
 835         },
 836  },
 837  [ C(LL  ) ] = {
 838         [ C(OP_READ) ] = {
 839                 [ C(RESULT_ACCESS) ] = 0x1b7,   /* OFFCORE_RESPONSE */
 840                 [ C(RESULT_MISS)   ] = 0x1b7,   /* OFFCORE_RESPONSE */
 841         },
 842         [ C(OP_WRITE) ] = {
 843                 [ C(RESULT_ACCESS) ] = 0x1b7,   /* OFFCORE_RESPONSE */
 844                 [ C(RESULT_MISS)   ] = 0x1b7,   /* OFFCORE_RESPONSE */
 845         },
 846         [ C(OP_PREFETCH) ] = {
 847                 [ C(RESULT_ACCESS) ] = 0x0,
 848                 [ C(RESULT_MISS)   ] = 0x0,
 849         },
 850  },
 851  [ C(DTLB) ] = {
 852         [ C(OP_READ) ] = {
 853                 [ C(RESULT_ACCESS) ] = 0x81d0,  /* MEM_UOPS_RETIRED.ALL_LOADS */
 854                 [ C(RESULT_MISS)   ] = 0x108,   /* DTLB_LOAD_MISSES.MISS_CAUSES_A_WALK */
 855         },
 856         [ C(OP_WRITE) ] = {
 857                 [ C(RESULT_ACCESS) ] = 0x82d0,  /* MEM_UOPS_RETIRED.ALL_STORES */
 858                 [ C(RESULT_MISS)   ] = 0x149,   /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
 859         },
 860         [ C(OP_PREFETCH) ] = {
 861                 [ C(RESULT_ACCESS) ] = 0x0,
 862                 [ C(RESULT_MISS)   ] = 0x0,
 863         },
 864  },
 865  [ C(ITLB) ] = {
 866         [ C(OP_READ) ] = {
 867                 [ C(RESULT_ACCESS) ] = 0x6085,  /* ITLB_MISSES.STLB_HIT */
 868                 [ C(RESULT_MISS)   ] = 0x185,   /* ITLB_MISSES.MISS_CAUSES_A_WALK */
 869         },
 870         [ C(OP_WRITE) ] = {
 871                 [ C(RESULT_ACCESS) ] = -1,
 872                 [ C(RESULT_MISS)   ] = -1,
 873         },
 874         [ C(OP_PREFETCH) ] = {
 875                 [ C(RESULT_ACCESS) ] = -1,
 876                 [ C(RESULT_MISS)   ] = -1,
 877         },
 878  },
 879  [ C(BPU ) ] = {
 880         [ C(OP_READ) ] = {
 881                 [ C(RESULT_ACCESS) ] = 0xc4,    /* BR_INST_RETIRED.ALL_BRANCHES */
 882                 [ C(RESULT_MISS)   ] = 0xc5,    /* BR_MISP_RETIRED.ALL_BRANCHES */
 883         },
 884         [ C(OP_WRITE) ] = {
 885                 [ C(RESULT_ACCESS) ] = -1,
 886                 [ C(RESULT_MISS)   ] = -1,
 887         },
 888         [ C(OP_PREFETCH) ] = {
 889                 [ C(RESULT_ACCESS) ] = -1,
 890                 [ C(RESULT_MISS)   ] = -1,
 891         },
 892  },
 893  [ C(NODE) ] = {
 894         [ C(OP_READ) ] = {
 895                 [ C(RESULT_ACCESS) ] = 0x1b7,   /* OFFCORE_RESPONSE */
 896                 [ C(RESULT_MISS)   ] = 0x1b7,   /* OFFCORE_RESPONSE */
 897         },
 898         [ C(OP_WRITE) ] = {
 899                 [ C(RESULT_ACCESS) ] = 0x1b7,   /* OFFCORE_RESPONSE */
 900                 [ C(RESULT_MISS)   ] = 0x1b7,   /* OFFCORE_RESPONSE */
 901         },
 902         [ C(OP_PREFETCH) ] = {
 903                 [ C(RESULT_ACCESS) ] = 0x0,
 904                 [ C(RESULT_MISS)   ] = 0x0,
 905         },
 906  },
 907 };
 908
 909 static __initconst const u64 hsw_hw_cache_extra_regs
 910                                 [PERF_COUNT_HW_CACHE_MAX]
 911                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 912                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 913 {
 914  [ C(LL  ) ] = {
 915         [ C(OP_READ) ] = {
 916                 [ C(RESULT_ACCESS) ] = HSW_DEMAND_READ|
 917                                        HSW_LLC_ACCESS,
 918                 [ C(RESULT_MISS)   ] = HSW_DEMAND_READ|
 919                                        HSW_L3_MISS|HSW_ANY_SNOOP,
 920         },
 921         [ C(OP_WRITE) ] = {
 922                 [ C(RESULT_ACCESS) ] = HSW_DEMAND_WRITE|
 923                                        HSW_LLC_ACCESS,
 924                 [ C(RESULT_MISS)   ] = HSW_DEMAND_WRITE|
 925                                        HSW_L3_MISS|HSW_ANY_SNOOP,
 926         },
 927         [ C(OP_PREFETCH) ] = {
 928                 [ C(RESULT_ACCESS) ] = 0x0,
 929                 [ C(RESULT_MISS)   ] = 0x0,
 930         },
 931  },
 932  [ C(NODE) ] = {
 933         [ C(OP_READ) ] = {
 934                 [ C(RESULT_ACCESS) ] = HSW_DEMAND_READ|
 935                                        HSW_L3_MISS_LOCAL_DRAM|
 936                                        HSW_SNOOP_DRAM,
 937                 [ C(RESULT_MISS)   ] = HSW_DEMAND_READ|
 938                                        HSW_L3_MISS_REMOTE|
 939                                        HSW_SNOOP_DRAM,
 940         },
 941         [ C(OP_WRITE) ] = {
 942                 [ C(RESULT_ACCESS) ] = HSW_DEMAND_WRITE|
 943                                        HSW_L3_MISS_LOCAL_DRAM|
 944                                        HSW_SNOOP_DRAM,
 945                 [ C(RESULT_MISS)   ] = HSW_DEMAND_WRITE|
 946                                        HSW_L3_MISS_REMOTE|
 947                                        HSW_SNOOP_DRAM,
 948         },
 949         [ C(OP_PREFETCH) ] = {
 950                 [ C(RESULT_ACCESS) ] = 0x0,
 951                 [ C(RESULT_MISS)   ] = 0x0,
 952         },
 953  },
 954 };
 955
 956 static __initconst const u64 westmere_hw_cache_event_ids
 957                                 [PERF_COUNT_HW_CACHE_MAX]
 958                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 959                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 960 {
 961  [ C(L1D) ] = {
 962         [ C(OP_READ) ] = {
 963                 [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
 964                 [ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPL                     */
 965         },
 966         [ C(OP_WRITE) ] = {
 967                 [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
 968                 [ C(RESULT_MISS)   ] = 0x0251, /* L1D.M_REPL                   */
 969         },
 970         [ C(OP_PREFETCH) ] = {
 971                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 972                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 973         },
 974  },
 975  [ C(L1I ) ] = {
 976         [ C(OP_READ) ] = {
 977                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 978                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 979         },
 980         [ C(OP_WRITE) ] = {
 981                 [ C(RESULT_ACCESS) ] = -1,
 982                 [ C(RESULT_MISS)   ] = -1,
 983         },
 984         [ C(OP_PREFETCH) ] = {
 985                 [ C(RESULT_ACCESS) ] = 0x0,
 986                 [ C(RESULT_MISS)   ] = 0x0,
 987         },
 988  },
 989  [ C(LL  ) ] = {
 990         [ C(OP_READ) ] = {
 991                 /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
 992                 [ C(RESULT_ACCESS) ] = 0x01b7,
 993                 /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
 994                 [ C(RESULT_MISS)   ] = 0x01b7,
 995         },
 996         /*
 997          * Use RFO, not WRITEBACK, because a write miss would typically occur
 998          * on RFO.
 999          */
1000         [ C(OP_WRITE) ] = {
1001                 /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
1002                 [ C(RESULT_ACCESS) ] = 0x01b7,
1003                 /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
1004                 [ C(RESULT_MISS)   ] = 0x01b7,
1005         },
1006         [ C(OP_PREFETCH) ] = {
1007                 /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
1008                 [ C(RESULT_ACCESS) ] = 0x01b7,
1009                 /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
1010                 [ C(RESULT_MISS)   ] = 0x01b7,
1011         },
1012  },
1013  [ C(DTLB) ] = {
1014         [ C(OP_READ) ] = {
1015                 [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
1016                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
1017         },
1018         [ C(OP_WRITE) ] = {
1019                 [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
1020                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
1021         },
1022         [ C(OP_PREFETCH) ] = {
1023                 [ C(RESULT_ACCESS) ] = 0x0,
1024                 [ C(RESULT_MISS)   ] = 0x0,
1025         },
1026  },
1027  [ C(ITLB) ] = {
1028         [ C(OP_READ) ] = {
1029                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
1030                 [ C(RESULT_MISS)   ] = 0x0185, /* ITLB_MISSES.ANY              */
1031         },
1032         [ C(OP_WRITE) ] = {
1033                 [ C(RESULT_ACCESS) ] = -1,
1034                 [ C(RESULT_MISS)   ] = -1,
1035         },
1036         [ C(OP_PREFETCH) ] = {
1037                 [ C(RESULT_ACCESS) ] = -1,
1038                 [ C(RESULT_MISS)   ] = -1,
1039         },
1040  },
1041  [ C(BPU ) ] = {
1042         [ C(OP_READ) ] = {
1043                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
1044                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
1045         },
1046         [ C(OP_WRITE) ] = {
1047                 [ C(RESULT_ACCESS) ] = -1,
1048                 [ C(RESULT_MISS)   ] = -1,
1049         },
1050         [ C(OP_PREFETCH) ] = {
1051                 [ C(RESULT_ACCESS) ] = -1,
1052                 [ C(RESULT_MISS)   ] = -1,
1053         },
1054  },
1055  [ C(NODE) ] = {
1056         [ C(OP_READ) ] = {
1057                 [ C(RESULT_ACCESS) ] = 0x01b7,
1058                 [ C(RESULT_MISS)   ] = 0x01b7,
1059         },
1060         [ C(OP_WRITE) ] = {
1061                 [ C(RESULT_ACCESS) ] = 0x01b7,
1062                 [ C(RESULT_MISS)   ] = 0x01b7,
1063         },
1064         [ C(OP_PREFETCH) ] = {
1065                 [ C(RESULT_ACCESS) ] = 0x01b7,
1066                 [ C(RESULT_MISS)   ] = 0x01b7,
1067         },
1068  },
1069 };
1070
1071 /*
1072  * Nehalem/Westmere MSR_OFFCORE_RESPONSE bits;
1073  * See IA32 SDM Vol 3B 30.6.1.3
1074  */
1075
1076 #define NHM_DMND_DATA_RD        (1 << 0)
1077 #define NHM_DMND_RFO            (1 << 1)
1078 #define NHM_DMND_IFETCH         (1 << 2)
1079 #define NHM_DMND_WB             (1 << 3)
1080 #define NHM_PF_DATA_RD          (1 << 4)
1081 #define NHM_PF_DATA_RFO         (1 << 5)
1082 #define NHM_PF_IFETCH           (1 << 6)
1083 #define NHM_OFFCORE_OTHER       (1 << 7)
1084 #define NHM_UNCORE_HIT          (1 << 8)
1085 #define NHM_OTHER_CORE_HIT_SNP  (1 << 9)
1086 #define NHM_OTHER_CORE_HITM     (1 << 10)
1087                                 /* reserved */
1088 #define NHM_REMOTE_CACHE_FWD    (1 << 12)
1089 #define NHM_REMOTE_DRAM         (1 << 13)
1090 #define NHM_LOCAL_DRAM          (1 << 14)
1091 #define NHM_NON_DRAM            (1 << 15)
1092
1093 #define NHM_LOCAL               (NHM_LOCAL_DRAM|NHM_REMOTE_CACHE_FWD)
1094 #define NHM_REMOTE              (NHM_REMOTE_DRAM)
1095
1096 #define NHM_DMND_READ           (NHM_DMND_DATA_RD)
1097 #define NHM_DMND_WRITE          (NHM_DMND_RFO|NHM_DMND_WB)
1098 #define NHM_DMND_PREFETCH       (NHM_PF_DATA_RD|NHM_PF_DATA_RFO)
1099
1100 #define NHM_L3_HIT      (NHM_UNCORE_HIT|NHM_OTHER_CORE_HIT_SNP|NHM_OTHER_CORE_HITM)
1101 #define NHM_L3_MISS     (NHM_NON_DRAM|NHM_LOCAL_DRAM|NHM_REMOTE_DRAM|NHM_REMOTE_CACHE_FWD)
1102 #define NHM_L3_ACCESS   (NHM_L3_HIT|NHM_L3_MISS)
1103
1104 static __initconst const u64 nehalem_hw_cache_extra_regs
1105                                 [PERF_COUNT_HW_CACHE_MAX]
1106                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1107                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
1108 {
1109  [ C(LL  ) ] = {
1110         [ C(OP_READ) ] = {
1111                 [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_L3_ACCESS,
1112                 [ C(RESULT_MISS)   ] = NHM_DMND_READ|NHM_L3_MISS,
1113         },
1114         [ C(OP_WRITE) ] = {
1115                 [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_L3_ACCESS,
1116                 [ C(RESULT_MISS)   ] = NHM_DMND_WRITE|NHM_L3_MISS,
1117         },
1118         [ C(OP_PREFETCH) ] = {
1119                 [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_L3_ACCESS,
1120                 [ C(RESULT_MISS)   ] = NHM_DMND_PREFETCH|NHM_L3_MISS,
1121         },
1122  },
1123  [ C(NODE) ] = {
1124         [ C(OP_READ) ] = {
1125                 [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_LOCAL|NHM_REMOTE,
1126                 [ C(RESULT_MISS)   ] = NHM_DMND_READ|NHM_REMOTE,
1127         },
1128         [ C(OP_WRITE) ] = {
1129                 [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_LOCAL|NHM_REMOTE,
1130                 [ C(RESULT_MISS)   ] = NHM_DMND_WRITE|NHM_REMOTE,
1131         },
1132         [ C(OP_PREFETCH) ] = {
1133                 [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_LOCAL|NHM_REMOTE,
1134                 [ C(RESULT_MISS)   ] = NHM_DMND_PREFETCH|NHM_REMOTE,
1135         },
1136  },
1137 };
1138
1139 static __initconst const u64 nehalem_hw_cache_event_ids
1140                                 [PERF_COUNT_HW_CACHE_MAX]
1141                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1142                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
1143 {
1144  [ C(L1D) ] = {
1145         [ C(OP_READ) ] = {
1146                 [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
1147                 [ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPL                     */
1148         },
1149         [ C(OP_WRITE) ] = {
1150                 [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
1151                 [ C(RESULT_MISS)   ] = 0x0251, /* L1D.M_REPL                   */
1152         },
1153         [ C(OP_PREFETCH) ] = {
1154                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
1155                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
1156         },
1157  },
1158  [ C(L1I ) ] = {
1159         [ C(OP_READ) ] = {
1160                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
1161                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
1162         },
1163         [ C(OP_WRITE) ] = {
1164                 [ C(RESULT_ACCESS) ] = -1,
1165                 [ C(RESULT_MISS)   ] = -1,
1166         },
1167         [ C(OP_PREFETCH) ] = {
1168                 [ C(RESULT_ACCESS) ] = 0x0,
1169                 [ C(RESULT_MISS)   ] = 0x0,
1170         },
1171  },
1172  [ C(LL  ) ] = {
1173         [ C(OP_READ) ] = {
1174                 /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
1175                 [ C(RESULT_ACCESS) ] = 0x01b7,
1176                 /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
1177                 [ C(RESULT_MISS)   ] = 0x01b7,
1178         },
1179         /*
1180          * Use RFO, not WRITEBACK, because a write miss would typically occur
1181          * on RFO.
1182          */
1183         [ C(OP_WRITE) ] = {
1184                 /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
1185                 [ C(RESULT_ACCESS) ] = 0x01b7,
1186                 /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
1187                 [ C(RESULT_MISS)   ] = 0x01b7,
1188         },
1189         [ C(OP_PREFETCH) ] = {
1190                 /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
1191                 [ C(RESULT_ACCESS) ] = 0x01b7,
1192                 /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
1193                 [ C(RESULT_MISS)   ] = 0x01b7,
1194         },
1195  },
1196  [ C(DTLB) ] = {
1197         [ C(OP_READ) ] = {
1198                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
1199                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
1200         },
1201         [ C(OP_WRITE) ] = {
1202                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
1203                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
1204         },
1205         [ C(OP_PREFETCH) ] = {
1206                 [ C(RESULT_ACCESS) ] = 0x0,
1207                 [ C(RESULT_MISS)   ] = 0x0,
1208         },
1209  },
1210  [ C(ITLB) ] = {
1211         [ C(OP_READ) ] = {
1212                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
1213                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
1214         },
1215         [ C(OP_WRITE) ] = {
1216                 [ C(RESULT_ACCESS) ] = -1,
1217                 [ C(RESULT_MISS)   ] = -1,
1218         },
1219         [ C(OP_PREFETCH) ] = {
1220                 [ C(RESULT_ACCESS) ] = -1,
1221                 [ C(RESULT_MISS)   ] = -1,
1222         },
1223  },
1224  [ C(BPU ) ] = {
1225         [ C(OP_READ) ] = {
1226                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
1227                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
1228         },
1229         [ C(OP_WRITE) ] = {
1230                 [ C(RESULT_ACCESS) ] = -1,
1231                 [ C(RESULT_MISS)   ] = -1,
1232         },
1233         [ C(OP_PREFETCH) ] = {
1234                 [ C(RESULT_ACCESS) ] = -1,
1235                 [ C(RESULT_MISS)   ] = -1,
1236         },
1237  },
1238  [ C(NODE) ] = {
1239         [ C(OP_READ) ] = {
1240                 [ C(RESULT_ACCESS) ] = 0x01b7,
1241                 [ C(RESULT_MISS)   ] = 0x01b7,
1242         },
1243         [ C(OP_WRITE) ] = {
1244                 [ C(RESULT_ACCESS) ] = 0x01b7,
1245                 [ C(RESULT_MISS)   ] = 0x01b7,
1246         },
1247         [ C(OP_PREFETCH) ] = {
1248                 [ C(RESULT_ACCESS) ] = 0x01b7,
1249                 [ C(RESULT_MISS)   ] = 0x01b7,
1250         },
1251  },
1252 };
1253
1254 static __initconst const u64 core2_hw_cache_event_ids
1255                                 [PERF_COUNT_HW_CACHE_MAX]
1256                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1257                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
1258 {
1259  [ C(L1D) ] = {
1260         [ C(OP_READ) ] = {
1261                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
1262                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
1263         },
1264         [ C(OP_WRITE) ] = {
1265                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
1266                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
1267         },
1268         [ C(OP_PREFETCH) ] = {
1269                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
1270                 [ C(RESULT_MISS)   ] = 0,
1271         },
1272  },
1273  [ C(L1I ) ] = {
1274         [ C(OP_READ) ] = {
1275                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
1276                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
1277         },
1278         [ C(OP_WRITE) ] = {
1279                 [ C(RESULT_ACCESS) ] = -1,
1280                 [ C(RESULT_MISS)   ] = -1,
1281         },
1282         [ C(OP_PREFETCH) ] = {
1283                 [ C(RESULT_ACCESS) ] = 0,
1284                 [ C(RESULT_MISS)   ] = 0,
1285         },
1286  },
1287  [ C(LL  ) ] = {
1288         [ C(OP_READ) ] = {
1289                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
1290                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
1291         },
1292         [ C(OP_WRITE) ] = {
1293                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
1294                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
1295         },
1296         [ C(OP_PREFETCH) ] = {
1297                 [ C(RESULT_ACCESS) ] = 0,
1298                 [ C(RESULT_MISS)   ] = 0,
1299         },
1300  },
1301  [ C(DTLB) ] = {
1302         [ C(OP_READ) ] = {
1303                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
1304                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
1305         },
1306         [ C(OP_WRITE) ] = {
1307                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
1308                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
1309         },
1310         [ C(OP_PREFETCH) ] = {
1311                 [ C(RESULT_ACCESS) ] = 0,
1312                 [ C(RESULT_MISS)   ] = 0,
1313         },
1314  },
1315  [ C(ITLB) ] = {
1316         [ C(OP_READ) ] = {
1317                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
1318                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
1319         },
1320         [ C(OP_WRITE) ] = {
1321                 [ C(RESULT_ACCESS) ] = -1,
1322                 [ C(RESULT_MISS)   ] = -1,
1323         },
1324         [ C(OP_PREFETCH) ] = {
1325                 [ C(RESULT_ACCESS) ] = -1,
1326                 [ C(RESULT_MISS)   ] = -1,
1327         },
1328  },
1329  [ C(BPU ) ] = {
1330         [ C(OP_READ) ] = {
1331                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
1332                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
1333         },
1334         [ C(OP_WRITE) ] = {
1335                 [ C(RESULT_ACCESS) ] = -1,
1336                 [ C(RESULT_MISS)   ] = -1,
1337         },
1338         [ C(OP_PREFETCH) ] = {
1339                 [ C(RESULT_ACCESS) ] = -1,
1340                 [ C(RESULT_MISS)   ] = -1,
1341         },
1342  },
1343 };
1344
1345 static __initconst const u64 atom_hw_cache_event_ids
1346                                 [PERF_COUNT_HW_CACHE_MAX]
1347                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1348                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
1349 {
1350  [ C(L1D) ] = {
1351         [ C(OP_READ) ] = {
1352                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
1353                 [ C(RESULT_MISS)   ] = 0,
1354         },
1355         [ C(OP_WRITE) ] = {
1356                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
1357                 [ C(RESULT_MISS)   ] = 0,
1358         },
1359         [ C(OP_PREFETCH) ] = {
1360                 [ C(RESULT_ACCESS) ] = 0x0,
1361                 [ C(RESULT_MISS)   ] = 0,
1362         },
1363  },
1364  [ C(L1I ) ] = {
1365         [ C(OP_READ) ] = {
1366                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
1367                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
1368         },
1369         [ C(OP_WRITE) ] = {
1370                 [ C(RESULT_ACCESS) ] = -1,
1371                 [ C(RESULT_MISS)   ] = -1,
1372         },
1373         [ C(OP_PREFETCH) ] = {
1374                 [ C(RESULT_ACCESS) ] = 0,
1375                 [ C(RESULT_MISS)   ] = 0,
1376         },
1377  },
1378  [ C(LL  ) ] = {
1379         [ C(OP_READ) ] = {
1380                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
1381                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
1382         },
1383         [ C(OP_WRITE) ] = {
1384                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
1385                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
1386         },
1387         [ C(OP_PREFETCH) ] = {
1388                 [ C(RESULT_ACCESS) ] = 0,
1389                 [ C(RESULT_MISS)   ] = 0,
1390         },
1391  },
1392  [ C(DTLB) ] = {
1393         [ C(OP_READ) ] = {
1394                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
1395                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
1396         },
1397         [ C(OP_WRITE) ] = {
1398                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
1399                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
1400         },
1401         [ C(OP_PREFETCH) ] = {
1402                 [ C(RESULT_ACCESS) ] = 0,
1403                 [ C(RESULT_MISS)   ] = 0,
1404         },
1405  },
1406  [ C(ITLB) ] = {
1407         [ C(OP_READ) ] = {
1408                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
1409                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
1410         },
1411         [ C(OP_WRITE) ] = {
1412                 [ C(RESULT_ACCESS) ] = -1,
1413                 [ C(RESULT_MISS)   ] = -1,
1414         },
1415         [ C(OP_PREFETCH) ] = {
1416                 [ C(RESULT_ACCESS) ] = -1,
1417                 [ C(RESULT_MISS)   ] = -1,
1418         },
1419  },
1420  [ C(BPU ) ] = {
1421         [ C(OP_READ) ] = {
1422                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
1423                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
1424         },
1425         [ C(OP_WRITE) ] = {
1426                 [ C(RESULT_ACCESS) ] = -1,
1427                 [ C(RESULT_MISS)   ] = -1,
1428         },
1429         [ C(OP_PREFETCH) ] = {
1430                 [ C(RESULT_ACCESS) ] = -1,
1431                 [ C(RESULT_MISS)   ] = -1,
1432         },
1433  },
1434 };
1435
1436 EVENT_ATTR_STR(topdown-total-slots, td_total_slots_slm, "event=0x3c");
1437 EVENT_ATTR_STR(topdown-total-slots.scale, td_total_slots_scale_slm, "2");
1438 /* no_alloc_cycles.not_delivered */
1439 EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles_slm,
1440                "event=0xca,umask=0x50");
1441 EVENT_ATTR_STR(topdown-fetch-bubbles.scale, td_fetch_bubbles_scale_slm, "2");
1442 /* uops_retired.all */
1443 EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued_slm,
1444                "event=0xc2,umask=0x10");
1445 /* uops_retired.all */
1446 EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired_slm,
1447                "event=0xc2,umask=0x10");
1448
1449 static struct attribute *slm_events_attrs[] = {
1450         EVENT_PTR(td_total_slots_slm),
1451         EVENT_PTR(td_total_slots_scale_slm),
1452         EVENT_PTR(td_fetch_bubbles_slm),
1453         EVENT_PTR(td_fetch_bubbles_scale_slm),
1454         EVENT_PTR(td_slots_issued_slm),
1455         EVENT_PTR(td_slots_retired_slm),
1456         NULL
1457 };
1458
1459 static struct extra_reg intel_slm_extra_regs[] __read_mostly =
1460 {
1461         /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
1462         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x768005ffffull, RSP_0),
1463         INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x368005ffffull, RSP_1),
1464         EVENT_EXTRA_END
1465 };
1466
1467 #define SLM_DMND_READ           SNB_DMND_DATA_RD
1468 #define SLM_DMND_WRITE          SNB_DMND_RFO
1469 #define SLM_DMND_PREFETCH       (SNB_PF_DATA_RD|SNB_PF_RFO)
1470
1471 #define SLM_SNP_ANY             (SNB_SNP_NONE|SNB_SNP_MISS|SNB_NO_FWD|SNB_HITM)
1472 #define SLM_LLC_ACCESS          SNB_RESP_ANY
1473 #define SLM_LLC_MISS            (SLM_SNP_ANY|SNB_NON_DRAM)
1474
1475 static __initconst const u64 slm_hw_cache_extra_regs
1476                                 [PERF_COUNT_HW_CACHE_MAX]
1477                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1478                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
1479 {
1480  [ C(LL  ) ] = {
1481         [ C(OP_READ) ] = {
1482                 [ C(RESULT_ACCESS) ] = SLM_DMND_READ|SLM_LLC_ACCESS,
1483                 [ C(RESULT_MISS)   ] = 0,
1484         },
1485         [ C(OP_WRITE) ] = {
1486                 [ C(RESULT_ACCESS) ] = SLM_DMND_WRITE|SLM_LLC_ACCESS,
1487                 [ C(RESULT_MISS)   ] = SLM_DMND_WRITE|SLM_LLC_MISS,
1488         },
1489         [ C(OP_PREFETCH) ] = {
1490                 [ C(RESULT_ACCESS) ] = SLM_DMND_PREFETCH|SLM_LLC_ACCESS,
1491                 [ C(RESULT_MISS)   ] = SLM_DMND_PREFETCH|SLM_LLC_MISS,
1492         },
1493  },
1494 };
1495
1496 static __initconst const u64 slm_hw_cache_event_ids
1497                                 [PERF_COUNT_HW_CACHE_MAX]
1498                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1499                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
1500 {
1501  [ C(L1D) ] = {
1502         [ C(OP_READ) ] = {
1503                 [ C(RESULT_ACCESS) ] = 0,
1504                 [ C(RESULT_MISS)   ] = 0x0104, /* LD_DCU_MISS */
1505         },
1506         [ C(OP_WRITE) ] = {
1507                 [ C(RESULT_ACCESS) ] = 0,
1508                 [ C(RESULT_MISS)   ] = 0,
1509         },
1510         [ C(OP_PREFETCH) ] = {
1511                 [ C(RESULT_ACCESS) ] = 0,
1512                 [ C(RESULT_MISS)   ] = 0,
1513         },
1514  },
1515  [ C(L1I ) ] = {
1516         [ C(OP_READ) ] = {
1517                 [ C(RESULT_ACCESS) ] = 0x0380, /* ICACHE.ACCESSES */
1518                 [ C(RESULT_MISS)   ] = 0x0280, /* ICACGE.MISSES */
1519         },
1520         [ C(OP_WRITE) ] = {
1521                 [ C(RESULT_ACCESS) ] = -1,
1522                 [ C(RESULT_MISS)   ] = -1,
1523         },
1524         [ C(OP_PREFETCH) ] = {
1525                 [ C(RESULT_ACCESS) ] = 0,
1526                 [ C(RESULT_MISS)   ] = 0,
1527         },
1528  },
1529  [ C(LL  ) ] = {
1530         [ C(OP_READ) ] = {
1531                 /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
1532                 [ C(RESULT_ACCESS) ] = 0x01b7,
1533                 [ C(RESULT_MISS)   ] = 0,
1534         },
1535         [ C(OP_WRITE) ] = {
1536                 /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
1537                 [ C(RESULT_ACCESS) ] = 0x01b7,
1538                 /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
1539                 [ C(RESULT_MISS)   ] = 0x01b7,
1540         },
1541         [ C(OP_PREFETCH) ] = {
1542                 /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
1543                 [ C(RESULT_ACCESS) ] = 0x01b7,
1544                 /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
1545                 [ C(RESULT_MISS)   ] = 0x01b7,
1546         },
1547  },
1548  [ C(DTLB) ] = {
1549         [ C(OP_READ) ] = {
1550                 [ C(RESULT_ACCESS) ] = 0,
1551                 [ C(RESULT_MISS)   ] = 0x0804, /* LD_DTLB_MISS */
1552         },
1553         [ C(OP_WRITE) ] = {
1554                 [ C(RESULT_ACCESS) ] = 0,
1555                 [ C(RESULT_MISS)   ] = 0,
1556         },
1557         [ C(OP_PREFETCH) ] = {
1558                 [ C(RESULT_ACCESS) ] = 0,
1559                 [ C(RESULT_MISS)   ] = 0,
1560         },
1561  },
1562  [ C(ITLB) ] = {
1563         [ C(OP_READ) ] = {
1564                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
1565                 [ C(RESULT_MISS)   ] = 0x40205, /* PAGE_WALKS.I_SIDE_WALKS */
1566         },
1567         [ C(OP_WRITE) ] = {
1568                 [ C(RESULT_ACCESS) ] = -1,
1569                 [ C(RESULT_MISS)   ] = -1,
1570         },
1571         [ C(OP_PREFETCH) ] = {
1572                 [ C(RESULT_ACCESS) ] = -1,
1573                 [ C(RESULT_MISS)   ] = -1,
1574         },
1575  },
1576  [ C(BPU ) ] = {
1577         [ C(OP_READ) ] = {
1578                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
1579                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
1580         },
1581         [ C(OP_WRITE) ] = {
1582                 [ C(RESULT_ACCESS) ] = -1,
1583                 [ C(RESULT_MISS)   ] = -1,
1584         },
1585         [ C(OP_PREFETCH) ] = {
1586                 [ C(RESULT_ACCESS) ] = -1,
1587                 [ C(RESULT_MISS)   ] = -1,
1588         },
1589  },
1590 };
1591
1592 EVENT_ATTR_STR(topdown-total-slots, td_total_slots_glm, "event=0x3c");
1593 EVENT_ATTR_STR(topdown-total-slots.scale, td_total_slots_scale_glm, "3");
1594 /* UOPS_NOT_DELIVERED.ANY */
1595 EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles_glm, "event=0x9c");
1596 /* ISSUE_SLOTS_NOT_CONSUMED.RECOVERY */
1597 EVENT_ATTR_STR(topdown-recovery-bubbles, td_recovery_bubbles_glm, "event=0xca,umask=0x02");
1598 /* UOPS_RETIRED.ANY */
1599 EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired_glm, "event=0xc2");
1600 /* UOPS_ISSUED.ANY */
1601 EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued_glm, "event=0x0e");
1602
1603 static struct attribute *glm_events_attrs[] = {
1604         EVENT_PTR(td_total_slots_glm),
1605         EVENT_PTR(td_total_slots_scale_glm),
1606         EVENT_PTR(td_fetch_bubbles_glm),
1607         EVENT_PTR(td_recovery_bubbles_glm),
1608         EVENT_PTR(td_slots_issued_glm),
1609         EVENT_PTR(td_slots_retired_glm),
1610         NULL
1611 };
1612
1613 static struct extra_reg intel_glm_extra_regs[] __read_mostly = {
1614         /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
1615         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x760005ffbfull, RSP_0),
1616         INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x360005ffbfull, RSP_1),
1617         EVENT_EXTRA_END
1618 };
1619
1620 #define GLM_DEMAND_DATA_RD              BIT_ULL(0)
1621 #define GLM_DEMAND_RFO                  BIT_ULL(1)
1622 #define GLM_ANY_RESPONSE                BIT_ULL(16)
1623 #define GLM_SNP_NONE_OR_MISS            BIT_ULL(33)
1624 #define GLM_DEMAND_READ                 GLM_DEMAND_DATA_RD
1625 #define GLM_DEMAND_WRITE                GLM_DEMAND_RFO
1626 #define GLM_DEMAND_PREFETCH             (SNB_PF_DATA_RD|SNB_PF_RFO)
1627 #define GLM_LLC_ACCESS                  GLM_ANY_RESPONSE
1628 #define GLM_SNP_ANY                     (GLM_SNP_NONE_OR_MISS|SNB_NO_FWD|SNB_HITM)
1629 #define GLM_LLC_MISS                    (GLM_SNP_ANY|SNB_NON_DRAM)
1630
1631 static __initconst const u64 glm_hw_cache_event_ids
1632                                 [PERF_COUNT_HW_CACHE_MAX]
1633                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1634                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1635         [C(L1D)] = {
1636                 [C(OP_READ)] = {
1637                         [C(RESULT_ACCESS)]      = 0x81d0,       /* MEM_UOPS_RETIRED.ALL_LOADS */
1638                         [C(RESULT_MISS)]        = 0x0,
1639                 },
1640                 [C(OP_WRITE)] = {
1641                         [C(RESULT_ACCESS)]      = 0x82d0,       /* MEM_UOPS_RETIRED.ALL_STORES */
1642                         [C(RESULT_MISS)]        = 0x0,
1643                 },
1644                 [C(OP_PREFETCH)] = {
1645                         [C(RESULT_ACCESS)]      = 0x0,
1646                         [C(RESULT_MISS)]        = 0x0,
1647                 },
1648         },
1649         [C(L1I)] = {
1650                 [C(OP_READ)] = {
1651                         [C(RESULT_ACCESS)]      = 0x0380,       /* ICACHE.ACCESSES */
1652                         [C(RESULT_MISS)]        = 0x0280,       /* ICACHE.MISSES */
1653                 },
1654                 [C(OP_WRITE)] = {
1655                         [C(RESULT_ACCESS)]      = -1,
1656                         [C(RESULT_MISS)]        = -1,
1657                 },
1658                 [C(OP_PREFETCH)] = {
1659                         [C(RESULT_ACCESS)]      = 0x0,
1660                         [C(RESULT_MISS)]        = 0x0,
1661                 },
1662         },
1663         [C(LL)] = {
1664                 [C(OP_READ)] = {
1665                         [C(RESULT_ACCESS)]      = 0x1b7,        /* OFFCORE_RESPONSE */
1666                         [C(RESULT_MISS)]        = 0x1b7,        /* OFFCORE_RESPONSE */
1667                 },
1668                 [C(OP_WRITE)] = {
1669                         [C(RESULT_ACCESS)]      = 0x1b7,        /* OFFCORE_RESPONSE */
1670                         [C(RESULT_MISS)]        = 0x1b7,        /* OFFCORE_RESPONSE */
1671                 },
1672                 [C(OP_PREFETCH)] = {
1673                         [C(RESULT_ACCESS)]      = 0x1b7,        /* OFFCORE_RESPONSE */
1674                         [C(RESULT_MISS)]        = 0x1b7,        /* OFFCORE_RESPONSE */
1675                 },
1676         },
1677         [C(DTLB)] = {
1678                 [C(OP_READ)] = {
1679                         [C(RESULT_ACCESS)]      = 0x81d0,       /* MEM_UOPS_RETIRED.ALL_LOADS */
1680                         [C(RESULT_MISS)]        = 0x0,
1681                 },
1682                 [C(OP_WRITE)] = {
1683                         [C(RESULT_ACCESS)]      = 0x82d0,       /* MEM_UOPS_RETIRED.ALL_STORES */
1684                         [C(RESULT_MISS)]        = 0x0,
1685                 },
1686                 [C(OP_PREFETCH)] = {
1687                         [C(RESULT_ACCESS)]      = 0x0,
1688                         [C(RESULT_MISS)]        = 0x0,
1689                 },
1690         },
1691         [C(ITLB)] = {
1692                 [C(OP_READ)] = {
1693                         [C(RESULT_ACCESS)]      = 0x00c0,       /* INST_RETIRED.ANY_P */
1694                         [C(RESULT_MISS)]        = 0x0481,       /* ITLB.MISS */
1695                 },
1696                 [C(OP_WRITE)] = {
1697                         [C(RESULT_ACCESS)]      = -1,
1698                         [C(RESULT_MISS)]        = -1,
1699                 },
1700                 [C(OP_PREFETCH)] = {
1701                         [C(RESULT_ACCESS)]      = -1,
1702                         [C(RESULT_MISS)]        = -1,
1703                 },
1704         },
1705         [C(BPU)] = {
1706                 [C(OP_READ)] = {
1707                         [C(RESULT_ACCESS)]      = 0x00c4,       /* BR_INST_RETIRED.ALL_BRANCHES */
1708                         [C(RESULT_MISS)]        = 0x00c5,       /* BR_MISP_RETIRED.ALL_BRANCHES */
1709                 },
1710                 [C(OP_WRITE)] = {
1711                         [C(RESULT_ACCESS)]      = -1,
1712                         [C(RESULT_MISS)]        = -1,
1713                 },
1714                 [C(OP_PREFETCH)] = {
1715                         [C(RESULT_ACCESS)]      = -1,
1716                         [C(RESULT_MISS)]        = -1,
1717                 },
1718         },
1719 };
1720
1721 static __initconst const u64 glm_hw_cache_extra_regs
1722                                 [PERF_COUNT_HW_CACHE_MAX]
1723                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1724                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1725         [C(LL)] = {
1726                 [C(OP_READ)] = {
1727                         [C(RESULT_ACCESS)]      = GLM_DEMAND_READ|
1728                                                   GLM_LLC_ACCESS,
1729                         [C(RESULT_MISS)]        = GLM_DEMAND_READ|
1730                                                   GLM_LLC_MISS,
1731                 },
1732                 [C(OP_WRITE)] = {
1733                         [C(RESULT_ACCESS)]      = GLM_DEMAND_WRITE|
1734                                                   GLM_LLC_ACCESS,
1735                         [C(RESULT_MISS)]        = GLM_DEMAND_WRITE|
1736                                                   GLM_LLC_MISS,
1737                 },
1738                 [C(OP_PREFETCH)] = {
1739                         [C(RESULT_ACCESS)]      = GLM_DEMAND_PREFETCH|
1740                                                   GLM_LLC_ACCESS,
1741                         [C(RESULT_MISS)]        = GLM_DEMAND_PREFETCH|
1742                                                   GLM_LLC_MISS,
1743                 },
1744         },
1745 };
1746
1747 static __initconst const u64 glp_hw_cache_event_ids
1748                                 [PERF_COUNT_HW_CACHE_MAX]
1749                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1750                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1751         [C(L1D)] = {
1752                 [C(OP_READ)] = {
1753                         [C(RESULT_ACCESS)]      = 0x81d0,       /* MEM_UOPS_RETIRED.ALL_LOADS */
1754                         [C(RESULT_MISS)]        = 0x0,
1755                 },
1756                 [C(OP_WRITE)] = {
1757                         [C(RESULT_ACCESS)]      = 0x82d0,       /* MEM_UOPS_RETIRED.ALL_STORES */
1758                         [C(RESULT_MISS)]        = 0x0,
1759                 },
1760                 [C(OP_PREFETCH)] = {
1761                         [C(RESULT_ACCESS)]      = 0x0,
1762                         [C(RESULT_MISS)]        = 0x0,
1763                 },
1764         },
1765         [C(L1I)] = {
1766                 [C(OP_READ)] = {
1767                         [C(RESULT_ACCESS)]      = 0x0380,       /* ICACHE.ACCESSES */
1768                         [C(RESULT_MISS)]        = 0x0280,       /* ICACHE.MISSES */
1769                 },
1770                 [C(OP_WRITE)] = {
1771                         [C(RESULT_ACCESS)]      = -1,
1772                         [C(RESULT_MISS)]        = -1,
1773                 },
1774                 [C(OP_PREFETCH)] = {
1775                         [C(RESULT_ACCESS)]      = 0x0,
1776                         [C(RESULT_MISS)]        = 0x0,
1777                 },
1778         },
1779         [C(LL)] = {
1780                 [C(OP_READ)] = {
1781                         [C(RESULT_ACCESS)]      = 0x1b7,        /* OFFCORE_RESPONSE */
1782                         [C(RESULT_MISS)]        = 0x1b7,        /* OFFCORE_RESPONSE */
1783                 },
1784                 [C(OP_WRITE)] = {
1785                         [C(RESULT_ACCESS)]      = 0x1b7,        /* OFFCORE_RESPONSE */
1786                         [C(RESULT_MISS)]        = 0x1b7,        /* OFFCORE_RESPONSE */
1787                 },
1788                 [C(OP_PREFETCH)] = {
1789                         [C(RESULT_ACCESS)]      = 0x0,
1790                         [C(RESULT_MISS)]        = 0x0,
1791                 },
1792         },
1793         [C(DTLB)] = {
1794                 [C(OP_READ)] = {
1795                         [C(RESULT_ACCESS)]      = 0x81d0,       /* MEM_UOPS_RETIRED.ALL_LOADS */
1796                         [C(RESULT_MISS)]        = 0xe08,        /* DTLB_LOAD_MISSES.WALK_COMPLETED */
1797                 },
1798                 [C(OP_WRITE)] = {
1799                         [C(RESULT_ACCESS)]      = 0x82d0,       /* MEM_UOPS_RETIRED.ALL_STORES */
1800                         [C(RESULT_MISS)]        = 0xe49,        /* DTLB_STORE_MISSES.WALK_COMPLETED */
1801                 },
1802                 [C(OP_PREFETCH)] = {
1803                         [C(RESULT_ACCESS)]      = 0x0,
1804                         [C(RESULT_MISS)]        = 0x0,
1805                 },
1806         },
1807         [C(ITLB)] = {
1808                 [C(OP_READ)] = {
1809                         [C(RESULT_ACCESS)]      = 0x00c0,       /* INST_RETIRED.ANY_P */
1810                         [C(RESULT_MISS)]        = 0x0481,       /* ITLB.MISS */
1811                 },
1812                 [C(OP_WRITE)] = {
1813                         [C(RESULT_ACCESS)]      = -1,
1814                         [C(RESULT_MISS)]        = -1,
1815                 },
1816                 [C(OP_PREFETCH)] = {
1817                         [C(RESULT_ACCESS)]      = -1,
1818                         [C(RESULT_MISS)]        = -1,
1819                 },
1820         },
1821         [C(BPU)] = {
1822                 [C(OP_READ)] = {
1823                         [C(RESULT_ACCESS)]      = 0x00c4,       /* BR_INST_RETIRED.ALL_BRANCHES */
1824                         [C(RESULT_MISS)]        = 0x00c5,       /* BR_MISP_RETIRED.ALL_BRANCHES */
1825                 },
1826                 [C(OP_WRITE)] = {
1827                         [C(RESULT_ACCESS)]      = -1,
1828                         [C(RESULT_MISS)]        = -1,
1829                 },
1830                 [C(OP_PREFETCH)] = {
1831                         [C(RESULT_ACCESS)]      = -1,
1832                         [C(RESULT_MISS)]        = -1,
1833                 },
1834         },
1835 };
1836
1837 static __initconst const u64 glp_hw_cache_extra_regs
1838                                 [PERF_COUNT_HW_CACHE_MAX]
1839                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1840                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1841         [C(LL)] = {
1842                 [C(OP_READ)] = {
1843                         [C(RESULT_ACCESS)]      = GLM_DEMAND_READ|
1844                                                   GLM_LLC_ACCESS,
1845                         [C(RESULT_MISS)]        = GLM_DEMAND_READ|
1846                                                   GLM_LLC_MISS,
1847                 },
1848                 [C(OP_WRITE)] = {
1849                         [C(RESULT_ACCESS)]      = GLM_DEMAND_WRITE|
1850                                                   GLM_LLC_ACCESS,
1851                         [C(RESULT_MISS)]        = GLM_DEMAND_WRITE|
1852                                                   GLM_LLC_MISS,
1853                 },
1854                 [C(OP_PREFETCH)] = {
1855                         [C(RESULT_ACCESS)]      = 0x0,
1856                         [C(RESULT_MISS)]        = 0x0,
1857                 },
1858         },
1859 };
1860
1861 #define TNT_LOCAL_DRAM                  BIT_ULL(26)
1862 #define TNT_DEMAND_READ                 GLM_DEMAND_DATA_RD
1863 #define TNT_DEMAND_WRITE                GLM_DEMAND_RFO
1864 #define TNT_LLC_ACCESS                  GLM_ANY_RESPONSE
1865 #define TNT_SNP_ANY                     (SNB_SNP_NOT_NEEDED|SNB_SNP_MISS| \
1866                                          SNB_NO_FWD|SNB_SNP_FWD|SNB_HITM)
1867 #define TNT_LLC_MISS                    (TNT_SNP_ANY|SNB_NON_DRAM|TNT_LOCAL_DRAM)
1868
1869 static __initconst const u64 tnt_hw_cache_extra_regs
1870                                 [PERF_COUNT_HW_CACHE_MAX]
1871                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1872                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1873         [C(LL)] = {
1874                 [C(OP_READ)] = {
1875                         [C(RESULT_ACCESS)]      = TNT_DEMAND_READ|
1876                                                   TNT_LLC_ACCESS,
1877                         [C(RESULT_MISS)]        = TNT_DEMAND_READ|
1878                                                   TNT_LLC_MISS,
1879                 },
1880                 [C(OP_WRITE)] = {
1881                         [C(RESULT_ACCESS)]      = TNT_DEMAND_WRITE|
1882                                                   TNT_LLC_ACCESS,
1883                         [C(RESULT_MISS)]        = TNT_DEMAND_WRITE|
1884                                                   TNT_LLC_MISS,
1885                 },
1886                 [C(OP_PREFETCH)] = {
1887                         [C(RESULT_ACCESS)]      = 0x0,
1888                         [C(RESULT_MISS)]        = 0x0,
1889                 },
1890         },
1891 };
1892
1893 static struct extra_reg intel_tnt_extra_regs[] __read_mostly = {
1894         /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
1895         INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x800ff0ffffff9fffull, RSP_0),
1896         INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0xff0ffffff9fffull, RSP_1),
1897         EVENT_EXTRA_END
1898 };
1899
1900 #define KNL_OT_L2_HITE          BIT_ULL(19) /* Other Tile L2 Hit */
1901 #define KNL_OT_L2_HITF          BIT_ULL(20) /* Other Tile L2 Hit */
1902 #define KNL_MCDRAM_LOCAL        BIT_ULL(21)
1903 #define KNL_MCDRAM_FAR          BIT_ULL(22)
1904 #define KNL_DDR_LOCAL           BIT_ULL(23)
1905 #define KNL_DDR_FAR             BIT_ULL(24)
1906 #define KNL_DRAM_ANY            (KNL_MCDRAM_LOCAL | KNL_MCDRAM_FAR | \
1907                                     KNL_DDR_LOCAL | KNL_DDR_FAR)
1908 #define KNL_L2_READ             SLM_DMND_READ
1909 #define KNL_L2_WRITE            SLM_DMND_WRITE
1910 #define KNL_L2_PREFETCH         SLM_DMND_PREFETCH
1911 #define KNL_L2_ACCESS           SLM_LLC_ACCESS
1912 #define KNL_L2_MISS             (KNL_OT_L2_HITE | KNL_OT_L2_HITF | \
1913                                    KNL_DRAM_ANY | SNB_SNP_ANY | \
1914                                                   SNB_NON_DRAM)
1915
1916 static __initconst const u64 knl_hw_cache_extra_regs
1917                                 [PERF_COUNT_HW_CACHE_MAX]
1918                                 [PERF_COUNT_HW_CACHE_OP_MAX]
1919                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1920         [C(LL)] = {
1921                 [C(OP_READ)] = {
1922                         [C(RESULT_ACCESS)] = KNL_L2_READ | KNL_L2_ACCESS,
1923                         [C(RESULT_MISS)]   = 0,
1924                 },
1925                 [C(OP_WRITE)] = {
1926                         [C(RESULT_ACCESS)] = KNL_L2_WRITE | KNL_L2_ACCESS,
1927                         [C(RESULT_MISS)]   = KNL_L2_WRITE | KNL_L2_MISS,
1928                 },
1929                 [C(OP_PREFETCH)] = {
1930                         [C(RESULT_ACCESS)] = KNL_L2_PREFETCH | KNL_L2_ACCESS,
1931                         [C(RESULT_MISS)]   = KNL_L2_PREFETCH | KNL_L2_MISS,
1932                 },
1933         },
1934 };
1935
1936 /*
1937  * Used from PMIs where the LBRs are already disabled.
1938  *
1939  * This function could be called consecutively. It is required to remain in
1940  * disabled state if called consecutively.
1941  *
1942  * During consecutive calls, the same disable value will be written to related
1943  * registers, so the PMU state remains unchanged.
1944  *
1945  * intel_bts events don't coexist with intel PMU's BTS events because of
1946  * x86_add_exclusive(x86_lbr_exclusive_lbr); there's no need to keep them
1947  * disabled around intel PMU's event batching etc, only inside the PMI handler.
1948  *
1949  * Avoid PEBS_ENABLE MSR access in PMIs.
1950  * The GLOBAL_CTRL has been disabled. All the counters do not count anymore.
1951  * It doesn't matter if the PEBS is enabled or not.
1952  * Usually, the PEBS status are not changed in PMIs. It's unnecessary to
1953  * access PEBS_ENABLE MSR in disable_all()/enable_all().
1954  * However, there are some cases which may change PEBS status, e.g. PMI
1955  * throttle. The PEBS_ENABLE should be updated where the status changes.
1956  */
1957 static void __intel_pmu_disable_all(void)
1958 {
1959         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1960
1961         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
1962
1963         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1964                 intel_pmu_disable_bts();
1965 }
1966
1967 static void intel_pmu_disable_all(void)
1968 {
1969         __intel_pmu_disable_all();
1970         intel_pmu_pebs_disable_all();
1971         intel_pmu_lbr_disable_all();
1972 }
1973
1974 static void __intel_pmu_enable_all(int added, bool pmi)
1975 {
1976         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1977
1978         intel_pmu_lbr_enable_all(pmi);
1979         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL,
1980                         x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
1981
1982         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
1983                 struct perf_event *event =
1984                         cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
1985
1986                 if (WARN_ON_ONCE(!event))
1987                         return;
1988
1989                 intel_pmu_enable_bts(event->hw.config);
1990         }
1991 }
1992
1993 static void intel_pmu_enable_all(int added)
1994 {
1995         intel_pmu_pebs_enable_all();
1996         __intel_pmu_enable_all(added, false);
1997 }
1998
1999 /*
2000  * Workaround for:
2001  *   Intel Errata AAK100 (model 26)
2002  *   Intel Errata AAP53  (model 30)
2003  *   Intel Errata BD53   (model 44)
2004  *
2005  * The official story:
2006  *   These chips need to be 'reset' when adding counters by programming the
2007  *   magic three (non-counting) events 0x4300B5, 0x4300D2, and 0x4300B1 either
2008  *   in sequence on the same PMC or on different PMCs.
2009  *
2010  * In practise it appears some of these events do in fact count, and
2011  * we need to program all 4 events.
2012  */
2013 static void intel_pmu_nhm_workaround(void)
2014 {
2015         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2016         static const unsigned long nhm_magic[4] = {
2017                 0x4300B5,
2018                 0x4300D2,
2019                 0x4300B1,
2020                 0x4300B1
2021         };
2022         struct perf_event *event;
2023         int i;
2024
2025         /*
2026          * The Errata requires below steps:
2027          * 1) Clear MSR_IA32_PEBS_ENABLE and MSR_CORE_PERF_GLOBAL_CTRL;
2028          * 2) Configure 4 PERFEVTSELx with the magic events and clear
2029          *    the corresponding PMCx;
2030          * 3) set bit0~bit3 of MSR_CORE_PERF_GLOBAL_CTRL;
2031          * 4) Clear MSR_CORE_PERF_GLOBAL_CTRL;
2032          * 5) Clear 4 pairs of ERFEVTSELx and PMCx;
2033          */
2034
2035         /*
2036          * The real steps we choose are a little different from above.
2037          * A) To reduce MSR operations, we don't run step 1) as they
2038          *    are already cleared before this function is called;
2039          * B) Call x86_perf_event_update to save PMCx before configuring
2040          *    PERFEVTSELx with magic number;
2041          * C) With step 5), we do clear only when the PERFEVTSELx is
2042          *    not used currently.
2043          * D) Call x86_perf_event_set_period to restore PMCx;
2044          */
2045
2046         /* We always operate 4 pairs of PERF Counters */
2047         for (i = 0; i < 4; i++) {
2048                 event = cpuc->events[i];
2049                 if (event)
2050                         x86_perf_event_update(event);
2051         }
2052
2053         for (i = 0; i < 4; i++) {
2054                 wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]);
2055                 wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0);
2056         }
2057
2058         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf);
2059         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0);
2060
2061         for (i = 0; i < 4; i++) {
2062                 event = cpuc->events[i];
2063
2064                 if (event) {
2065                         x86_perf_event_set_period(event);
2066                         __x86_pmu_enable_event(&event->hw,
2067                                         ARCH_PERFMON_EVENTSEL_ENABLE);
2068                 } else
2069                         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0);
2070         }
2071 }
2072
2073 static void intel_pmu_nhm_enable_all(int added)
2074 {
2075         if (added)
2076                 intel_pmu_nhm_workaround();
2077         intel_pmu_enable_all(added);
2078 }
2079
2080 static void intel_set_tfa(struct cpu_hw_events *cpuc, bool on)
2081 {
2082         u64 val = on ? MSR_TFA_RTM_FORCE_ABORT : 0;
2083
2084         if (cpuc->tfa_shadow != val) {
2085                 cpuc->tfa_shadow = val;
2086                 wrmsrl(MSR_TSX_FORCE_ABORT, val);
2087         }
2088 }
2089
2090 static void intel_tfa_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)
2091 {
2092         /*
2093          * We're going to use PMC3, make sure TFA is set before we touch it.
2094          */
2095         if (cntr == 3)
2096                 intel_set_tfa(cpuc, true);
2097 }
2098
2099 static void intel_tfa_pmu_enable_all(int added)
2100 {
2101         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2102
2103         /*
2104          * If we find PMC3 is no longer used when we enable the PMU, we can
2105          * clear TFA.
2106          */
2107         if (!test_bit(3, cpuc->active_mask))
2108                 intel_set_tfa(cpuc, false);
2109
2110         intel_pmu_enable_all(added);
2111 }
2112
2113 static void enable_counter_freeze(void)
2114 {
2115         update_debugctlmsr(get_debugctlmsr() |
2116                         DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI);
2117 }
2118
2119 static void disable_counter_freeze(void)
2120 {
2121         update_debugctlmsr(get_debugctlmsr() &
2122                         ~DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI);
2123 }
2124
2125 static inline u64 intel_pmu_get_status(void)
2126 {
2127         u64 status;
2128
2129         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
2130
2131         return status;
2132 }
2133
2134 static inline void intel_pmu_ack_status(u64 ack)
2135 {
2136         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
2137 }
2138
2139 static inline bool event_is_checkpointed(struct perf_event *event)
2140 {
2141         return unlikely(event->hw.config & HSW_IN_TX_CHECKPOINTED) != 0;
2142 }
2143
2144 static inline void intel_set_masks(struct perf_event *event, int idx)
2145 {
2146         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2147
2148         if (event->attr.exclude_host)
2149                 __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask);
2150         if (event->attr.exclude_guest)
2151                 __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask);
2152         if (event_is_checkpointed(event))
2153                 __set_bit(idx, (unsigned long *)&cpuc->intel_cp_status);
2154 }
2155
2156 static inline void intel_clear_masks(struct perf_event *event, int idx)
2157 {
2158         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2159
2160         __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask);
2161         __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask);
2162         __clear_bit(idx, (unsigned long *)&cpuc->intel_cp_status);
2163 }
2164
2165 static void intel_pmu_disable_fixed(struct perf_event *event)
2166 {
2167         struct hw_perf_event *hwc = &event->hw;
2168         int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
2169         u64 ctrl_val, mask;
2170
2171         mask = 0xfULL << (idx * 4);
2172
2173         rdmsrl(hwc->config_base, ctrl_val);
2174         ctrl_val &= ~mask;
2175         wrmsrl(hwc->config_base, ctrl_val);
2176 }
2177
2178 static void intel_pmu_disable_event(struct perf_event *event)
2179 {
2180         struct hw_perf_event *hwc = &event->hw;
2181         int idx = hwc->idx;
2182
2183         switch (idx) {
2184         case 0 ... INTEL_PMC_IDX_FIXED - 1:
2185                 intel_clear_masks(event, idx);
2186                 x86_pmu_disable_event(event);
2187                 break;
2188         case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
2189                 intel_clear_masks(event, idx);
2190                 intel_pmu_disable_fixed(event);
2191                 break;
2192         case INTEL_PMC_IDX_FIXED_BTS:
2193                 intel_pmu_disable_bts();
2194                 intel_pmu_drain_bts_buffer();
2195                 return;
2196         case INTEL_PMC_IDX_FIXED_VLBR:
2197                 intel_clear_masks(event, idx);
2198                 break;
2199         default:
2200                 intel_clear_masks(event, idx);
2201                 pr_warn("Failed to disable the event with invalid index %d\n",
2202                         idx);
2203                 return;
2204         }
2205
2206         /*
2207          * Needs to be called after x86_pmu_disable_event,
2208          * so we don't trigger the event without PEBS bit set.
2209          */
2210         if (unlikely(event->attr.precise_ip))
2211                 intel_pmu_pebs_disable(event);
2212 }
2213
2214 static void intel_pmu_del_event(struct perf_event *event)
2215 {
2216         if (needs_branch_stack(event))
2217                 intel_pmu_lbr_del(event);
2218         if (event->attr.precise_ip)
2219                 intel_pmu_pebs_del(event);
2220 }
2221
2222 static void intel_pmu_read_event(struct perf_event *event)
2223 {
2224         if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
2225                 intel_pmu_auto_reload_read(event);
2226         else
2227                 x86_perf_event_update(event);
2228 }
2229
2230 static void intel_pmu_enable_fixed(struct perf_event *event)
2231 {
2232         struct hw_perf_event *hwc = &event->hw;
2233         int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
2234         u64 ctrl_val, mask, bits = 0;
2235
2236         /*
2237          * Enable IRQ generation (0x8), if not PEBS,
2238          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
2239          * if requested:
2240          */
2241         if (!event->attr.precise_ip)
2242                 bits |= 0x8;
2243         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
2244                 bits |= 0x2;
2245         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
2246                 bits |= 0x1;
2247
2248         /*
2249          * ANY bit is supported in v3 and up
2250          */
2251         if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
2252                 bits |= 0x4;
2253
2254         bits <<= (idx * 4);
2255         mask = 0xfULL << (idx * 4);
2256
2257         if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip) {
2258                 bits |= ICL_FIXED_0_ADAPTIVE << (idx * 4);
2259                 mask |= ICL_FIXED_0_ADAPTIVE << (idx * 4);
2260         }
2261
2262         rdmsrl(hwc->config_base, ctrl_val);
2263         ctrl_val &= ~mask;
2264         ctrl_val |= bits;
2265         wrmsrl(hwc->config_base, ctrl_val);
2266 }
2267
2268 static void intel_pmu_enable_event(struct perf_event *event)
2269 {
2270         struct hw_perf_event *hwc = &event->hw;
2271         int idx = hwc->idx;
2272
2273         if (unlikely(event->attr.precise_ip))
2274                 intel_pmu_pebs_enable(event);
2275
2276         switch (idx) {
2277         case 0 ... INTEL_PMC_IDX_FIXED - 1:
2278                 intel_set_masks(event, idx);
2279                 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
2280                 break;
2281         case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
2282                 intel_set_masks(event, idx);
2283                 intel_pmu_enable_fixed(event);
2284                 break;
2285         case INTEL_PMC_IDX_FIXED_BTS:
2286                 if (!__this_cpu_read(cpu_hw_events.enabled))
2287                         return;
2288                 intel_pmu_enable_bts(hwc->config);
2289                 break;
2290         case INTEL_PMC_IDX_FIXED_VLBR:
2291                 intel_set_masks(event, idx);
2292                 break;
2293         default:
2294                 pr_warn("Failed to enable the event with invalid index %d\n",
2295                         idx);
2296         }
2297 }
2298
2299 static void intel_pmu_add_event(struct perf_event *event)
2300 {
2301         if (event->attr.precise_ip)
2302                 intel_pmu_pebs_add(event);
2303         if (needs_branch_stack(event))
2304                 intel_pmu_lbr_add(event);
2305 }
2306
2307 /*
2308  * Save and restart an expired event. Called by NMI contexts,
2309  * so it has to be careful about preempting normal event ops:
2310  */
2311 int intel_pmu_save_and_restart(struct perf_event *event)
2312 {
2313         x86_perf_event_update(event);
2314         /*
2315          * For a checkpointed counter always reset back to 0.  This
2316          * avoids a situation where the counter overflows, aborts the
2317          * transaction and is then set back to shortly before the
2318          * overflow, and overflows and aborts again.
2319          */
2320         if (unlikely(event_is_checkpointed(event))) {
2321                 /* No race with NMIs because the counter should not be armed */
2322                 wrmsrl(event->hw.event_base, 0);
2323                 local64_set(&event->hw.prev_count, 0);
2324         }
2325         return x86_perf_event_set_period(event);
2326 }
2327
2328 static void intel_pmu_reset(void)
2329 {
2330         struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
2331         unsigned long flags;
2332         int idx;
2333
2334         if (!x86_pmu.num_counters)
2335                 return;
2336
2337         local_irq_save(flags);
2338
2339         pr_info("clearing PMU state on CPU#%d\n", smp_processor_id());
2340
2341         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
2342                 wrmsrl_safe(x86_pmu_config_addr(idx), 0ull);
2343                 wrmsrl_safe(x86_pmu_event_addr(idx),  0ull);
2344         }
2345         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++)
2346                 wrmsrl_safe(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
2347
2348         if (ds)
2349                 ds->bts_index = ds->bts_buffer_base;
2350
2351         /* Ack all overflows and disable fixed counters */
2352         if (x86_pmu.version >= 2) {
2353                 intel_pmu_ack_status(intel_pmu_get_status());
2354                 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
2355         }
2356
2357         /* Reset LBRs and LBR freezing */
2358         if (x86_pmu.lbr_nr) {
2359                 update_debugctlmsr(get_debugctlmsr() &
2360                         ~(DEBUGCTLMSR_FREEZE_LBRS_ON_PMI|DEBUGCTLMSR_LBR));
2361         }
2362
2363         local_irq_restore(flags);
2364 }
2365
2366 static int handle_pmi_common(struct pt_regs *regs, u64 status)
2367 {
2368         struct perf_sample_data data;
2369         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2370         int bit;
2371         int handled = 0;
2372
2373         inc_irq_stat(apic_perf_irqs);
2374
2375         /*
2376          * Ignore a range of extra bits in status that do not indicate
2377          * overflow by themselves.
2378          */
2379         status &= ~(GLOBAL_STATUS_COND_CHG |
2380                     GLOBAL_STATUS_ASIF |
2381                     GLOBAL_STATUS_LBRS_FROZEN);
2382         if (!status)
2383                 return 0;
2384         /*
2385          * In case multiple PEBS events are sampled at the same time,
2386          * it is possible to have GLOBAL_STATUS bit 62 set indicating
2387          * PEBS buffer overflow and also seeing at most 3 PEBS counters
2388          * having their bits set in the status register. This is a sign
2389          * that there was at least one PEBS record pending at the time
2390          * of the PMU interrupt. PEBS counters must only be processed
2391          * via the drain_pebs() calls and not via the regular sample
2392          * processing loop coming after that the function, otherwise
2393          * phony regular samples may be generated in the sampling buffer
2394          * not marked with the EXACT tag. Another possibility is to have
2395          * one PEBS event and at least one non-PEBS event whic hoverflows
2396          * while PEBS has armed. In this case, bit 62 of GLOBAL_STATUS will
2397          * not be set, yet the overflow status bit for the PEBS counter will
2398          * be on Skylake.
2399          *
2400          * To avoid this problem, we systematically ignore the PEBS-enabled
2401          * counters from the GLOBAL_STATUS mask and we always process PEBS
2402          * events via drain_pebs().
2403          */
2404         if (x86_pmu.flags & PMU_FL_PEBS_ALL)
2405                 status &= ~cpuc->pebs_enabled;
2406         else
2407                 status &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK);
2408
2409         /*
2410          * PEBS overflow sets bit 62 in the global status register
2411          */
2412         if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) {
2413                 u64 pebs_enabled = cpuc->pebs_enabled;
2414
2415                 handled++;
2416                 x86_pmu.drain_pebs(regs);
2417                 status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
2418
2419                 /*
2420                  * PMI throttle may be triggered, which stops the PEBS event.
2421                  * Although cpuc->pebs_enabled is updated accordingly, the
2422                  * MSR_IA32_PEBS_ENABLE is not updated. Because the
2423                  * cpuc->enabled has been forced to 0 in PMI.
2424                  * Update the MSR if pebs_enabled is changed.
2425                  */
2426                 if (pebs_enabled != cpuc->pebs_enabled)
2427                         wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
2428         }
2429
2430         /*
2431          * Intel PT
2432          */
2433         if (__test_and_clear_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT, (unsigned long *)&status)) {
2434                 handled++;
2435                 if (unlikely(perf_guest_cbs && perf_guest_cbs->is_in_guest() &&
2436                         perf_guest_cbs->handle_intel_pt_intr))
2437                         perf_guest_cbs->handle_intel_pt_intr();
2438                 else
2439                         intel_pt_interrupt();
2440         }
2441
2442         /*
2443          * Checkpointed counters can lead to 'spurious' PMIs because the
2444          * rollback caused by the PMI will have cleared the overflow status
2445          * bit. Therefore always force probe these counters.
2446          */
2447         status |= cpuc->intel_cp_status;
2448
2449         for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
2450                 struct perf_event *event = cpuc->events[bit];
2451
2452                 handled++;
2453
2454                 if (!test_bit(bit, cpuc->active_mask))
2455                         continue;
2456
2457                 if (!intel_pmu_save_and_restart(event))
2458                         continue;
2459
2460                 perf_sample_data_init(&data, 0, event->hw.last_period);
2461
2462                 if (has_branch_stack(event))
2463                         data.br_stack = &cpuc->lbr_stack;
2464
2465                 if (perf_event_overflow(event, &data, regs))
2466                         x86_pmu_stop(event, 0);
2467         }
2468
2469         return handled;
2470 }
2471
2472 static bool disable_counter_freezing = true;
2473 static int __init intel_perf_counter_freezing_setup(char *s)
2474 {
2475         bool res;
2476
2477         if (kstrtobool(s, &res))
2478                 return -EINVAL;
2479
2480         disable_counter_freezing = !res;
2481         return 1;
2482 }
2483 __setup("perf_v4_pmi=", intel_perf_counter_freezing_setup);
2484
2485 /*
2486  * Simplified handler for Arch Perfmon v4:
2487  * - We rely on counter freezing/unfreezing to enable/disable the PMU.
2488  * This is done automatically on PMU ack.
2489  * - Ack the PMU only after the APIC.
2490  */
2491
2492 static int intel_pmu_handle_irq_v4(struct pt_regs *regs)
2493 {
2494         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2495         int handled = 0;
2496         bool bts = false;
2497         u64 status;
2498         int pmu_enabled = cpuc->enabled;
2499         int loops = 0;
2500
2501         /* PMU has been disabled because of counter freezing */
2502         cpuc->enabled = 0;
2503         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
2504                 bts = true;
2505                 intel_bts_disable_local();
2506                 handled = intel_pmu_drain_bts_buffer();
2507                 handled += intel_bts_interrupt();
2508         }
2509         status = intel_pmu_get_status();
2510         if (!status)
2511                 goto done;
2512 again:
2513         intel_pmu_lbr_read();
2514         if (++loops > 100) {
2515                 static bool warned;
2516
2517                 if (!warned) {
2518                         WARN(1, "perfevents: irq loop stuck!\n");
2519                         perf_event_print_debug();
2520                         warned = true;
2521                 }
2522                 intel_pmu_reset();
2523                 goto done;
2524         }
2525
2526
2527         handled += handle_pmi_common(regs, status);
2528 done:
2529         /* Ack the PMI in the APIC */
2530         apic_write(APIC_LVTPC, APIC_DM_NMI);
2531
2532         /*
2533          * The counters start counting immediately while ack the status.
2534          * Make it as close as possible to IRET. This avoids bogus
2535          * freezing on Skylake CPUs.
2536          */
2537         if (status) {
2538                 intel_pmu_ack_status(status);
2539         } else {
2540                 /*
2541                  * CPU may issues two PMIs very close to each other.
2542                  * When the PMI handler services the first one, the
2543                  * GLOBAL_STATUS is already updated to reflect both.
2544                  * When it IRETs, the second PMI is immediately
2545                  * handled and it sees clear status. At the meantime,
2546                  * there may be a third PMI, because the freezing bit
2547                  * isn't set since the ack in first PMI handlers.
2548                  * Double check if there is more work to be done.
2549                  */
2550                 status = intel_pmu_get_status();
2551                 if (status)
2552                         goto again;
2553         }
2554
2555         if (bts)
2556                 intel_bts_enable_local();
2557         cpuc->enabled = pmu_enabled;
2558         return handled;
2559 }
2560
2561 /*
2562  * This handler is triggered by the local APIC, so the APIC IRQ handling
2563  * rules apply:
2564  */
2565 static int intel_pmu_handle_irq(struct pt_regs *regs)
2566 {
2567         struct cpu_hw_events *cpuc;
2568         int loops;
2569         u64 status;
2570         int handled;
2571         int pmu_enabled;
2572
2573         cpuc = this_cpu_ptr(&cpu_hw_events);
2574
2575         /*
2576          * Save the PMU state.
2577          * It needs to be restored when leaving the handler.
2578          */
2579         pmu_enabled = cpuc->enabled;
2580         /*
2581          * No known reason to not always do late ACK,
2582          * but just in case do it opt-in.
2583          */
2584         if (!x86_pmu.late_ack)
2585                 apic_write(APIC_LVTPC, APIC_DM_NMI);
2586         intel_bts_disable_local();
2587         cpuc->enabled = 0;
2588         __intel_pmu_disable_all();
2589         handled = intel_pmu_drain_bts_buffer();
2590         handled += intel_bts_interrupt();
2591         status = intel_pmu_get_status();
2592         if (!status)
2593                 goto done;
2594
2595         loops = 0;
2596 again:
2597         intel_pmu_lbr_read();
2598         intel_pmu_ack_status(status);
2599         if (++loops > 100) {
2600                 static bool warned;
2601
2602                 if (!warned) {
2603                         WARN(1, "perfevents: irq loop stuck!\n");
2604                         perf_event_print_debug();
2605                         warned = true;
2606                 }
2607                 intel_pmu_reset();
2608                 goto done;
2609         }
2610
2611         handled += handle_pmi_common(regs, status);
2612
2613         /*
2614          * Repeat if there is more work to be done:
2615          */
2616         status = intel_pmu_get_status();
2617         if (status)
2618                 goto again;
2619
2620 done:
2621         /* Only restore PMU state when it's active. See x86_pmu_disable(). */
2622         cpuc->enabled = pmu_enabled;
2623         if (pmu_enabled)
2624                 __intel_pmu_enable_all(0, true);
2625         intel_bts_enable_local();
2626
2627         /*
2628          * Only unmask the NMI after the overflow counters
2629          * have been reset. This avoids spurious NMIs on
2630          * Haswell CPUs.
2631          */
2632         if (x86_pmu.late_ack)
2633                 apic_write(APIC_LVTPC, APIC_DM_NMI);
2634         return handled;
2635 }
2636
2637 static struct event_constraint *
2638 intel_bts_constraints(struct perf_event *event)
2639 {
2640         if (unlikely(intel_pmu_has_bts(event)))
2641                 return &bts_constraint;
2642
2643         return NULL;
2644 }
2645
2646 /*
2647  * Note: matches a fake event, like Fixed2.
2648  */
2649 static struct event_constraint *
2650 intel_vlbr_constraints(struct perf_event *event)
2651 {
2652         struct event_constraint *c = &vlbr_constraint;
2653
2654         if (unlikely(constraint_match(c, event->hw.config)))
2655                 return c;
2656
2657         return NULL;
2658 }
2659
2660 static int intel_alt_er(int idx, u64 config)
2661 {
2662         int alt_idx = idx;
2663
2664         if (!(x86_pmu.flags & PMU_FL_HAS_RSP_1))
2665                 return idx;
2666
2667         if (idx == EXTRA_REG_RSP_0)
2668                 alt_idx = EXTRA_REG_RSP_1;
2669
2670         if (idx == EXTRA_REG_RSP_1)
2671                 alt_idx = EXTRA_REG_RSP_0;
2672
2673         if (config & ~x86_pmu.extra_regs[alt_idx].valid_mask)
2674                 return idx;
2675
2676         return alt_idx;
2677 }
2678
2679 static void intel_fixup_er(struct perf_event *event, int idx)
2680 {
2681         event->hw.extra_reg.idx = idx;
2682
2683         if (idx == EXTRA_REG_RSP_0) {
2684                 event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
2685                 event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_0].event;
2686                 event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0;
2687         } else if (idx == EXTRA_REG_RSP_1) {
2688                 event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
2689                 event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_1].event;
2690                 event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1;
2691         }
2692 }
2693
2694 /*
2695  * manage allocation of shared extra msr for certain events
2696  *
2697  * sharing can be:
2698  * per-cpu: to be shared between the various events on a single PMU
2699  * per-core: per-cpu + shared by HT threads
2700  */
2701 static struct event_constraint *
2702 __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,
2703                                    struct perf_event *event,
2704                                    struct hw_perf_event_extra *reg)
2705 {
2706         struct event_constraint *c = &emptyconstraint;
2707         struct er_account *era;
2708         unsigned long flags;
2709         int idx = reg->idx;
2710
2711         /*
2712          * reg->alloc can be set due to existing state, so for fake cpuc we
2713          * need to ignore this, otherwise we might fail to allocate proper fake
2714          * state for this extra reg constraint. Also see the comment below.
2715          */
2716         if (reg->alloc && !cpuc->is_fake)
2717                 return NULL; /* call x86_get_event_constraint() */
2718
2719 again:
2720         era = &cpuc->shared_regs->regs[idx];
2721         /*
2722          * we use spin_lock_irqsave() to avoid lockdep issues when
2723          * passing a fake cpuc
2724          */
2725         raw_spin_lock_irqsave(&era->lock, flags);
2726
2727         if (!atomic_read(&era->ref) || era->config == reg->config) {
2728
2729                 /*
2730                  * If its a fake cpuc -- as per validate_{group,event}() we
2731                  * shouldn't touch event state and we can avoid doing so
2732                  * since both will only call get_event_constraints() once
2733                  * on each event, this avoids the need for reg->alloc.
2734                  *
2735                  * Not doing the ER fixup will only result in era->reg being
2736                  * wrong, but since we won't actually try and program hardware
2737                  * this isn't a problem either.
2738                  */
2739                 if (!cpuc->is_fake) {
2740                         if (idx != reg->idx)
2741                                 intel_fixup_er(event, idx);
2742
2743                         /*
2744                          * x86_schedule_events() can call get_event_constraints()
2745                          * multiple times on events in the case of incremental
2746                          * scheduling(). reg->alloc ensures we only do the ER
2747                          * allocation once.
2748                          */
2749                         reg->alloc = 1;
2750                 }
2751
2752                 /* lock in msr value */
2753                 era->config = reg->config;
2754                 era->reg = reg->reg;
2755
2756                 /* one more user */
2757                 atomic_inc(&era->ref);
2758
2759                 /*
2760                  * need to call x86_get_event_constraint()
2761                  * to check if associated event has constraints
2762                  */
2763                 c = NULL;
2764         } else {
2765                 idx = intel_alt_er(idx, reg->config);
2766                 if (idx != reg->idx) {
2767                         raw_spin_unlock_irqrestore(&era->lock, flags);
2768                         goto again;
2769                 }
2770         }
2771         raw_spin_unlock_irqrestore(&era->lock, flags);
2772
2773         return c;
2774 }
2775
2776 static void
2777 __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc,
2778                                    struct hw_perf_event_extra *reg)
2779 {
2780         struct er_account *era;
2781
2782         /*
2783          * Only put constraint if extra reg was actually allocated. Also takes
2784          * care of event which do not use an extra shared reg.
2785          *
2786          * Also, if this is a fake cpuc we shouldn't touch any event state
2787          * (reg->alloc) and we don't care about leaving inconsistent cpuc state
2788          * either since it'll be thrown out.
2789          */
2790         if (!reg->alloc || cpuc->is_fake)
2791                 return;
2792
2793         era = &cpuc->shared_regs->regs[reg->idx];
2794
2795         /* one fewer user */
2796         atomic_dec(&era->ref);
2797
2798         /* allocate again next time */
2799         reg->alloc = 0;
2800 }
2801
2802 static struct event_constraint *
2803 intel_shared_regs_constraints(struct cpu_hw_events *cpuc,
2804                               struct perf_event *event)
2805 {
2806         struct event_constraint *c = NULL, *d;
2807         struct hw_perf_event_extra *xreg, *breg;
2808
2809         xreg = &event->hw.extra_reg;
2810         if (xreg->idx != EXTRA_REG_NONE) {
2811                 c = __intel_shared_reg_get_constraints(cpuc, event, xreg);
2812                 if (c == &emptyconstraint)
2813                         return c;
2814         }
2815         breg = &event->hw.branch_reg;
2816         if (breg->idx != EXTRA_REG_NONE) {
2817                 d = __intel_shared_reg_get_constraints(cpuc, event, breg);
2818                 if (d == &emptyconstraint) {
2819                         __intel_shared_reg_put_constraints(cpuc, xreg);
2820                         c = d;
2821                 }
2822         }
2823         return c;
2824 }
2825
2826 struct event_constraint *
2827 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
2828                           struct perf_event *event)
2829 {
2830         struct event_constraint *c;
2831
2832         if (x86_pmu.event_constraints) {
2833                 for_each_event_constraint(c, x86_pmu.event_constraints) {
2834                         if (constraint_match(c, event->hw.config)) {
2835                                 event->hw.flags |= c->flags;
2836                                 return c;
2837                         }
2838                 }
2839         }
2840
2841         return &unconstrained;
2842 }
2843
2844 static struct event_constraint *
2845 __intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
2846                             struct perf_event *event)
2847 {
2848         struct event_constraint *c;
2849
2850         c = intel_vlbr_constraints(event);
2851         if (c)
2852                 return c;
2853
2854         c = intel_bts_constraints(event);
2855         if (c)
2856                 return c;
2857
2858         c = intel_shared_regs_constraints(cpuc, event);
2859         if (c)
2860                 return c;
2861
2862         c = intel_pebs_constraints(event);
2863         if (c)
2864                 return c;
2865
2866         return x86_get_event_constraints(cpuc, idx, event);
2867 }
2868
2869 static void
2870 intel_start_scheduling(struct cpu_hw_events *cpuc)
2871 {
2872         struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2873         struct intel_excl_states *xl;
2874         int tid = cpuc->excl_thread_id;
2875
2876         /*
2877          * nothing needed if in group validation mode
2878          */
2879         if (cpuc->is_fake || !is_ht_workaround_enabled())
2880                 return;
2881
2882         /*
2883          * no exclusion needed
2884          */
2885         if (WARN_ON_ONCE(!excl_cntrs))
2886                 return;
2887
2888         xl = &excl_cntrs->states[tid];
2889
2890         xl->sched_started = true;
2891         /*
2892          * lock shared state until we are done scheduling
2893          * in stop_event_scheduling()
2894          * makes scheduling appear as a transaction
2895          */
2896         raw_spin_lock(&excl_cntrs->lock);
2897 }
2898
2899 static void intel_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)
2900 {
2901         struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2902         struct event_constraint *c = cpuc->event_constraint[idx];
2903         struct intel_excl_states *xl;
2904         int tid = cpuc->excl_thread_id;
2905
2906         if (cpuc->is_fake || !is_ht_workaround_enabled())
2907                 return;
2908
2909         if (WARN_ON_ONCE(!excl_cntrs))
2910                 return;
2911
2912         if (!(c->flags & PERF_X86_EVENT_DYNAMIC))
2913                 return;
2914
2915         xl = &excl_cntrs->states[tid];
2916
2917         lockdep_assert_held(&excl_cntrs->lock);
2918
2919         if (c->flags & PERF_X86_EVENT_EXCL)
2920                 xl->state[cntr] = INTEL_EXCL_EXCLUSIVE;
2921         else
2922                 xl->state[cntr] = INTEL_EXCL_SHARED;
2923 }
2924
2925 static void
2926 intel_stop_scheduling(struct cpu_hw_events *cpuc)
2927 {
2928         struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2929         struct intel_excl_states *xl;
2930         int tid = cpuc->excl_thread_id;
2931
2932         /*
2933          * nothing needed if in group validation mode
2934          */
2935         if (cpuc->is_fake || !is_ht_workaround_enabled())
2936                 return;
2937         /*
2938          * no exclusion needed
2939          */
2940         if (WARN_ON_ONCE(!excl_cntrs))
2941                 return;
2942
2943         xl = &excl_cntrs->states[tid];
2944
2945         xl->sched_started = false;
2946         /*
2947          * release shared state lock (acquired in intel_start_scheduling())
2948          */
2949         raw_spin_unlock(&excl_cntrs->lock);
2950 }
2951
2952 static struct event_constraint *
2953 dyn_constraint(struct cpu_hw_events *cpuc, struct event_constraint *c, int idx)
2954 {
2955         WARN_ON_ONCE(!cpuc->constraint_list);
2956
2957         if (!(c->flags & PERF_X86_EVENT_DYNAMIC)) {
2958                 struct event_constraint *cx;
2959
2960                 /*
2961                  * grab pre-allocated constraint entry
2962                  */
2963                 cx = &cpuc->constraint_list[idx];
2964
2965                 /*
2966                  * initialize dynamic constraint
2967                  * with static constraint
2968                  */
2969                 *cx = *c;
2970
2971                 /*
2972                  * mark constraint as dynamic
2973                  */
2974                 cx->flags |= PERF_X86_EVENT_DYNAMIC;
2975                 c = cx;
2976         }
2977
2978         return c;
2979 }
2980
2981 static struct event_constraint *
2982 intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
2983                            int idx, struct event_constraint *c)
2984 {
2985         struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2986         struct intel_excl_states *xlo;
2987         int tid = cpuc->excl_thread_id;
2988         int is_excl, i, w;
2989
2990         /*
2991          * validating a group does not require
2992          * enforcing cross-thread  exclusion
2993          */
2994         if (cpuc->is_fake || !is_ht_workaround_enabled())
2995                 return c;
2996
2997         /*
2998          * no exclusion needed
2999          */
3000         if (WARN_ON_ONCE(!excl_cntrs))
3001                 return c;
3002
3003         /*
3004          * because we modify the constraint, we need
3005          * to make a copy. Static constraints come
3006          * from static const tables.
3007          *
3008          * only needed when constraint has not yet
3009          * been cloned (marked dynamic)
3010          */
3011         c = dyn_constraint(cpuc, c, idx);
3012
3013         /*
3014          * From here on, the constraint is dynamic.
3015          * Either it was just allocated above, or it
3016          * was allocated during a earlier invocation
3017          * of this function
3018          */
3019
3020         /*
3021          * state of sibling HT
3022          */
3023         xlo = &excl_cntrs->states[tid ^ 1];
3024
3025         /*
3026          * event requires exclusive counter access
3027          * across HT threads
3028          */
3029         is_excl = c->flags & PERF_X86_EVENT_EXCL;
3030         if (is_excl && !(event->hw.flags & PERF_X86_EVENT_EXCL_ACCT)) {
3031                 event->hw.flags |= PERF_X86_EVENT_EXCL_ACCT;
3032                 if (!cpuc->n_excl++)
3033                         WRITE_ONCE(excl_cntrs->has_exclusive[tid], 1);
3034         }
3035
3036         /*
3037          * Modify static constraint with current dynamic
3038          * state of thread
3039          *
3040          * EXCLUSIVE: sibling counter measuring exclusive event
3041          * SHARED   : sibling counter measuring non-exclusive event
3042          * UNUSED   : sibling counter unused
3043          */
3044         w = c->weight;
3045         for_each_set_bit(i, c->idxmsk, X86_PMC_IDX_MAX) {
3046                 /*
3047                  * exclusive event in sibling counter
3048                  * our corresponding counter cannot be used
3049                  * regardless of our event
3050                  */
3051                 if (xlo->state[i] == INTEL_EXCL_EXCLUSIVE) {
3052                         __clear_bit(i, c->idxmsk);
3053                         w--;
3054                         continue;
3055                 }
3056                 /*
3057                  * if measuring an exclusive event, sibling
3058                  * measuring non-exclusive, then counter cannot
3059                  * be used
3060                  */
3061                 if (is_excl && xlo->state[i] == INTEL_EXCL_SHARED) {
3062                         __clear_bit(i, c->idxmsk);
3063                         w--;
3064                         continue;
3065                 }
3066         }
3067
3068         /*
3069          * if we return an empty mask, then switch
3070          * back to static empty constraint to avoid
3071          * the cost of freeing later on
3072          */
3073         if (!w)
3074                 c = &emptyconstraint;
3075
3076         c->weight = w;
3077
3078         return c;
3079 }
3080
3081 static struct event_constraint *
3082 intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3083                             struct perf_event *event)
3084 {
3085         struct event_constraint *c1, *c2;
3086
3087         c1 = cpuc->event_constraint[idx];
3088
3089         /*
3090          * first time only
3091          * - static constraint: no change across incremental scheduling calls
3092          * - dynamic constraint: handled by intel_get_excl_constraints()
3093          */
3094         c2 = __intel_get_event_constraints(cpuc, idx, event);
3095         if (c1) {
3096                 WARN_ON_ONCE(!(c1->flags & PERF_X86_EVENT_DYNAMIC));
3097                 bitmap_copy(c1->idxmsk, c2->idxmsk, X86_PMC_IDX_MAX);
3098                 c1->weight = c2->weight;
3099                 c2 = c1;
3100         }
3101
3102         if (cpuc->excl_cntrs)
3103                 return intel_get_excl_constraints(cpuc, event, idx, c2);
3104
3105         return c2;
3106 }
3107
3108 static void intel_put_excl_constraints(struct cpu_hw_events *cpuc,
3109                 struct perf_event *event)
3110 {
3111         struct hw_perf_event *hwc = &event->hw;
3112         struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
3113         int tid = cpuc->excl_thread_id;
3114         struct intel_excl_states *xl;
3115
3116         /*
3117          * nothing needed if in group validation mode
3118          */
3119         if (cpuc->is_fake)
3120                 return;
3121
3122         if (WARN_ON_ONCE(!excl_cntrs))
3123                 return;
3124
3125         if (hwc->flags & PERF_X86_EVENT_EXCL_ACCT) {
3126                 hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT;
3127                 if (!--cpuc->n_excl)
3128                         WRITE_ONCE(excl_cntrs->has_exclusive[tid], 0);
3129         }
3130
3131         /*
3132          * If event was actually assigned, then mark the counter state as
3133          * unused now.
3134          */
3135         if (hwc->idx >= 0) {
3136                 xl = &excl_cntrs->states[tid];
3137
3138                 /*
3139                  * put_constraint may be called from x86_schedule_events()
3140                  * which already has the lock held so here make locking
3141                  * conditional.
3142                  */
3143                 if (!xl->sched_started)
3144                         raw_spin_lock(&excl_cntrs->lock);
3145
3146                 xl->state[hwc->idx] = INTEL_EXCL_UNUSED;
3147
3148                 if (!xl->sched_started)
3149                         raw_spin_unlock(&excl_cntrs->lock);
3150         }
3151 }
3152
3153 static void
3154 intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,
3155                                         struct perf_event *event)
3156 {
3157         struct hw_perf_event_extra *reg;
3158
3159         reg = &event->hw.extra_reg;
3160         if (reg->idx != EXTRA_REG_NONE)
3161                 __intel_shared_reg_put_constraints(cpuc, reg);
3162
3163         reg = &event->hw.branch_reg;
3164         if (reg->idx != EXTRA_REG_NONE)
3165                 __intel_shared_reg_put_constraints(cpuc, reg);
3166 }
3167
3168 static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
3169                                         struct perf_event *event)
3170 {
3171         intel_put_shared_regs_event_constraints(cpuc, event);
3172
3173         /*
3174          * is PMU has exclusive counter restrictions, then
3175          * all events are subject to and must call the
3176          * put_excl_constraints() routine
3177          */
3178         if (cpuc->excl_cntrs)
3179                 intel_put_excl_constraints(cpuc, event);
3180 }
3181
3182 static void intel_pebs_aliases_core2(struct perf_event *event)
3183 {
3184         if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
3185                 /*
3186                  * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
3187                  * (0x003c) so that we can use it with PEBS.
3188                  *
3189                  * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
3190                  * PEBS capable. However we can use INST_RETIRED.ANY_P
3191                  * (0x00c0), which is a PEBS capable event, to get the same
3192                  * count.
3193                  *
3194                  * INST_RETIRED.ANY_P counts the number of cycles that retires
3195                  * CNTMASK instructions. By setting CNTMASK to a value (16)
3196                  * larger than the maximum number of instructions that can be
3197                  * retired per cycle (4) and then inverting the condition, we
3198                  * count all cycles that retire 16 or less instructions, which
3199                  * is every cycle.
3200                  *
3201                  * Thereby we gain a PEBS capable cycle counter.
3202                  */
3203                 u64 alt_config = X86_CONFIG(.event=0xc0, .inv=1, .cmask=16);
3204
3205                 alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
3206                 event->hw.config = alt_config;
3207         }
3208 }
3209
3210 static void intel_pebs_aliases_snb(struct perf_event *event)
3211 {
3212         if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
3213                 /*
3214                  * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
3215                  * (0x003c) so that we can use it with PEBS.
3216                  *
3217                  * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
3218                  * PEBS capable. However we can use UOPS_RETIRED.ALL
3219                  * (0x01c2), which is a PEBS capable event, to get the same
3220                  * count.
3221                  *
3222                  * UOPS_RETIRED.ALL counts the number of cycles that retires
3223                  * CNTMASK micro-ops. By setting CNTMASK to a value (16)
3224                  * larger than the maximum number of micro-ops that can be
3225                  * retired per cycle (4) and then inverting the condition, we
3226                  * count all cycles that retire 16 or less micro-ops, which
3227                  * is every cycle.
3228                  *
3229                  * Thereby we gain a PEBS capable cycle counter.
3230                  */
3231                 u64 alt_config = X86_CONFIG(.event=0xc2, .umask=0x01, .inv=1, .cmask=16);
3232
3233                 alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
3234                 event->hw.config = alt_config;
3235         }
3236 }
3237
3238 static void intel_pebs_aliases_precdist(struct perf_event *event)
3239 {
3240         if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
3241                 /*
3242                  * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
3243                  * (0x003c) so that we can use it with PEBS.
3244                  *
3245                  * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
3246                  * PEBS capable. However we can use INST_RETIRED.PREC_DIST
3247                  * (0x01c0), which is a PEBS capable event, to get the same
3248                  * count.
3249                  *
3250                  * The PREC_DIST event has special support to minimize sample
3251                  * shadowing effects. One drawback is that it can be
3252                  * only programmed on counter 1, but that seems like an
3253                  * acceptable trade off.
3254                  */
3255                 u64 alt_config = X86_CONFIG(.event=0xc0, .umask=0x01, .inv=1, .cmask=16);
3256
3257                 alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
3258                 event->hw.config = alt_config;
3259         }
3260 }
3261
3262 static void intel_pebs_aliases_ivb(struct perf_event *event)
3263 {
3264         if (event->attr.precise_ip < 3)
3265                 return intel_pebs_aliases_snb(event);
3266         return intel_pebs_aliases_precdist(event);
3267 }
3268
3269 static void intel_pebs_aliases_skl(struct perf_event *event)
3270 {
3271         if (event->attr.precise_ip < 3)
3272                 return intel_pebs_aliases_core2(event);
3273         return intel_pebs_aliases_precdist(event);
3274 }
3275
3276 static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event)
3277 {
3278         unsigned long flags = x86_pmu.large_pebs_flags;
3279
3280         if (event->attr.use_clockid)
3281                 flags &= ~PERF_SAMPLE_TIME;
3282         if (!event->attr.exclude_kernel)
3283                 flags &= ~PERF_SAMPLE_REGS_USER;
3284         if (event->attr.sample_regs_user & ~PEBS_GP_REGS)
3285                 flags &= ~(PERF_SAMPLE_REGS_USER | PERF_SAMPLE_REGS_INTR);
3286         return flags;
3287 }
3288
3289 static int intel_pmu_bts_config(struct perf_event *event)
3290 {
3291         struct perf_event_attr *attr = &event->attr;
3292
3293         if (unlikely(intel_pmu_has_bts(event))) {
3294                 /* BTS is not supported by this architecture. */
3295                 if (!x86_pmu.bts_active)
3296                         return -EOPNOTSUPP;
3297
3298                 /* BTS is currently only allowed for user-mode. */
3299                 if (!attr->exclude_kernel)
3300                         return -EOPNOTSUPP;
3301
3302                 /* BTS is not allowed for precise events. */
3303                 if (attr->precise_ip)
3304                         return -EOPNOTSUPP;
3305
3306                 /* disallow bts if conflicting events are present */
3307                 if (x86_add_exclusive(x86_lbr_exclusive_lbr))
3308                         return -EBUSY;
3309
3310                 event->destroy = hw_perf_lbr_event_destroy;
3311         }
3312
3313         return 0;
3314 }
3315
3316 static int core_pmu_hw_config(struct perf_event *event)
3317 {
3318         int ret = x86_pmu_hw_config(event);
3319
3320         if (ret)
3321                 return ret;
3322
3323         return intel_pmu_bts_config(event);
3324 }
3325
3326 static int intel_pmu_hw_config(struct perf_event *event)
3327 {
3328         int ret = x86_pmu_hw_config(event);
3329
3330         if (ret)
3331                 return ret;
3332
3333         ret = intel_pmu_bts_config(event);
3334         if (ret)
3335                 return ret;
3336
3337         if (event->attr.precise_ip) {
3338                 if (!(event->attr.freq || (event->attr.wakeup_events && !event->attr.watermark))) {
3339                         event->hw.flags |= PERF_X86_EVENT_AUTO_RELOAD;
3340                         if (!(event->attr.sample_type &
3341                               ~intel_pmu_large_pebs_flags(event)))
3342                                 event->hw.flags |= PERF_X86_EVENT_LARGE_PEBS;
3343                 }
3344                 if (x86_pmu.pebs_aliases)
3345                         x86_pmu.pebs_aliases(event);
3346
3347                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
3348                         event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY;
3349         }
3350
3351         if (needs_branch_stack(event)) {
3352                 ret = intel_pmu_setup_lbr_filter(event);
3353                 if (ret)
3354                         return ret;
3355
3356                 /*
3357                  * BTS is set up earlier in this path, so don't account twice
3358                  */
3359                 if (!unlikely(intel_pmu_has_bts(event))) {
3360                         /* disallow lbr if conflicting events are present */
3361                         if (x86_add_exclusive(x86_lbr_exclusive_lbr))
3362                                 return -EBUSY;
3363
3364                         event->destroy = hw_perf_lbr_event_destroy;
3365                 }
3366         }
3367
3368         if (event->attr.aux_output) {
3369                 if (!event->attr.precise_ip)
3370                         return -EINVAL;
3371
3372                 event->hw.flags |= PERF_X86_EVENT_PEBS_VIA_PT;
3373         }
3374
3375         if (event->attr.type != PERF_TYPE_RAW)
3376                 return 0;
3377
3378         if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))
3379                 return 0;
3380
3381         if (x86_pmu.version < 3)
3382                 return -EINVAL;
3383
3384         ret = perf_allow_cpu(&event->attr);
3385         if (ret)
3386                 return ret;
3387
3388         event->hw.config |= ARCH_PERFMON_EVENTSEL_ANY;
3389
3390         return 0;
3391 }
3392
3393 #ifdef CONFIG_RETPOLINE
3394 static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr);
3395 static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr);
3396 #endif
3397
3398 struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr)
3399 {
3400 #ifdef CONFIG_RETPOLINE
3401         if (x86_pmu.guest_get_msrs == intel_guest_get_msrs)
3402                 return intel_guest_get_msrs(nr);
3403         else if (x86_pmu.guest_get_msrs == core_guest_get_msrs)
3404                 return core_guest_get_msrs(nr);
3405 #endif
3406         if (x86_pmu.guest_get_msrs)
3407                 return x86_pmu.guest_get_msrs(nr);
3408         *nr = 0;
3409         return NULL;
3410 }
3411 EXPORT_SYMBOL_GPL(perf_guest_get_msrs);
3412
3413 static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr)
3414 {
3415         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
3416         struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
3417
3418         arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL;
3419         arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;
3420         arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;
3421         if (x86_pmu.flags & PMU_FL_PEBS_ALL)
3422                 arr[0].guest &= ~cpuc->pebs_enabled;
3423         else
3424                 arr[0].guest &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK);
3425         *nr = 1;
3426
3427         if (x86_pmu.pebs && x86_pmu.pebs_no_isolation) {
3428                 /*
3429                  * If PMU counter has PEBS enabled it is not enough to
3430                  * disable counter on a guest entry since PEBS memory
3431                  * write can overshoot guest entry and corrupt guest
3432                  * memory. Disabling PEBS solves the problem.
3433                  *
3434                  * Don't do this if the CPU already enforces it.
3435                  */
3436                 arr[1].msr = MSR_IA32_PEBS_ENABLE;
3437                 arr[1].host = cpuc->pebs_enabled;
3438                 arr[1].guest = 0;
3439                 *nr = 2;
3440         }
3441
3442         return arr;
3443 }
3444
3445 static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr)
3446 {
3447         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
3448         struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
3449         int idx;
3450
3451         for (idx = 0; idx < x86_pmu.num_counters; idx++)  {
3452                 struct perf_event *event = cpuc->events[idx];
3453
3454                 arr[idx].msr = x86_pmu_config_addr(idx);
3455                 arr[idx].host = arr[idx].guest = 0;
3456
3457                 if (!test_bit(idx, cpuc->active_mask))
3458                         continue;
3459
3460                 arr[idx].host = arr[idx].guest =
3461                         event->hw.config | ARCH_PERFMON_EVENTSEL_ENABLE;
3462
3463                 if (event->attr.exclude_host)
3464                         arr[idx].host &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
3465                 else if (event->attr.exclude_guest)
3466                         arr[idx].guest &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
3467         }
3468
3469         *nr = x86_pmu.num_counters;
3470         return arr;
3471 }
3472
3473 static void core_pmu_enable_event(struct perf_event *event)
3474 {
3475         if (!event->attr.exclude_host)
3476                 x86_pmu_enable_event(event);
3477 }
3478
3479 static void core_pmu_enable_all(int added)
3480 {
3481         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
3482         int idx;
3483
3484         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
3485                 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
3486
3487                 if (!test_bit(idx, cpuc->active_mask) ||
3488                                 cpuc->events[idx]->attr.exclude_host)
3489                         continue;
3490
3491                 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
3492         }
3493 }
3494
3495 static int hsw_hw_config(struct perf_event *event)
3496 {
3497         int ret = intel_pmu_hw_config(event);
3498
3499         if (ret)
3500                 return ret;
3501         if (!boot_cpu_has(X86_FEATURE_RTM) && !boot_cpu_has(X86_FEATURE_HLE))
3502                 return 0;
3503         event->hw.config |= event->attr.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED);
3504
3505         /*
3506          * IN_TX/IN_TX-CP filters are not supported by the Haswell PMU with
3507          * PEBS or in ANY thread mode. Since the results are non-sensical forbid
3508          * this combination.
3509          */
3510         if ((event->hw.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)) &&
3511              ((event->hw.config & ARCH_PERFMON_EVENTSEL_ANY) ||
3512               event->attr.precise_ip > 0))
3513                 return -EOPNOTSUPP;
3514
3515         if (event_is_checkpointed(event)) {
3516                 /*
3517                  * Sampling of checkpointed events can cause situations where
3518                  * the CPU constantly aborts because of a overflow, which is
3519                  * then checkpointed back and ignored. Forbid checkpointing
3520                  * for sampling.
3521                  *
3522                  * But still allow a long sampling period, so that perf stat
3523                  * from KVM works.
3524                  */
3525                 if (event->attr.sample_period > 0 &&
3526                     event->attr.sample_period < 0x7fffffff)
3527                         return -EOPNOTSUPP;
3528         }
3529         return 0;
3530 }
3531
3532 static struct event_constraint counter0_constraint =
3533                         INTEL_ALL_EVENT_CONSTRAINT(0, 0x1);
3534
3535 static struct event_constraint counter2_constraint =
3536                         EVENT_CONSTRAINT(0, 0x4, 0);
3537
3538 static struct event_constraint fixed0_constraint =
3539                         FIXED_EVENT_CONSTRAINT(0x00c0, 0);
3540
3541 static struct event_constraint fixed0_counter0_constraint =
3542                         INTEL_ALL_EVENT_CONSTRAINT(0, 0x100000001ULL);
3543
3544 static struct event_constraint *
3545 hsw_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3546                           struct perf_event *event)
3547 {
3548         struct event_constraint *c;
3549
3550         c = intel_get_event_constraints(cpuc, idx, event);
3551
3552         /* Handle special quirk on in_tx_checkpointed only in counter 2 */
3553         if (event->hw.config & HSW_IN_TX_CHECKPOINTED) {
3554                 if (c->idxmsk64 & (1U << 2))
3555                         return &counter2_constraint;
3556                 return &emptyconstraint;
3557         }
3558
3559         return c;
3560 }
3561
3562 static struct event_constraint *
3563 icl_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3564                           struct perf_event *event)
3565 {
3566         /*
3567          * Fixed counter 0 has less skid.
3568          * Force instruction:ppp in Fixed counter 0
3569          */
3570         if ((event->attr.precise_ip == 3) &&
3571             constraint_match(&fixed0_constraint, event->hw.config))
3572                 return &fixed0_constraint;
3573
3574         return hsw_get_event_constraints(cpuc, idx, event);
3575 }
3576
3577 static struct event_constraint *
3578 glp_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3579                           struct perf_event *event)
3580 {
3581         struct event_constraint *c;
3582
3583         /* :ppp means to do reduced skid PEBS which is PMC0 only. */
3584         if (event->attr.precise_ip == 3)
3585                 return &counter0_constraint;
3586
3587         c = intel_get_event_constraints(cpuc, idx, event);
3588
3589         return c;
3590 }
3591
3592 static struct event_constraint *
3593 tnt_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3594                           struct perf_event *event)
3595 {
3596         struct event_constraint *c;
3597
3598         /*
3599          * :ppp means to do reduced skid PEBS,
3600          * which is available on PMC0 and fixed counter 0.
3601          */
3602         if (event->attr.precise_ip == 3) {
3603                 /* Force instruction:ppp on PMC0 and Fixed counter 0 */
3604                 if (constraint_match(&fixed0_constraint, event->hw.config))
3605                         return &fixed0_counter0_constraint;
3606
3607                 return &counter0_constraint;
3608         }
3609
3610         c = intel_get_event_constraints(cpuc, idx, event);
3611
3612         return c;
3613 }
3614
3615 static bool allow_tsx_force_abort = true;
3616
3617 static struct event_constraint *
3618 tfa_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3619                           struct perf_event *event)
3620 {
3621         struct event_constraint *c = hsw_get_event_constraints(cpuc, idx, event);
3622
3623         /*
3624          * Without TFA we must not use PMC3.
3625          */
3626         if (!allow_tsx_force_abort && test_bit(3, c->idxmsk)) {
3627                 c = dyn_constraint(cpuc, c, idx);
3628                 c->idxmsk64 &= ~(1ULL << 3);
3629                 c->weight--;
3630         }
3631
3632         return c;
3633 }
3634
3635 /*
3636  * Broadwell:
3637  *
3638  * The INST_RETIRED.ALL period always needs to have lowest 6 bits cleared
3639  * (BDM55) and it must not use a period smaller than 100 (BDM11). We combine
3640  * the two to enforce a minimum period of 128 (the smallest value that has bits
3641  * 0-5 cleared and >= 100).
3642  *
3643  * Because of how the code in x86_perf_event_set_period() works, the truncation
3644  * of the lower 6 bits is 'harmless' as we'll occasionally add a longer period
3645  * to make up for the 'lost' events due to carrying the 'error' in period_left.
3646  *
3647  * Therefore the effective (average) period matches the requested period,
3648  * despite coarser hardware granularity.
3649  */
3650 static u64 bdw_limit_period(struct perf_event *event, u64 left)
3651 {
3652         if ((event->hw.config & INTEL_ARCH_EVENT_MASK) ==
3653                         X86_CONFIG(.event=0xc0, .umask=0x01)) {
3654                 if (left < 128)
3655                         left = 128;
3656                 left &= ~0x3fULL;
3657         }
3658         return left;
3659 }
3660
3661 static u64 nhm_limit_period(struct perf_event *event, u64 left)
3662 {
3663         return max(left, 32ULL);
3664 }
3665
3666 PMU_FORMAT_ATTR(event,  "config:0-7"    );
3667 PMU_FORMAT_ATTR(umask,  "config:8-15"   );
3668 PMU_FORMAT_ATTR(edge,   "config:18"     );
3669 PMU_FORMAT_ATTR(pc,     "config:19"     );
3670 PMU_FORMAT_ATTR(any,    "config:21"     ); /* v3 + */
3671 PMU_FORMAT_ATTR(inv,    "config:23"     );
3672 PMU_FORMAT_ATTR(cmask,  "config:24-31"  );
3673 PMU_FORMAT_ATTR(in_tx,  "config:32");
3674 PMU_FORMAT_ATTR(in_tx_cp, "config:33");
3675
3676 static struct attribute *intel_arch_formats_attr[] = {
3677         &format_attr_event.attr,
3678         &format_attr_umask.attr,
3679         &format_attr_edge.attr,
3680         &format_attr_pc.attr,
3681         &format_attr_inv.attr,
3682         &format_attr_cmask.attr,
3683         NULL,
3684 };
3685
3686 ssize_t intel_event_sysfs_show(char *page, u64 config)
3687 {
3688         u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);
3689
3690         return x86_event_sysfs_show(page, config, event);
3691 }
3692
3693 static struct intel_shared_regs *allocate_shared_regs(int cpu)
3694 {
3695         struct intel_shared_regs *regs;
3696         int i;
3697
3698         regs = kzalloc_node(sizeof(struct intel_shared_regs),
3699                             GFP_KERNEL, cpu_to_node(cpu));
3700         if (regs) {
3701                 /*
3702                  * initialize the locks to keep lockdep happy
3703                  */
3704                 for (i = 0; i < EXTRA_REG_MAX; i++)
3705                         raw_spin_lock_init(&regs->regs[i].lock);
3706
3707                 regs->core_id = -1;
3708         }
3709         return regs;
3710 }
3711
3712 static struct intel_excl_cntrs *allocate_excl_cntrs(int cpu)
3713 {
3714         struct intel_excl_cntrs *c;
3715
3716         c = kzalloc_node(sizeof(struct intel_excl_cntrs),
3717                          GFP_KERNEL, cpu_to_node(cpu));
3718         if (c) {
3719                 raw_spin_lock_init(&c->lock);
3720                 c->core_id = -1;
3721         }
3722         return c;
3723 }
3724
3725
3726 int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
3727 {
3728         cpuc->pebs_record_size = x86_pmu.pebs_record_size;
3729
3730         if (x86_pmu.extra_regs || x86_pmu.lbr_sel_map) {
3731                 cpuc->shared_regs = allocate_shared_regs(cpu);
3732                 if (!cpuc->shared_regs)
3733                         goto err;
3734         }
3735
3736         if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA)) {
3737                 size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint);
3738
3739                 cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
3740                 if (!cpuc->constraint_list)
3741                         goto err_shared_regs;
3742         }
3743
3744         if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {
3745                 cpuc->excl_cntrs = allocate_excl_cntrs(cpu);
3746                 if (!cpuc->excl_cntrs)
3747                         goto err_constraint_list;
3748
3749                 cpuc->excl_thread_id = 0;
3750         }
3751
3752         return 0;
3753
3754 err_constraint_list:
3755         kfree(cpuc->constraint_list);
3756         cpuc->constraint_list = NULL;
3757
3758 err_shared_regs:
3759         kfree(cpuc->shared_regs);
3760         cpuc->shared_regs = NULL;
3761
3762 err:
3763         return -ENOMEM;
3764 }
3765
3766 static int intel_pmu_cpu_prepare(int cpu)
3767 {
3768         return intel_cpuc_prepare(&per_cpu(cpu_hw_events, cpu), cpu);
3769 }
3770
3771 static void flip_smm_bit(void *data)
3772 {
3773         unsigned long set = *(unsigned long *)data;
3774
3775         if (set > 0) {
3776                 msr_set_bit(MSR_IA32_DEBUGCTLMSR,
3777                             DEBUGCTLMSR_FREEZE_IN_SMM_BIT);
3778         } else {
3779                 msr_clear_bit(MSR_IA32_DEBUGCTLMSR,
3780                               DEBUGCTLMSR_FREEZE_IN_SMM_BIT);
3781         }
3782 }
3783
3784 static void intel_pmu_cpu_starting(int cpu)
3785 {
3786         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
3787         int core_id = topology_core_id(cpu);
3788         int i;
3789
3790         init_debug_store_on_cpu(cpu);
3791         /*
3792          * Deal with CPUs that don't clear their LBRs on power-up.
3793          */
3794         intel_pmu_lbr_reset();
3795
3796         cpuc->lbr_sel = NULL;
3797
3798         if (x86_pmu.flags & PMU_FL_TFA) {
3799                 WARN_ON_ONCE(cpuc->tfa_shadow);
3800                 cpuc->tfa_shadow = ~0ULL;
3801                 intel_set_tfa(cpuc, false);
3802         }
3803
3804         if (x86_pmu.version > 1)
3805                 flip_smm_bit(&x86_pmu.attr_freeze_on_smi);
3806
3807         if (x86_pmu.counter_freezing)
3808                 enable_counter_freeze();
3809
3810         if (!cpuc->shared_regs)
3811                 return;
3812
3813         if (!(x86_pmu.flags & PMU_FL_NO_HT_SHARING)) {
3814                 for_each_cpu(i, topology_sibling_cpumask(cpu)) {
3815                         struct intel_shared_regs *pc;
3816
3817                         pc = per_cpu(cpu_hw_events, i).shared_regs;
3818                         if (pc && pc->core_id == core_id) {
3819                                 cpuc->kfree_on_online[0] = cpuc->shared_regs;
3820                                 cpuc->shared_regs = pc;
3821                                 break;
3822                         }
3823                 }
3824                 cpuc->shared_regs->core_id = core_id;
3825                 cpuc->shared_regs->refcnt++;
3826         }
3827
3828         if (x86_pmu.lbr_sel_map)
3829                 cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];
3830
3831         if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {
3832                 for_each_cpu(i, topology_sibling_cpumask(cpu)) {
3833                         struct cpu_hw_events *sibling;
3834                         struct intel_excl_cntrs *c;
3835
3836                         sibling = &per_cpu(cpu_hw_events, i);
3837                         c = sibling->excl_cntrs;
3838                         if (c && c->core_id == core_id) {
3839                                 cpuc->kfree_on_online[1] = cpuc->excl_cntrs;
3840                                 cpuc->excl_cntrs = c;
3841                                 if (!sibling->excl_thread_id)
3842                                         cpuc->excl_thread_id = 1;
3843                                 break;
3844                         }
3845                 }
3846                 cpuc->excl_cntrs->core_id = core_id;
3847                 cpuc->excl_cntrs->refcnt++;
3848         }
3849 }
3850
3851 static void free_excl_cntrs(struct cpu_hw_events *cpuc)
3852 {
3853         struct intel_excl_cntrs *c;
3854
3855         c = cpuc->excl_cntrs;
3856         if (c) {
3857                 if (c->core_id == -1 || --c->refcnt == 0)
3858                         kfree(c);
3859                 cpuc->excl_cntrs = NULL;
3860         }
3861
3862         kfree(cpuc->constraint_list);
3863         cpuc->constraint_list = NULL;
3864 }
3865
3866 static void intel_pmu_cpu_dying(int cpu)
3867 {
3868         fini_debug_store_on_cpu(cpu);
3869
3870         if (x86_pmu.counter_freezing)
3871                 disable_counter_freeze();
3872 }
3873
3874 void intel_cpuc_finish(struct cpu_hw_events *cpuc)
3875 {
3876         struct intel_shared_regs *pc;
3877
3878         pc = cpuc->shared_regs;
3879         if (pc) {
3880                 if (pc->core_id == -1 || --pc->refcnt == 0)
3881                         kfree(pc);
3882                 cpuc->shared_regs = NULL;
3883         }
3884
3885         free_excl_cntrs(cpuc);
3886 }
3887
3888 static void intel_pmu_cpu_dead(int cpu)
3889 {
3890         intel_cpuc_finish(&per_cpu(cpu_hw_events, cpu));
3891 }
3892
3893 static void intel_pmu_sched_task(struct perf_event_context *ctx,
3894                                  bool sched_in)
3895 {
3896         intel_pmu_pebs_sched_task(ctx, sched_in);
3897         intel_pmu_lbr_sched_task(ctx, sched_in);
3898 }
3899
3900 static void intel_pmu_swap_task_ctx(struct perf_event_context *prev,
3901                                     struct perf_event_context *next)
3902 {
3903         intel_pmu_lbr_swap_task_ctx(prev, next);
3904 }
3905
3906 static int intel_pmu_check_period(struct perf_event *event, u64 value)
3907 {
3908         return intel_pmu_has_bts_period(event, value) ? -EINVAL : 0;
3909 }
3910
3911 static int intel_pmu_aux_output_match(struct perf_event *event)
3912 {
3913         if (!x86_pmu.intel_cap.pebs_output_pt_available)
3914                 return 0;
3915
3916         return is_intel_pt_event(event);
3917 }
3918
3919 PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
3920
3921 PMU_FORMAT_ATTR(ldlat, "config1:0-15");
3922
3923 PMU_FORMAT_ATTR(frontend, "config1:0-23");
3924
3925 static struct attribute *intel_arch3_formats_attr[] = {
3926         &format_attr_event.attr,
3927         &format_attr_umask.attr,
3928         &format_attr_edge.attr,
3929         &format_attr_pc.attr,
3930         &format_attr_any.attr,
3931         &format_attr_inv.attr,
3932         &format_attr_cmask.attr,
3933         NULL,
3934 };
3935
3936 static struct attribute *hsw_format_attr[] = {
3937         &format_attr_in_tx.attr,
3938         &format_attr_in_tx_cp.attr,
3939         &format_attr_offcore_rsp.attr,
3940         &format_attr_ldlat.attr,
3941         NULL
3942 };
3943
3944 static struct attribute *nhm_format_attr[] = {
3945         &format_attr_offcore_rsp.attr,
3946         &format_attr_ldlat.attr,
3947         NULL
3948 };
3949
3950 static struct attribute *slm_format_attr[] = {
3951         &format_attr_offcore_rsp.attr,
3952         NULL
3953 };
3954
3955 static struct attribute *skl_format_attr[] = {
3956         &format_attr_frontend.attr,
3957         NULL,
3958 };
3959
3960 static __initconst const struct x86_pmu core_pmu = {
3961         .name                   = "core",
3962         .handle_irq             = x86_pmu_handle_irq,
3963         .disable_all            = x86_pmu_disable_all,
3964         .enable_all             = core_pmu_enable_all,
3965         .enable                 = core_pmu_enable_event,
3966         .disable                = x86_pmu_disable_event,
3967         .hw_config              = core_pmu_hw_config,
3968         .schedule_events        = x86_schedule_events,
3969         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
3970         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
3971         .event_map              = intel_pmu_event_map,
3972         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
3973         .apic                   = 1,
3974         .large_pebs_flags       = LARGE_PEBS_FLAGS,
3975
3976         /*
3977          * Intel PMCs cannot be accessed sanely above 32-bit width,
3978          * so we install an artificial 1<<31 period regardless of
3979          * the generic event period:
3980          */
3981         .max_period             = (1ULL<<31) - 1,
3982         .get_event_constraints  = intel_get_event_constraints,
3983         .put_event_constraints  = intel_put_event_constraints,
3984         .event_constraints      = intel_core_event_constraints,
3985         .guest_get_msrs         = core_guest_get_msrs,
3986         .format_attrs           = intel_arch_formats_attr,
3987         .events_sysfs_show      = intel_event_sysfs_show,
3988
3989         /*
3990          * Virtual (or funny metal) CPU can define x86_pmu.extra_regs
3991          * together with PMU version 1 and thus be using core_pmu with
3992          * shared_regs. We need following callbacks here to allocate
3993          * it properly.
3994          */
3995         .cpu_prepare            = intel_pmu_cpu_prepare,
3996         .cpu_starting           = intel_pmu_cpu_starting,
3997         .cpu_dying              = intel_pmu_cpu_dying,
3998         .cpu_dead               = intel_pmu_cpu_dead,
3999
4000         .check_period           = intel_pmu_check_period,
4001
4002         .lbr_reset              = intel_pmu_lbr_reset_64,
4003         .lbr_read               = intel_pmu_lbr_read_64,
4004         .lbr_save               = intel_pmu_lbr_save,
4005         .lbr_restore            = intel_pmu_lbr_restore,
4006 };
4007
4008 static __initconst const struct x86_pmu intel_pmu = {
4009         .name                   = "Intel",
4010         .handle_irq             = intel_pmu_handle_irq,
4011         .disable_all            = intel_pmu_disable_all,
4012         .enable_all             = intel_pmu_enable_all,
4013         .enable                 = intel_pmu_enable_event,
4014         .disable                = intel_pmu_disable_event,
4015         .add                    = intel_pmu_add_event,
4016         .del                    = intel_pmu_del_event,
4017         .read                   = intel_pmu_read_event,
4018         .hw_config              = intel_pmu_hw_config,
4019         .schedule_events        = x86_schedule_events,
4020         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
4021         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
4022         .event_map              = intel_pmu_event_map,
4023         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
4024         .apic                   = 1,
4025         .large_pebs_flags       = LARGE_PEBS_FLAGS,
4026         /*
4027          * Intel PMCs cannot be accessed sanely above 32 bit width,
4028          * so we install an artificial 1<<31 period regardless of
4029          * the generic event period:
4030          */
4031         .max_period             = (1ULL << 31) - 1,
4032         .get_event_constraints  = intel_get_event_constraints,
4033         .put_event_constraints  = intel_put_event_constraints,
4034         .pebs_aliases           = intel_pebs_aliases_core2,
4035
4036         .format_attrs           = intel_arch3_formats_attr,
4037         .events_sysfs_show      = intel_event_sysfs_show,
4038
4039         .cpu_prepare            = intel_pmu_cpu_prepare,
4040         .cpu_starting           = intel_pmu_cpu_starting,
4041         .cpu_dying              = intel_pmu_cpu_dying,
4042         .cpu_dead               = intel_pmu_cpu_dead,
4043
4044         .guest_get_msrs         = intel_guest_get_msrs,
4045         .sched_task             = intel_pmu_sched_task,
4046         .swap_task_ctx          = intel_pmu_swap_task_ctx,
4047
4048         .check_period           = intel_pmu_check_period,
4049
4050         .aux_output_match       = intel_pmu_aux_output_match,
4051
4052         .lbr_reset              = intel_pmu_lbr_reset_64,
4053         .lbr_read               = intel_pmu_lbr_read_64,
4054         .lbr_save               = intel_pmu_lbr_save,
4055         .lbr_restore            = intel_pmu_lbr_restore,
4056 };
4057
4058 static __init void intel_clovertown_quirk(void)
4059 {
4060         /*
4061          * PEBS is unreliable due to:
4062          *
4063          *   AJ67  - PEBS may experience CPL leaks
4064          *   AJ68  - PEBS PMI may be delayed by one event
4065          *   AJ69  - GLOBAL_STATUS[62] will only be set when DEBUGCTL[12]
4066          *   AJ106 - FREEZE_LBRS_ON_PMI doesn't work in combination with PEBS
4067          *
4068          * AJ67 could be worked around by restricting the OS/USR flags.
4069          * AJ69 could be worked around by setting PMU_FREEZE_ON_PMI.
4070          *
4071          * AJ106 could possibly be worked around by not allowing LBR
4072          *       usage from PEBS, including the fixup.
4073          * AJ68  could possibly be worked around by always programming
4074          *       a pebs_event_reset[0] value and coping with the lost events.
4075          *
4076          * But taken together it might just make sense to not enable PEBS on
4077          * these chips.
4078          */
4079         pr_warn("PEBS disabled due to CPU errata\n");
4080         x86_pmu.pebs = 0;
4081         x86_pmu.pebs_constraints = NULL;
4082 }
4083
4084 static const struct x86_cpu_desc isolation_ucodes[] = {
4085         INTEL_CPU_DESC(INTEL_FAM6_HASWELL,               3, 0x0000001f),
4086         INTEL_CPU_DESC(INTEL_FAM6_HASWELL_L,             1, 0x0000001e),
4087         INTEL_CPU_DESC(INTEL_FAM6_HASWELL_G,             1, 0x00000015),
4088         INTEL_CPU_DESC(INTEL_FAM6_HASWELL_X,             2, 0x00000037),
4089         INTEL_CPU_DESC(INTEL_FAM6_HASWELL_X,             4, 0x0000000a),
4090         INTEL_CPU_DESC(INTEL_FAM6_BROADWELL,             4, 0x00000023),
4091         INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_G,           1, 0x00000014),
4092         INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_D,           2, 0x00000010),
4093         INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_D,           3, 0x07000009),
4094         INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_D,           4, 0x0f000009),
4095         INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_D,           5, 0x0e000002),
4096         INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_X,           2, 0x0b000014),
4097         INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X,             3, 0x00000021),
4098         INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X,             4, 0x00000000),
4099         INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_L,             3, 0x0000007c),
4100         INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE,               3, 0x0000007c),
4101         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE,              9, 0x0000004e),
4102         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_L,            9, 0x0000004e),
4103         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_L,           10, 0x0000004e),
4104         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_L,           11, 0x0000004e),
4105         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_L,           12, 0x0000004e),
4106         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE,             10, 0x0000004e),
4107         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE,             11, 0x0000004e),
4108         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE,             12, 0x0000004e),
4109         INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE,             13, 0x0000004e),
4110         {}
4111 };
4112
4113 static void intel_check_pebs_isolation(void)
4114 {
4115         x86_pmu.pebs_no_isolation = !x86_cpu_has_min_microcode_rev(isolation_ucodes);
4116 }
4117
4118 static __init void intel_pebs_isolation_quirk(void)
4119 {
4120         WARN_ON_ONCE(x86_pmu.check_microcode);
4121         x86_pmu.check_microcode = intel_check_pebs_isolation;
4122         intel_check_pebs_isolation();
4123 }
4124
4125 static const struct x86_cpu_desc pebs_ucodes[] = {
4126         INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE,          7, 0x00000028),
4127         INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE_X,        6, 0x00000618),
4128         INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE_X,        7, 0x0000070c),
4129         {}
4130 };
4131
4132 static bool intel_snb_pebs_broken(void)
4133 {
4134         return !x86_cpu_has_min_microcode_rev(pebs_ucodes);
4135 }
4136
4137 static void intel_snb_check_microcode(void)
4138 {
4139         if (intel_snb_pebs_broken() == x86_pmu.pebs_broken)
4140                 return;
4141
4142         /*
4143          * Serialized by the microcode lock..
4144          */
4145         if (x86_pmu.pebs_broken) {
4146                 pr_info("PEBS enabled due to microcode update\n");
4147                 x86_pmu.pebs_broken = 0;
4148         } else {
4149                 pr_info("PEBS disabled due to CPU errata, please upgrade microcode\n");
4150                 x86_pmu.pebs_broken = 1;
4151         }
4152 }
4153
4154 static bool is_lbr_from(unsigned long msr)
4155 {
4156         unsigned long lbr_from_nr = x86_pmu.lbr_from + x86_pmu.lbr_nr;
4157
4158         return x86_pmu.lbr_from <= msr && msr < lbr_from_nr;
4159 }
4160
4161 /*
4162  * Under certain circumstances, access certain MSR may cause #GP.
4163  * The function tests if the input MSR can be safely accessed.
4164  */
4165 static bool check_msr(unsigned long msr, u64 mask)
4166 {
4167         u64 val_old, val_new, val_tmp;
4168
4169         /*
4170          * Disable the check for real HW, so we don't
4171          * mess with potentionaly enabled registers:
4172          */
4173         if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
4174                 return true;
4175
4176         /*
4177          * Read the current value, change it and read it back to see if it
4178          * matches, this is needed to detect certain hardware emulators
4179          * (qemu/kvm) that don't trap on the MSR access and always return 0s.
4180          */
4181         if (rdmsrl_safe(msr, &val_old))
4182                 return false;
4183
4184         /*
4185          * Only change the bits which can be updated by wrmsrl.
4186          */
4187         val_tmp = val_old ^ mask;
4188
4189         if (is_lbr_from(msr))
4190                 val_tmp = lbr_from_signext_quirk_wr(val_tmp);
4191
4192         if (wrmsrl_safe(msr, val_tmp) ||
4193             rdmsrl_safe(msr, &val_new))
4194                 return false;
4195
4196         /*
4197          * Quirk only affects validation in wrmsr(), so wrmsrl()'s value
4198          * should equal rdmsrl()'s even with the quirk.
4199          */
4200         if (val_new != val_tmp)
4201                 return false;
4202
4203         if (is_lbr_from(msr))
4204                 val_old = lbr_from_signext_quirk_wr(val_old);
4205
4206         /* Here it's sure that the MSR can be safely accessed.
4207          * Restore the old value and return.
4208          */
4209         wrmsrl(msr, val_old);
4210
4211         return true;
4212 }
4213
4214 static __init void intel_sandybridge_quirk(void)
4215 {
4216         x86_pmu.check_microcode = intel_snb_check_microcode;
4217         cpus_read_lock();
4218         intel_snb_check_microcode();
4219         cpus_read_unlock();
4220 }
4221
4222 static const struct { int id; char *name; } intel_arch_events_map[] __initconst = {
4223         { PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" },
4224         { PERF_COUNT_HW_INSTRUCTIONS, "instructions" },
4225         { PERF_COUNT_HW_BUS_CYCLES, "bus cycles" },
4226         { PERF_COUNT_HW_CACHE_REFERENCES, "cache references" },
4227         { PERF_COUNT_HW_CACHE_MISSES, "cache misses" },
4228         { PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "branch instructions" },
4229         { PERF_COUNT_HW_BRANCH_MISSES, "branch misses" },
4230 };
4231
4232 static __init void intel_arch_events_quirk(void)
4233 {
4234         int bit;
4235
4236         /* disable event that reported as not presend by cpuid */
4237         for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(intel_arch_events_map)) {
4238                 intel_perfmon_event_map[intel_arch_events_map[bit].id] = 0;
4239                 pr_warn("CPUID marked event: \'%s\' unavailable\n",
4240                         intel_arch_events_map[bit].name);
4241         }
4242 }
4243
4244 static __init void intel_nehalem_quirk(void)
4245 {
4246         union cpuid10_ebx ebx;
4247
4248         ebx.full = x86_pmu.events_maskl;
4249         if (ebx.split.no_branch_misses_retired) {
4250                 /*
4251                  * Erratum AAJ80 detected, we work it around by using
4252                  * the BR_MISP_EXEC.ANY event. This will over-count
4253                  * branch-misses, but it's still much better than the
4254                  * architectural event which is often completely bogus:
4255                  */
4256                 intel_perfmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x7f89;
4257                 ebx.split.no_branch_misses_retired = 0;
4258                 x86_pmu.events_maskl = ebx.full;
4259                 pr_info("CPU erratum AAJ80 worked around\n");
4260         }
4261 }
4262
4263 static const struct x86_cpu_desc counter_freezing_ucodes[] = {
4264         INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT,         2, 0x0000000e),
4265         INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT,         9, 0x0000002e),
4266         INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT,        10, 0x00000008),
4267         INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_D,       1, 0x00000028),
4268         INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_PLUS,    1, 0x00000028),
4269         INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_PLUS,    8, 0x00000006),
4270         {}
4271 };
4272
4273 static bool intel_counter_freezing_broken(void)
4274 {
4275         return !x86_cpu_has_min_microcode_rev(counter_freezing_ucodes);
4276 }
4277
4278 static __init void intel_counter_freezing_quirk(void)
4279 {
4280         /* Check if it's already disabled */
4281         if (disable_counter_freezing)
4282                 return;
4283
4284         /*
4285          * If the system starts with the wrong ucode, leave the
4286          * counter-freezing feature permanently disabled.
4287          */
4288         if (intel_counter_freezing_broken()) {
4289                 pr_info("PMU counter freezing disabled due to CPU errata,"
4290                         "please upgrade microcode\n");
4291                 x86_pmu.counter_freezing = false;
4292                 x86_pmu.handle_irq = intel_pmu_handle_irq;
4293         }
4294 }
4295
4296 /*
4297  * enable software workaround for errata:
4298  * SNB: BJ122
4299  * IVB: BV98
4300  * HSW: HSD29
4301  *
4302  * Only needed when HT is enabled. However detecting
4303  * if HT is enabled is difficult (model specific). So instead,
4304  * we enable the workaround in the early boot, and verify if
4305  * it is needed in a later initcall phase once we have valid
4306  * topology information to check if HT is actually enabled
4307  */
4308 static __init void intel_ht_bug(void)
4309 {
4310         x86_pmu.flags |= PMU_FL_EXCL_CNTRS | PMU_FL_EXCL_ENABLED;
4311
4312         x86_pmu.start_scheduling = intel_start_scheduling;
4313         x86_pmu.commit_scheduling = intel_commit_scheduling;
4314         x86_pmu.stop_scheduling = intel_stop_scheduling;
4315 }
4316
4317 EVENT_ATTR_STR(mem-loads,       mem_ld_hsw,     "event=0xcd,umask=0x1,ldlat=3");
4318 EVENT_ATTR_STR(mem-stores,      mem_st_hsw,     "event=0xd0,umask=0x82")
4319
4320 /* Haswell special events */
4321 EVENT_ATTR_STR(tx-start,        tx_start,       "event=0xc9,umask=0x1");
4322 EVENT_ATTR_STR(tx-commit,       tx_commit,      "event=0xc9,umask=0x2");
4323 EVENT_ATTR_STR(tx-abort,        tx_abort,       "event=0xc9,umask=0x4");
4324 EVENT_ATTR_STR(tx-capacity,     tx_capacity,    "event=0x54,umask=0x2");
4325 EVENT_ATTR_STR(tx-conflict,     tx_conflict,    "event=0x54,umask=0x1");
4326 EVENT_ATTR_STR(el-start,        el_start,       "event=0xc8,umask=0x1");
4327 EVENT_ATTR_STR(el-commit,       el_commit,      "event=0xc8,umask=0x2");
4328 EVENT_ATTR_STR(el-abort,        el_abort,       "event=0xc8,umask=0x4");
4329 EVENT_ATTR_STR(el-capacity,     el_capacity,    "event=0x54,umask=0x2");
4330 EVENT_ATTR_STR(el-conflict,     el_conflict,    "event=0x54,umask=0x1");
4331 EVENT_ATTR_STR(cycles-t,        cycles_t,       "event=0x3c,in_tx=1");
4332 EVENT_ATTR_STR(cycles-ct,       cycles_ct,      "event=0x3c,in_tx=1,in_tx_cp=1");
4333
4334 static struct attribute *hsw_events_attrs[] = {
4335         EVENT_PTR(td_slots_issued),
4336         EVENT_PTR(td_slots_retired),
4337         EVENT_PTR(td_fetch_bubbles),
4338         EVENT_PTR(td_total_slots),
4339         EVENT_PTR(td_total_slots_scale),
4340         EVENT_PTR(td_recovery_bubbles),
4341         EVENT_PTR(td_recovery_bubbles_scale),
4342         NULL
4343 };
4344
4345 static struct attribute *hsw_mem_events_attrs[] = {
4346         EVENT_PTR(mem_ld_hsw),
4347         EVENT_PTR(mem_st_hsw),
4348         NULL,
4349 };
4350
4351 static struct attribute *hsw_tsx_events_attrs[] = {
4352         EVENT_PTR(tx_start),
4353         EVENT_PTR(tx_commit),
4354         EVENT_PTR(tx_abort),
4355         EVENT_PTR(tx_capacity),
4356         EVENT_PTR(tx_conflict),
4357         EVENT_PTR(el_start),
4358         EVENT_PTR(el_commit),
4359         EVENT_PTR(el_abort),
4360         EVENT_PTR(el_capacity),
4361         EVENT_PTR(el_conflict),
4362         EVENT_PTR(cycles_t),
4363         EVENT_PTR(cycles_ct),
4364         NULL
4365 };
4366
4367 EVENT_ATTR_STR(tx-capacity-read,  tx_capacity_read,  "event=0x54,umask=0x80");
4368 EVENT_ATTR_STR(tx-capacity-write, tx_capacity_write, "event=0x54,umask=0x2");
4369 EVENT_ATTR_STR(el-capacity-read,  el_capacity_read,  "event=0x54,umask=0x80");
4370 EVENT_ATTR_STR(el-capacity-write, el_capacity_write, "event=0x54,umask=0x2");
4371
4372 static struct attribute *icl_events_attrs[] = {
4373         EVENT_PTR(mem_ld_hsw),
4374         EVENT_PTR(mem_st_hsw),
4375         NULL,
4376 };
4377
4378 static struct attribute *icl_tsx_events_attrs[] = {
4379         EVENT_PTR(tx_start),
4380         EVENT_PTR(tx_abort),
4381         EVENT_PTR(tx_commit),
4382         EVENT_PTR(tx_capacity_read),
4383         EVENT_PTR(tx_capacity_write),
4384         EVENT_PTR(tx_conflict),
4385         EVENT_PTR(el_start),
4386         EVENT_PTR(el_abort),
4387         EVENT_PTR(el_commit),
4388         EVENT_PTR(el_capacity_read),
4389         EVENT_PTR(el_capacity_write),
4390         EVENT_PTR(el_conflict),
4391         EVENT_PTR(cycles_t),
4392         EVENT_PTR(cycles_ct),
4393         NULL,
4394 };
4395
4396 static ssize_t freeze_on_smi_show(struct device *cdev,
4397                                   struct device_attribute *attr,
4398                                   char *buf)
4399 {
4400         return sprintf(buf, "%lu\n", x86_pmu.attr_freeze_on_smi);
4401 }
4402
4403 static DEFINE_MUTEX(freeze_on_smi_mutex);
4404
4405 static ssize_t freeze_on_smi_store(struct device *cdev,
4406                                    struct device_attribute *attr,
4407                                    const char *buf, size_t count)
4408 {
4409         unsigned long val;
4410         ssize_t ret;
4411
4412         ret = kstrtoul(buf, 0, &val);
4413         if (ret)
4414                 return ret;
4415
4416         if (val > 1)
4417                 return -EINVAL;
4418
4419         mutex_lock(&freeze_on_smi_mutex);
4420
4421         if (x86_pmu.attr_freeze_on_smi == val)
4422                 goto done;
4423
4424         x86_pmu.attr_freeze_on_smi = val;
4425
4426         get_online_cpus();
4427         on_each_cpu(flip_smm_bit, &val, 1);
4428         put_online_cpus();
4429 done:
4430         mutex_unlock(&freeze_on_smi_mutex);
4431
4432         return count;
4433 }
4434
4435 static void update_tfa_sched(void *ignored)
4436 {
4437         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
4438
4439         /*
4440          * check if PMC3 is used
4441          * and if so force schedule out for all event types all contexts
4442          */
4443         if (test_bit(3, cpuc->active_mask))
4444                 perf_pmu_resched(x86_get_pmu());
4445 }
4446
4447 static ssize_t show_sysctl_tfa(struct device *cdev,
4448                               struct device_attribute *attr,
4449                               char *buf)
4450 {
4451         return snprintf(buf, 40, "%d\n", allow_tsx_force_abort);
4452 }
4453
4454 static ssize_t set_sysctl_tfa(struct device *cdev,
4455                               struct device_attribute *attr,
4456                               const char *buf, size_t count)
4457 {
4458         bool val;
4459         ssize_t ret;
4460
4461         ret = kstrtobool(buf, &val);
4462         if (ret)
4463                 return ret;
4464
4465         /* no change */
4466         if (val == allow_tsx_force_abort)
4467                 return count;
4468
4469         allow_tsx_force_abort = val;
4470
4471         get_online_cpus();
4472         on_each_cpu(update_tfa_sched, NULL, 1);
4473         put_online_cpus();
4474
4475         return count;
4476 }
4477
4478
4479 static DEVICE_ATTR_RW(freeze_on_smi);
4480
4481 static ssize_t branches_show(struct device *cdev,
4482                              struct device_attribute *attr,
4483                              char *buf)
4484 {
4485         return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu.lbr_nr);
4486 }
4487
4488 static DEVICE_ATTR_RO(branches);
4489
4490 static struct attribute *lbr_attrs[] = {
4491         &dev_attr_branches.attr,
4492         NULL
4493 };
4494
4495 static char pmu_name_str[30];
4496
4497 static ssize_t pmu_name_show(struct device *cdev,
4498                              struct device_attribute *attr,
4499                              char *buf)
4500 {
4501         return snprintf(buf, PAGE_SIZE, "%s\n", pmu_name_str);
4502 }
4503
4504 static DEVICE_ATTR_RO(pmu_name);
4505
4506 static struct attribute *intel_pmu_caps_attrs[] = {
4507        &dev_attr_pmu_name.attr,
4508        NULL
4509 };
4510
4511 static DEVICE_ATTR(allow_tsx_force_abort, 0644,
4512                    show_sysctl_tfa,
4513                    set_sysctl_tfa);
4514
4515 static struct attribute *intel_pmu_attrs[] = {
4516         &dev_attr_freeze_on_smi.attr,
4517         &dev_attr_allow_tsx_force_abort.attr,
4518         NULL,
4519 };
4520
4521 static umode_t
4522 tsx_is_visible(struct kobject *kobj, struct attribute *attr, int i)
4523 {
4524         return boot_cpu_has(X86_FEATURE_RTM) ? attr->mode : 0;
4525 }
4526
4527 static umode_t
4528 pebs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
4529 {
4530         return x86_pmu.pebs ? attr->mode : 0;
4531 }
4532
4533 static umode_t
4534 lbr_is_visible(struct kobject *kobj, struct attribute *attr, int i)
4535 {
4536         return x86_pmu.lbr_nr ? attr->mode : 0;
4537 }
4538
4539 static umode_t
4540 exra_is_visible(struct kobject *kobj, struct attribute *attr, int i)
4541 {
4542         return x86_pmu.version >= 2 ? attr->mode : 0;
4543 }
4544
4545 static umode_t
4546 default_is_visible(struct kobject *kobj, struct attribute *attr, int i)
4547 {
4548         if (attr == &dev_attr_allow_tsx_force_abort.attr)
4549                 return x86_pmu.flags & PMU_FL_TFA ? attr->mode : 0;
4550
4551         return attr->mode;
4552 }
4553
4554 static struct attribute_group group_events_td  = {
4555         .name = "events",
4556 };
4557
4558 static struct attribute_group group_events_mem = {
4559         .name       = "events",
4560         .is_visible = pebs_is_visible,
4561 };
4562
4563 static struct attribute_group group_events_tsx = {
4564         .name       = "events",
4565         .is_visible = tsx_is_visible,
4566 };
4567
4568 static struct attribute_group group_caps_gen = {
4569         .name  = "caps",
4570         .attrs = intel_pmu_caps_attrs,
4571 };
4572
4573 static struct attribute_group group_caps_lbr = {
4574         .name       = "caps",
4575         .attrs      = lbr_attrs,
4576         .is_visible = lbr_is_visible,
4577 };
4578
4579 static struct attribute_group group_format_extra = {
4580         .name       = "format",
4581         .is_visible = exra_is_visible,
4582 };
4583
4584 static struct attribute_group group_format_extra_skl = {
4585         .name       = "format",
4586         .is_visible = exra_is_visible,
4587 };
4588
4589 static struct attribute_group group_default = {
4590         .attrs      = intel_pmu_attrs,
4591         .is_visible = default_is_visible,
4592 };
4593
4594 static const struct attribute_group *attr_update[] = {
4595         &group_events_td,
4596         &group_events_mem,
4597         &group_events_tsx,
4598         &group_caps_gen,
4599         &group_caps_lbr,
4600         &group_format_extra,
4601         &group_format_extra_skl,
4602         &group_default,
4603         NULL,
4604 };
4605
4606 static struct attribute *empty_attrs;
4607
4608 __init int intel_pmu_init(void)
4609 {
4610         struct attribute **extra_skl_attr = &empty_attrs;
4611         struct attribute **extra_attr = &empty_attrs;
4612         struct attribute **td_attr    = &empty_attrs;
4613         struct attribute **mem_attr   = &empty_attrs;
4614         struct attribute **tsx_attr   = &empty_attrs;
4615         union cpuid10_edx edx;
4616         union cpuid10_eax eax;
4617         union cpuid10_ebx ebx;
4618         struct event_constraint *c;
4619         unsigned int unused;
4620         struct extra_reg *er;
4621         bool pmem = false;
4622         int version, i;
4623         char *name;
4624
4625         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
4626                 switch (boot_cpu_data.x86) {
4627                 case 0x6:
4628                         return p6_pmu_init();
4629                 case 0xb:
4630                         return knc_pmu_init();
4631                 case 0xf:
4632                         return p4_pmu_init();
4633                 }
4634                 return -ENODEV;
4635         }
4636
4637         /*
4638          * Check whether the Architectural PerfMon supports
4639          * Branch Misses Retired hw_event or not.
4640          */
4641         cpuid(10, &eax.full, &ebx.full, &unused, &edx.full);
4642         if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT)
4643                 return -ENODEV;
4644
4645         version = eax.split.version_id;
4646         if (version < 2)
4647                 x86_pmu = core_pmu;
4648         else
4649                 x86_pmu = intel_pmu;
4650
4651         x86_pmu.version                 = version;
4652         x86_pmu.num_counters            = eax.split.num_counters;
4653         x86_pmu.cntval_bits             = eax.split.bit_width;
4654         x86_pmu.cntval_mask             = (1ULL << eax.split.bit_width) - 1;
4655
4656         x86_pmu.events_maskl            = ebx.full;
4657         x86_pmu.events_mask_len         = eax.split.mask_length;
4658
4659         x86_pmu.max_pebs_events         = min_t(unsigned, MAX_PEBS_EVENTS, x86_pmu.num_counters);
4660
4661         /*
4662          * Quirk: v2 perfmon does not report fixed-purpose events, so
4663          * assume at least 3 events, when not running in a hypervisor:
4664          */
4665         if (version > 1) {
4666                 int assume = 3 * !boot_cpu_has(X86_FEATURE_HYPERVISOR);
4667
4668                 x86_pmu.num_counters_fixed =
4669                         max((int)edx.split.num_counters_fixed, assume);
4670         }
4671
4672         if (version >= 4)
4673                 x86_pmu.counter_freezing = !disable_counter_freezing;
4674
4675         if (boot_cpu_has(X86_FEATURE_PDCM)) {
4676                 u64 capabilities;
4677
4678                 rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
4679                 x86_pmu.intel_cap.capabilities = capabilities;
4680         }
4681
4682         if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) {
4683                 x86_pmu.lbr_reset = intel_pmu_lbr_reset_32;
4684                 x86_pmu.lbr_read = intel_pmu_lbr_read_32;
4685         }
4686
4687         if (boot_cpu_has(X86_FEATURE_ARCH_LBR))
4688                 intel_pmu_arch_lbr_init();
4689
4690         intel_ds_init();
4691
4692         x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
4693
4694         /*
4695          * Install the hw-cache-events table:
4696          */
4697         switch (boot_cpu_data.x86_model) {
4698         case INTEL_FAM6_CORE_YONAH:
4699                 pr_cont("Core events, ");
4700                 name = "core";
4701                 break;
4702
4703         case INTEL_FAM6_CORE2_MEROM:
4704                 x86_add_quirk(intel_clovertown_quirk);
4705                 /* fall through */
4706
4707         case INTEL_FAM6_CORE2_MEROM_L:
4708         case INTEL_FAM6_CORE2_PENRYN:
4709         case INTEL_FAM6_CORE2_DUNNINGTON:
4710                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
4711                        sizeof(hw_cache_event_ids));
4712
4713                 intel_pmu_lbr_init_core();
4714
4715                 x86_pmu.event_constraints = intel_core2_event_constraints;
4716                 x86_pmu.pebs_constraints = intel_core2_pebs_event_constraints;
4717                 pr_cont("Core2 events, ");
4718                 name = "core2";
4719                 break;
4720
4721         case INTEL_FAM6_NEHALEM:
4722         case INTEL_FAM6_NEHALEM_EP:
4723         case INTEL_FAM6_NEHALEM_EX:
4724                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
4725                        sizeof(hw_cache_event_ids));
4726                 memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
4727                        sizeof(hw_cache_extra_regs));
4728
4729                 intel_pmu_lbr_init_nhm();
4730
4731                 x86_pmu.event_constraints = intel_nehalem_event_constraints;
4732                 x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints;
4733                 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
4734                 x86_pmu.extra_regs = intel_nehalem_extra_regs;
4735                 x86_pmu.limit_period = nhm_limit_period;
4736
4737                 mem_attr = nhm_mem_events_attrs;
4738
4739                 /* UOPS_ISSUED.STALLED_CYCLES */
4740                 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
4741                         X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
4742                 /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
4743                 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
4744                         X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
4745
4746                 intel_pmu_pebs_data_source_nhm();
4747                 x86_add_quirk(intel_nehalem_quirk);
4748                 x86_pmu.pebs_no_tlb = 1;
4749                 extra_attr = nhm_format_attr;
4750
4751                 pr_cont("Nehalem events, ");
4752                 name = "nehalem";
4753                 break;
4754
4755         case INTEL_FAM6_ATOM_BONNELL:
4756         case INTEL_FAM6_ATOM_BONNELL_MID:
4757         case INTEL_FAM6_ATOM_SALTWELL:
4758         case INTEL_FAM6_ATOM_SALTWELL_MID:
4759         case INTEL_FAM6_ATOM_SALTWELL_TABLET:
4760                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
4761                        sizeof(hw_cache_event_ids));
4762
4763                 intel_pmu_lbr_init_atom();
4764
4765                 x86_pmu.event_constraints = intel_gen_event_constraints;
4766                 x86_pmu.pebs_constraints = intel_atom_pebs_event_constraints;
4767                 x86_pmu.pebs_aliases = intel_pebs_aliases_core2;
4768                 pr_cont("Atom events, ");
4769                 name = "bonnell";
4770                 break;
4771
4772         case INTEL_FAM6_ATOM_SILVERMONT:
4773         case INTEL_FAM6_ATOM_SILVERMONT_D:
4774         case INTEL_FAM6_ATOM_SILVERMONT_MID:
4775         case INTEL_FAM6_ATOM_AIRMONT:
4776         case INTEL_FAM6_ATOM_AIRMONT_MID:
4777                 memcpy(hw_cache_event_ids, slm_hw_cache_event_ids,
4778                         sizeof(hw_cache_event_ids));
4779                 memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs,
4780                        sizeof(hw_cache_extra_regs));
4781
4782                 intel_pmu_lbr_init_slm();
4783
4784                 x86_pmu.event_constraints = intel_slm_event_constraints;
4785                 x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints;
4786                 x86_pmu.extra_regs = intel_slm_extra_regs;
4787                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4788                 td_attr = slm_events_attrs;
4789                 extra_attr = slm_format_attr;
4790                 pr_cont("Silvermont events, ");
4791                 name = "silvermont";
4792                 break;
4793
4794         case INTEL_FAM6_ATOM_GOLDMONT:
4795         case INTEL_FAM6_ATOM_GOLDMONT_D:
4796                 x86_add_quirk(intel_counter_freezing_quirk);
4797                 memcpy(hw_cache_event_ids, glm_hw_cache_event_ids,
4798                        sizeof(hw_cache_event_ids));
4799                 memcpy(hw_cache_extra_regs, glm_hw_cache_extra_regs,
4800                        sizeof(hw_cache_extra_regs));
4801
4802                 intel_pmu_lbr_init_skl();
4803
4804                 x86_pmu.event_constraints = intel_slm_event_constraints;
4805                 x86_pmu.pebs_constraints = intel_glm_pebs_event_constraints;
4806                 x86_pmu.extra_regs = intel_glm_extra_regs;
4807                 /*
4808                  * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS
4809                  * for precise cycles.
4810                  * :pp is identical to :ppp
4811                  */
4812                 x86_pmu.pebs_aliases = NULL;
4813                 x86_pmu.pebs_prec_dist = true;
4814                 x86_pmu.lbr_pt_coexist = true;
4815                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4816                 td_attr = glm_events_attrs;
4817                 extra_attr = slm_format_attr;
4818                 pr_cont("Goldmont events, ");
4819                 name = "goldmont";
4820                 break;
4821
4822         case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
4823                 x86_add_quirk(intel_counter_freezing_quirk);
4824                 memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
4825                        sizeof(hw_cache_event_ids));
4826                 memcpy(hw_cache_extra_regs, glp_hw_cache_extra_regs,
4827                        sizeof(hw_cache_extra_regs));
4828
4829                 intel_pmu_lbr_init_skl();
4830
4831                 x86_pmu.event_constraints = intel_slm_event_constraints;
4832                 x86_pmu.extra_regs = intel_glm_extra_regs;
4833                 /*
4834                  * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS
4835                  * for precise cycles.
4836                  */
4837                 x86_pmu.pebs_aliases = NULL;
4838                 x86_pmu.pebs_prec_dist = true;
4839                 x86_pmu.lbr_pt_coexist = true;
4840                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4841                 x86_pmu.flags |= PMU_FL_PEBS_ALL;
4842                 x86_pmu.get_event_constraints = glp_get_event_constraints;
4843                 td_attr = glm_events_attrs;
4844                 /* Goldmont Plus has 4-wide pipeline */
4845                 event_attr_td_total_slots_scale_glm.event_str = "4";
4846                 extra_attr = slm_format_attr;
4847                 pr_cont("Goldmont plus events, ");
4848                 name = "goldmont_plus";
4849                 break;
4850
4851         case INTEL_FAM6_ATOM_TREMONT_D:
4852         case INTEL_FAM6_ATOM_TREMONT:
4853                 x86_pmu.late_ack = true;
4854                 memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
4855                        sizeof(hw_cache_event_ids));
4856                 memcpy(hw_cache_extra_regs, tnt_hw_cache_extra_regs,
4857                        sizeof(hw_cache_extra_regs));
4858                 hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
4859
4860                 intel_pmu_lbr_init_skl();
4861
4862                 x86_pmu.event_constraints = intel_slm_event_constraints;
4863                 x86_pmu.extra_regs = intel_tnt_extra_regs;
4864                 /*
4865                  * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS
4866                  * for precise cycles.
4867                  */
4868                 x86_pmu.pebs_aliases = NULL;
4869                 x86_pmu.pebs_prec_dist = true;
4870                 x86_pmu.lbr_pt_coexist = true;
4871                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4872                 x86_pmu.get_event_constraints = tnt_get_event_constraints;
4873                 extra_attr = slm_format_attr;
4874                 pr_cont("Tremont events, ");
4875                 name = "Tremont";
4876                 break;
4877
4878         case INTEL_FAM6_WESTMERE:
4879         case INTEL_FAM6_WESTMERE_EP:
4880         case INTEL_FAM6_WESTMERE_EX:
4881                 memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
4882                        sizeof(hw_cache_event_ids));
4883                 memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
4884                        sizeof(hw_cache_extra_regs));
4885
4886                 intel_pmu_lbr_init_nhm();
4887
4888                 x86_pmu.event_constraints = intel_westmere_event_constraints;
4889                 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
4890                 x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
4891                 x86_pmu.extra_regs = intel_westmere_extra_regs;
4892                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4893
4894                 mem_attr = nhm_mem_events_attrs;
4895
4896                 /* UOPS_ISSUED.STALLED_CYCLES */
4897                 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
4898                         X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
4899                 /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
4900                 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
4901                         X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
4902
4903                 intel_pmu_pebs_data_source_nhm();
4904                 extra_attr = nhm_format_attr;
4905                 pr_cont("Westmere events, ");
4906                 name = "westmere";
4907                 break;
4908
4909         case INTEL_FAM6_SANDYBRIDGE:
4910         case INTEL_FAM6_SANDYBRIDGE_X:
4911                 x86_add_quirk(intel_sandybridge_quirk);
4912                 x86_add_quirk(intel_ht_bug);
4913                 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
4914                        sizeof(hw_cache_event_ids));
4915                 memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
4916                        sizeof(hw_cache_extra_regs));
4917
4918                 intel_pmu_lbr_init_snb();
4919
4920                 x86_pmu.event_constraints = intel_snb_event_constraints;
4921                 x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints;
4922                 x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
4923                 if (boot_cpu_data.x86_model == INTEL_FAM6_SANDYBRIDGE_X)
4924                         x86_pmu.extra_regs = intel_snbep_extra_regs;
4925                 else
4926                         x86_pmu.extra_regs = intel_snb_extra_regs;
4927
4928
4929                 /* all extra regs are per-cpu when HT is on */
4930                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4931                 x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
4932
4933                 td_attr  = snb_events_attrs;
4934                 mem_attr = snb_mem_events_attrs;
4935
4936                 /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
4937                 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
4938                         X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
4939                 /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/
4940                 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
4941                         X86_CONFIG(.event=0xb1, .umask=0x01, .inv=1, .cmask=1);
4942
4943                 extra_attr = nhm_format_attr;
4944
4945                 pr_cont("SandyBridge events, ");
4946                 name = "sandybridge";
4947                 break;
4948
4949         case INTEL_FAM6_IVYBRIDGE:
4950         case INTEL_FAM6_IVYBRIDGE_X:
4951                 x86_add_quirk(intel_ht_bug);
4952                 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
4953                        sizeof(hw_cache_event_ids));
4954                 /* dTLB-load-misses on IVB is different than SNB */
4955                 hw_cache_event_ids[C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = 0x8108; /* DTLB_LOAD_MISSES.DEMAND_LD_MISS_CAUSES_A_WALK */
4956
4957                 memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
4958                        sizeof(hw_cache_extra_regs));
4959
4960                 intel_pmu_lbr_init_snb();
4961
4962                 x86_pmu.event_constraints = intel_ivb_event_constraints;
4963                 x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints;
4964                 x86_pmu.pebs_aliases = intel_pebs_aliases_ivb;
4965                 x86_pmu.pebs_prec_dist = true;
4966                 if (boot_cpu_data.x86_model == INTEL_FAM6_IVYBRIDGE_X)
4967                         x86_pmu.extra_regs = intel_snbep_extra_regs;
4968                 else
4969                         x86_pmu.extra_regs = intel_snb_extra_regs;
4970                 /* all extra regs are per-cpu when HT is on */
4971                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4972                 x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
4973
4974                 td_attr  = snb_events_attrs;
4975                 mem_attr = snb_mem_events_attrs;
4976
4977                 /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
4978                 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
4979                         X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
4980
4981                 extra_attr = nhm_format_attr;
4982
4983                 pr_cont("IvyBridge events, ");
4984                 name = "ivybridge";
4985                 break;
4986
4987
4988         case INTEL_FAM6_HASWELL:
4989         case INTEL_FAM6_HASWELL_X:
4990         case INTEL_FAM6_HASWELL_L:
4991         case INTEL_FAM6_HASWELL_G:
4992                 x86_add_quirk(intel_ht_bug);
4993                 x86_add_quirk(intel_pebs_isolation_quirk);
4994                 x86_pmu.late_ack = true;
4995                 memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids));
4996                 memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
4997
4998                 intel_pmu_lbr_init_hsw();
4999
5000                 x86_pmu.event_constraints = intel_hsw_event_constraints;
5001                 x86_pmu.pebs_constraints = intel_hsw_pebs_event_constraints;
5002                 x86_pmu.extra_regs = intel_snbep_extra_regs;
5003                 x86_pmu.pebs_aliases = intel_pebs_aliases_ivb;
5004                 x86_pmu.pebs_prec_dist = true;
5005                 /* all extra regs are per-cpu when HT is on */
5006                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
5007                 x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
5008
5009                 x86_pmu.hw_config = hsw_hw_config;
5010                 x86_pmu.get_event_constraints = hsw_get_event_constraints;
5011                 x86_pmu.lbr_double_abort = true;
5012                 extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
5013                         hsw_format_attr : nhm_format_attr;
5014                 td_attr  = hsw_events_attrs;
5015                 mem_attr = hsw_mem_events_attrs;
5016                 tsx_attr = hsw_tsx_events_attrs;
5017                 pr_cont("Haswell events, ");
5018                 name = "haswell";
5019                 break;
5020
5021         case INTEL_FAM6_BROADWELL:
5022         case INTEL_FAM6_BROADWELL_D:
5023         case INTEL_FAM6_BROADWELL_G:
5024         case INTEL_FAM6_BROADWELL_X:
5025                 x86_add_quirk(intel_pebs_isolation_quirk);
5026                 x86_pmu.late_ack = true;
5027                 memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids));
5028                 memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
5029
5030                 /* L3_MISS_LOCAL_DRAM is BIT(26) in Broadwell */
5031                 hw_cache_extra_regs[C(LL)][C(OP_READ)][C(RESULT_MISS)] = HSW_DEMAND_READ |
5032                                                                          BDW_L3_MISS|HSW_SNOOP_DRAM;
5033                 hw_cache_extra_regs[C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = HSW_DEMAND_WRITE|BDW_L3_MISS|
5034                                                                           HSW_SNOOP_DRAM;
5035                 hw_cache_extra_regs[C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = HSW_DEMAND_READ|
5036                                                                              BDW_L3_MISS_LOCAL|HSW_SNOOP_DRAM;
5037                 hw_cache_extra_regs[C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = HSW_DEMAND_WRITE|
5038                                                                               BDW_L3_MISS_LOCAL|HSW_SNOOP_DRAM;
5039
5040                 intel_pmu_lbr_init_hsw();
5041
5042                 x86_pmu.event_constraints = intel_bdw_event_constraints;
5043                 x86_pmu.pebs_constraints = intel_bdw_pebs_event_constraints;
5044                 x86_pmu.extra_regs = intel_snbep_extra_regs;
5045                 x86_pmu.pebs_aliases = intel_pebs_aliases_ivb;
5046                 x86_pmu.pebs_prec_dist = true;
5047                 /* all extra regs are per-cpu when HT is on */
5048                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
5049                 x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
5050
5051                 x86_pmu.hw_config = hsw_hw_config;
5052                 x86_pmu.get_event_constraints = hsw_get_event_constraints;
5053                 x86_pmu.limit_period = bdw_limit_period;
5054                 extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
5055                         hsw_format_attr : nhm_format_attr;
5056                 td_attr  = hsw_events_attrs;
5057                 mem_attr = hsw_mem_events_attrs;
5058                 tsx_attr = hsw_tsx_events_attrs;
5059                 pr_cont("Broadwell events, ");
5060                 name = "broadwell";
5061                 break;
5062
5063         case INTEL_FAM6_XEON_PHI_KNL:
5064         case INTEL_FAM6_XEON_PHI_KNM:
5065                 memcpy(hw_cache_event_ids,
5066                        slm_hw_cache_event_ids, sizeof(hw_cache_event_ids));
5067                 memcpy(hw_cache_extra_regs,
5068                        knl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
5069                 intel_pmu_lbr_init_knl();
5070
5071                 x86_pmu.event_constraints = intel_slm_event_constraints;
5072                 x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints;
5073                 x86_pmu.extra_regs = intel_knl_extra_regs;
5074
5075                 /* all extra regs are per-cpu when HT is on */
5076                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
5077                 x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
5078                 extra_attr = slm_format_attr;
5079                 pr_cont("Knights Landing/Mill events, ");
5080                 name = "knights-landing";
5081                 break;
5082
5083         case INTEL_FAM6_SKYLAKE_X:
5084                 pmem = true;
5085                 /* fall through */
5086         case INTEL_FAM6_SKYLAKE_L:
5087         case INTEL_FAM6_SKYLAKE:
5088         case INTEL_FAM6_KABYLAKE_L:
5089         case INTEL_FAM6_KABYLAKE:
5090         case INTEL_FAM6_COMETLAKE_L:
5091         case INTEL_FAM6_COMETLAKE:
5092                 x86_add_quirk(intel_pebs_isolation_quirk);
5093                 x86_pmu.late_ack = true;
5094                 memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids));
5095                 memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
5096                 intel_pmu_lbr_init_skl();
5097
5098                 /* INT_MISC.RECOVERY_CYCLES has umask 1 in Skylake */
5099                 event_attr_td_recovery_bubbles.event_str_noht =
5100                         "event=0xd,umask=0x1,cmask=1";
5101                 event_attr_td_recovery_bubbles.event_str_ht =
5102                         "event=0xd,umask=0x1,cmask=1,any=1";
5103
5104                 x86_pmu.event_constraints = intel_skl_event_constraints;
5105                 x86_pmu.pebs_constraints = intel_skl_pebs_event_constraints;
5106                 x86_pmu.extra_regs = intel_skl_extra_regs;
5107                 x86_pmu.pebs_aliases = intel_pebs_aliases_skl;
5108                 x86_pmu.pebs_prec_dist = true;
5109                 /* all extra regs are per-cpu when HT is on */
5110                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
5111                 x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
5112
5113                 x86_pmu.hw_config = hsw_hw_config;
5114                 x86_pmu.get_event_constraints = hsw_get_event_constraints;
5115                 extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
5116                         hsw_format_attr : nhm_format_attr;
5117                 extra_skl_attr = skl_format_attr;
5118                 td_attr  = hsw_events_attrs;
5119                 mem_attr = hsw_mem_events_attrs;
5120                 tsx_attr = hsw_tsx_events_attrs;
5121                 intel_pmu_pebs_data_source_skl(pmem);
5122
5123                 if (boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
5124                         x86_pmu.flags |= PMU_FL_TFA;
5125                         x86_pmu.get_event_constraints = tfa_get_event_constraints;
5126                         x86_pmu.enable_all = intel_tfa_pmu_enable_all;
5127                         x86_pmu.commit_scheduling = intel_tfa_commit_scheduling;
5128                 }
5129
5130                 pr_cont("Skylake events, ");
5131                 name = "skylake";
5132                 break;
5133
5134         case INTEL_FAM6_ICELAKE_X:
5135         case INTEL_FAM6_ICELAKE_D:
5136                 pmem = true;
5137                 /* fall through */
5138         case INTEL_FAM6_ICELAKE_L:
5139         case INTEL_FAM6_ICELAKE:
5140         case INTEL_FAM6_TIGERLAKE_L:
5141         case INTEL_FAM6_TIGERLAKE:
5142                 x86_pmu.late_ack = true;
5143                 memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids));
5144                 memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
5145                 hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
5146                 intel_pmu_lbr_init_skl();
5147
5148                 x86_pmu.event_constraints = intel_icl_event_constraints;
5149                 x86_pmu.pebs_constraints = intel_icl_pebs_event_constraints;
5150                 x86_pmu.extra_regs = intel_icl_extra_regs;
5151                 x86_pmu.pebs_aliases = NULL;
5152                 x86_pmu.pebs_prec_dist = true;
5153                 x86_pmu.flags |= PMU_FL_HAS_RSP_1;
5154                 x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
5155
5156                 x86_pmu.hw_config = hsw_hw_config;
5157                 x86_pmu.get_event_constraints = icl_get_event_constraints;
5158                 extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
5159                         hsw_format_attr : nhm_format_attr;
5160                 extra_skl_attr = skl_format_attr;
5161                 mem_attr = icl_events_attrs;
5162                 tsx_attr = icl_tsx_events_attrs;
5163                 x86_pmu.rtm_abort_event = X86_CONFIG(.event=0xca, .umask=0x02);
5164                 x86_pmu.lbr_pt_coexist = true;
5165                 intel_pmu_pebs_data_source_skl(pmem);
5166                 pr_cont("Icelake events, ");
5167                 name = "icelake";
5168                 break;
5169
5170         default:
5171                 switch (x86_pmu.version) {
5172                 case 1:
5173                         x86_pmu.event_constraints = intel_v1_event_constraints;
5174                         pr_cont("generic architected perfmon v1, ");
5175                         name = "generic_arch_v1";
5176                         break;
5177                 default:
5178                         /*
5179                          * default constraints for v2 and up
5180                          */
5181                         x86_pmu.event_constraints = intel_gen_event_constraints;
5182                         pr_cont("generic architected perfmon, ");
5183                         name = "generic_arch_v2+";
5184                         break;
5185                 }
5186         }
5187
5188         snprintf(pmu_name_str, sizeof(pmu_name_str), "%s", name);
5189
5190
5191         group_events_td.attrs  = td_attr;
5192         group_events_mem.attrs = mem_attr;
5193         group_events_tsx.attrs = tsx_attr;
5194         group_format_extra.attrs = extra_attr;
5195         group_format_extra_skl.attrs = extra_skl_attr;
5196
5197         x86_pmu.attr_update = attr_update;
5198
5199         if (x86_pmu.num_counters > INTEL_PMC_MAX_GENERIC) {
5200                 WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
5201                      x86_pmu.num_counters, INTEL_PMC_MAX_GENERIC);
5202                 x86_pmu.num_counters = INTEL_PMC_MAX_GENERIC;
5203         }
5204         x86_pmu.intel_ctrl = (1ULL << x86_pmu.num_counters) - 1;
5205
5206         if (x86_pmu.num_counters_fixed > INTEL_PMC_MAX_FIXED) {
5207                 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
5208                      x86_pmu.num_counters_fixed, INTEL_PMC_MAX_FIXED);
5209                 x86_pmu.num_counters_fixed = INTEL_PMC_MAX_FIXED;
5210         }
5211
5212         x86_pmu.intel_ctrl |=
5213                 ((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED;
5214
5215         if (x86_pmu.event_constraints) {
5216                 /*
5217                  * event on fixed counter2 (REF_CYCLES) only works on this
5218                  * counter, so do not extend mask to generic counters
5219                  */
5220                 for_each_event_constraint(c, x86_pmu.event_constraints) {
5221                         if (c->cmask == FIXED_EVENT_FLAGS
5222                             && c->idxmsk64 != INTEL_PMC_MSK_FIXED_REF_CYCLES) {
5223                                 c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
5224                         }
5225                         c->idxmsk64 &=
5226                                 ~(~0ULL << (INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed));
5227                         c->weight = hweight64(c->idxmsk64);
5228                 }
5229         }
5230
5231         /*
5232          * Access LBR MSR may cause #GP under certain circumstances.
5233          * E.g. KVM doesn't support LBR MSR
5234          * Check all LBT MSR here.
5235          * Disable LBR access if any LBR MSRs can not be accessed.
5236          */
5237         if (x86_pmu.lbr_nr && !check_msr(x86_pmu.lbr_tos, 0x3UL))
5238                 x86_pmu.lbr_nr = 0;
5239         for (i = 0; i < x86_pmu.lbr_nr; i++) {
5240                 if (!(check_msr(x86_pmu.lbr_from + i, 0xffffUL) &&
5241                       check_msr(x86_pmu.lbr_to + i, 0xffffUL)))
5242                         x86_pmu.lbr_nr = 0;
5243         }
5244
5245         if (x86_pmu.lbr_nr)
5246                 pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr);
5247
5248         /*
5249          * Access extra MSR may cause #GP under certain circumstances.
5250          * E.g. KVM doesn't support offcore event
5251          * Check all extra_regs here.
5252          */
5253         if (x86_pmu.extra_regs) {
5254                 for (er = x86_pmu.extra_regs; er->msr; er++) {
5255                         er->extra_msr_access = check_msr(er->msr, 0x11UL);
5256                         /* Disable LBR select mapping */
5257                         if ((er->idx == EXTRA_REG_LBR) && !er->extra_msr_access)
5258                                 x86_pmu.lbr_sel_map = NULL;
5259                 }
5260         }
5261
5262         /* Support full width counters using alternative MSR range */
5263         if (x86_pmu.intel_cap.full_width_write) {
5264                 x86_pmu.max_period = x86_pmu.cntval_mask >> 1;
5265                 x86_pmu.perfctr = MSR_IA32_PMC0;
5266                 pr_cont("full-width counters, ");
5267         }
5268
5269         /*
5270          * For arch perfmon 4 use counter freezing to avoid
5271          * several MSR accesses in the PMI.
5272          */
5273         if (x86_pmu.counter_freezing)
5274                 x86_pmu.handle_irq = intel_pmu_handle_irq_v4;
5275
5276         return 0;
5277 }
5278
5279 /*
5280  * HT bug: phase 2 init
5281  * Called once we have valid topology information to check
5282  * whether or not HT is enabled
5283  * If HT is off, then we disable the workaround
5284  */
5285 static __init int fixup_ht_bug(void)
5286 {
5287         int c;
5288         /*
5289          * problem not present on this CPU model, nothing to do
5290          */
5291         if (!(x86_pmu.flags & PMU_FL_EXCL_ENABLED))
5292                 return 0;
5293
5294         if (topology_max_smt_threads() > 1) {
5295                 pr_info("PMU erratum BJ122, BV98, HSD29 worked around, HT is on\n");
5296                 return 0;
5297         }
5298
5299         cpus_read_lock();
5300
5301         hardlockup_detector_perf_stop();
5302
5303         x86_pmu.flags &= ~(PMU_FL_EXCL_CNTRS | PMU_FL_EXCL_ENABLED);
5304
5305         x86_pmu.start_scheduling = NULL;
5306         x86_pmu.commit_scheduling = NULL;
5307         x86_pmu.stop_scheduling = NULL;
5308
5309         hardlockup_detector_perf_restart();
5310
5311         for_each_online_cpu(c)
5312                 free_excl_cntrs(&per_cpu(cpu_hw_events, c));
5313
5314         cpus_read_unlock();
5315         pr_info("PMU erratum BJ122, BV98, HSD29 workaround disabled, HT off\n");
5316         return 0;
5317 }
5318 subsys_initcall(fixup_ht_bug)