tools/perf/util/evsel.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
   4  *
   5  * Parts came from builtin-{top,stat,record}.c, see those files for further
   6  * copyright notes.
   7  */
   8
   9 #include <byteswap.h>
  10 #include <errno.h>
  11 #include <inttypes.h>
  12 #include <linux/bitops.h>
  13 #include <api/fs/fs.h>
  14 #include <api/fs/tracing_path.h>
  15 #include <traceevent/event-parse.h>
  16 #include <linux/hw_breakpoint.h>
  17 #include <linux/perf_event.h>
  18 #include <linux/compiler.h>
  19 #include <linux/err.h>
  20 #include <linux/zalloc.h>
  21 #include <sys/ioctl.h>
  22 #include <sys/resource.h>
  23 #include <sys/types.h>
  24 #include <dirent.h>
  25 #include <stdlib.h>
  26 #include <perf/evsel.h>
  27 #include "asm/bug.h"
  28 #include "bpf_counter.h"
  29 #include "callchain.h"
  30 #include "cgroup.h"
  31 #include "counts.h"
  32 #include "event.h"
  33 #include "evsel.h"
  34 #include "util/env.h"
  35 #include "util/evsel_config.h"
  36 #include "util/evsel_fprintf.h"
  37 #include "evlist.h"
  38 #include <perf/cpumap.h>
  39 #include "thread_map.h"
  40 #include "target.h"
  41 #include "perf_regs.h"
  42 #include "record.h"
  43 #include "debug.h"
  44 #include "trace-event.h"
  45 #include "stat.h"
  46 #include "string2.h"
  47 #include "memswap.h"
  48 #include "util.h"
  49 #include "hashmap.h"
  50 #include "../perf-sys.h"
  51 #include "util/parse-branch-options.h"
  52 #include <internal/xyarray.h>
  53 #include <internal/lib.h>
  54
  55 #include <linux/ctype.h>
  56
  57 struct perf_missing_features perf_missing_features;
  58
  59 static clockid_t clockid;
  60
  61 static int evsel__no_extra_init(struct evsel *evsel __maybe_unused)
  62 {
  63         return 0;
  64 }
  65
  66 void __weak test_attr__ready(void) { }
  67
  68 static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused)
  69 {
  70 }
  71
  72 static struct {
  73         size_t  size;
  74         int     (*init)(struct evsel *evsel);
  75         void    (*fini)(struct evsel *evsel);
  76 } perf_evsel__object = {
  77         .size = sizeof(struct evsel),
  78         .init = evsel__no_extra_init,
  79         .fini = evsel__no_extra_fini,
  80 };
  81
  82 int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel),
  83                          void (*fini)(struct evsel *evsel))
  84 {
  85
  86         if (object_size == 0)
  87                 goto set_methods;
  88
  89         if (perf_evsel__object.size > object_size)
  90                 return -EINVAL;
  91
  92         perf_evsel__object.size = object_size;
  93
  94 set_methods:
  95         if (init != NULL)
  96                 perf_evsel__object.init = init;
  97
  98         if (fini != NULL)
  99                 perf_evsel__object.fini = fini;
 100
 101         return 0;
 102 }
 103
 104 #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y))
 105
 106 int __evsel__sample_size(u64 sample_type)
 107 {
 108         u64 mask = sample_type & PERF_SAMPLE_MASK;
 109         int size = 0;
 110         int i;
 111
 112         for (i = 0; i < 64; i++) {
 113                 if (mask & (1ULL << i))
 114                         size++;
 115         }
 116
 117         size *= sizeof(u64);
 118
 119         return size;
 120 }
 121
 122 /**
 123  * __perf_evsel__calc_id_pos - calculate id_pos.
 124  * @sample_type: sample type
 125  *
 126  * This function returns the position of the event id (PERF_SAMPLE_ID or
 127  * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct
 128  * perf_record_sample.
 129  */
 130 static int __perf_evsel__calc_id_pos(u64 sample_type)
 131 {
 132         int idx = 0;
 133
 134         if (sample_type & PERF_SAMPLE_IDENTIFIER)
 135                 return 0;
 136
 137         if (!(sample_type & PERF_SAMPLE_ID))
 138                 return -1;
 139
 140         if (sample_type & PERF_SAMPLE_IP)
 141                 idx += 1;
 142
 143         if (sample_type & PERF_SAMPLE_TID)
 144                 idx += 1;
 145
 146         if (sample_type & PERF_SAMPLE_TIME)
 147                 idx += 1;
 148
 149         if (sample_type & PERF_SAMPLE_ADDR)
 150                 idx += 1;
 151
 152         return idx;
 153 }
 154
 155 /**
 156  * __perf_evsel__calc_is_pos - calculate is_pos.
 157  * @sample_type: sample type
 158  *
 159  * This function returns the position (counting backwards) of the event id
 160  * (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if
 161  * sample_id_all is used there is an id sample appended to non-sample events.
 162  */
 163 static int __perf_evsel__calc_is_pos(u64 sample_type)
 164 {
 165         int idx = 1;
 166
 167         if (sample_type & PERF_SAMPLE_IDENTIFIER)
 168                 return 1;
 169
 170         if (!(sample_type & PERF_SAMPLE_ID))
 171                 return -1;
 172
 173         if (sample_type & PERF_SAMPLE_CPU)
 174                 idx += 1;
 175
 176         if (sample_type & PERF_SAMPLE_STREAM_ID)
 177                 idx += 1;
 178
 179         return idx;
 180 }
 181
 182 void evsel__calc_id_pos(struct evsel *evsel)
 183 {
 184         evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type);
 185         evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type);
 186 }
 187
 188 void __evsel__set_sample_bit(struct evsel *evsel,
 189                                   enum perf_event_sample_format bit)
 190 {
 191         if (!(evsel->core.attr.sample_type & bit)) {
 192                 evsel->core.attr.sample_type |= bit;
 193                 evsel->sample_size += sizeof(u64);
 194                 evsel__calc_id_pos(evsel);
 195         }
 196 }
 197
 198 void __evsel__reset_sample_bit(struct evsel *evsel,
 199                                     enum perf_event_sample_format bit)
 200 {
 201         if (evsel->core.attr.sample_type & bit) {
 202                 evsel->core.attr.sample_type &= ~bit;
 203                 evsel->sample_size -= sizeof(u64);
 204                 evsel__calc_id_pos(evsel);
 205         }
 206 }
 207
 208 void evsel__set_sample_id(struct evsel *evsel,
 209                                bool can_sample_identifier)
 210 {
 211         if (can_sample_identifier) {
 212                 evsel__reset_sample_bit(evsel, ID);
 213                 evsel__set_sample_bit(evsel, IDENTIFIER);
 214         } else {
 215                 evsel__set_sample_bit(evsel, ID);
 216         }
 217         evsel->core.attr.read_format |= PERF_FORMAT_ID;
 218 }
 219
 220 /**
 221  * evsel__is_function_event - Return whether given evsel is a function
 222  * trace event
 223  *
 224  * @evsel - evsel selector to be tested
 225  *
 226  * Return %true if event is function trace event
 227  */
 228 bool evsel__is_function_event(struct evsel *evsel)
 229 {
 230 #define FUNCTION_EVENT "ftrace:function"
 231
 232         return evsel->name &&
 233                !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT));
 234
 235 #undef FUNCTION_EVENT
 236 }
 237
 238 void evsel__init(struct evsel *evsel,
 239                  struct perf_event_attr *attr, int idx)
 240 {
 241         perf_evsel__init(&evsel->core, attr);
 242         evsel->idx         = idx;
 243         evsel->tracking    = !idx;
 244         evsel->leader      = evsel;
 245         evsel->unit        = "";
 246         evsel->scale       = 1.0;
 247         evsel->max_events  = ULONG_MAX;
 248         evsel->evlist      = NULL;
 249         evsel->bpf_obj     = NULL;
 250         evsel->bpf_fd      = -1;
 251         INIT_LIST_HEAD(&evsel->config_terms);
 252         INIT_LIST_HEAD(&evsel->bpf_counter_list);
 253         perf_evsel__object.init(evsel);
 254         evsel->sample_size = __evsel__sample_size(attr->sample_type);
 255         evsel__calc_id_pos(evsel);
 256         evsel->cmdline_group_boundary = false;
 257         evsel->metric_expr   = NULL;
 258         evsel->metric_name   = NULL;
 259         evsel->metric_events = NULL;
 260         evsel->per_pkg_mask  = NULL;
 261         evsel->collect_stat  = false;
 262         evsel->pmu_name      = NULL;
 263 }
 264
 265 struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx)
 266 {
 267         struct evsel *evsel = zalloc(perf_evsel__object.size);
 268
 269         if (!evsel)
 270                 return NULL;
 271         evsel__init(evsel, attr, idx);
 272
 273         if (evsel__is_bpf_output(evsel)) {
 274                 evsel->core.attr.sample_type |= (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
 275                                             PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
 276                 evsel->core.attr.sample_period = 1;
 277         }
 278
 279         if (evsel__is_clock(evsel)) {
 280                 /*
 281                  * The evsel->unit points to static alias->unit
 282                  * so it's ok to use static string in here.
 283                  */
 284                 static const char *unit = "msec";
 285
 286                 evsel->unit = unit;
 287                 evsel->scale = 1e-6;
 288         }
 289
 290         return evsel;
 291 }
 292
 293 static bool perf_event_can_profile_kernel(void)
 294 {
 295         return perf_event_paranoid_check(1);
 296 }
 297
 298 struct evsel *evsel__new_cycles(bool precise)
 299 {
 300         struct perf_event_attr attr = {
 301                 .type   = PERF_TYPE_HARDWARE,
 302                 .config = PERF_COUNT_HW_CPU_CYCLES,
 303                 .exclude_kernel = !perf_event_can_profile_kernel(),
 304         };
 305         struct evsel *evsel;
 306
 307         event_attr_init(&attr);
 308
 309         if (!precise)
 310                 goto new_event;
 311
 312         /*
 313          * Now let the usual logic to set up the perf_event_attr defaults
 314          * to kick in when we return and before perf_evsel__open() is called.
 315          */
 316 new_event:
 317         evsel = evsel__new(&attr);
 318         if (evsel == NULL)
 319                 goto out;
 320
 321         evsel->precise_max = true;
 322
 323         /* use asprintf() because free(evsel) assumes name is allocated */
 324         if (asprintf(&evsel->name, "cycles%s%s%.*s",
 325                      (attr.precise_ip || attr.exclude_kernel) ? ":" : "",
 326                      attr.exclude_kernel ? "u" : "",
 327                      attr.precise_ip ? attr.precise_ip + 1 : 0, "ppp") < 0)
 328                 goto error_free;
 329 out:
 330         return evsel;
 331 error_free:
 332         evsel__delete(evsel);
 333         evsel = NULL;
 334         goto out;
 335 }
 336
 337 static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src)
 338 {
 339         struct evsel_config_term *pos, *tmp;
 340
 341         list_for_each_entry(pos, &src->config_terms, list) {
 342                 tmp = malloc(sizeof(*tmp));
 343                 if (tmp == NULL)
 344                         return -ENOMEM;
 345
 346                 *tmp = *pos;
 347                 if (tmp->free_str) {
 348                         tmp->val.str = strdup(pos->val.str);
 349                         if (tmp->val.str == NULL) {
 350                                 free(tmp);
 351                                 return -ENOMEM;
 352                         }
 353                 }
 354                 list_add_tail(&tmp->list, &dst->config_terms);
 355         }
 356         return 0;
 357 }
 358
 359 /**
 360  * evsel__clone - create a new evsel copied from @orig
 361  * @orig: original evsel
 362  *
 363  * The assumption is that @orig is not configured nor opened yet.
 364  * So we only care about the attributes that can be set while it's parsed.
 365  */
 366 struct evsel *evsel__clone(struct evsel *orig)
 367 {
 368         struct evsel *evsel;
 369
 370         BUG_ON(orig->core.fd);
 371         BUG_ON(orig->counts);
 372         BUG_ON(orig->priv);
 373         BUG_ON(orig->per_pkg_mask);
 374
 375         /* cannot handle BPF objects for now */
 376         if (orig->bpf_obj)
 377                 return NULL;
 378
 379         evsel = evsel__new(&orig->core.attr);
 380         if (evsel == NULL)
 381                 return NULL;
 382
 383         evsel->core.cpus = perf_cpu_map__get(orig->core.cpus);
 384         evsel->core.own_cpus = perf_cpu_map__get(orig->core.own_cpus);
 385         evsel->core.threads = perf_thread_map__get(orig->core.threads);
 386         evsel->core.nr_members = orig->core.nr_members;
 387         evsel->core.system_wide = orig->core.system_wide;
 388
 389         if (orig->name) {
 390                 evsel->name = strdup(orig->name);
 391                 if (evsel->name == NULL)
 392                         goto out_err;
 393         }
 394         if (orig->group_name) {
 395                 evsel->group_name = strdup(orig->group_name);
 396                 if (evsel->group_name == NULL)
 397                         goto out_err;
 398         }
 399         if (orig->pmu_name) {
 400                 evsel->pmu_name = strdup(orig->pmu_name);
 401                 if (evsel->pmu_name == NULL)
 402                         goto out_err;
 403         }
 404         if (orig->filter) {
 405                 evsel->filter = strdup(orig->filter);
 406                 if (evsel->filter == NULL)
 407                         goto out_err;
 408         }
 409         evsel->cgrp = cgroup__get(orig->cgrp);
 410         evsel->tp_format = orig->tp_format;
 411         evsel->handler = orig->handler;
 412         evsel->leader = orig->leader;
 413
 414         evsel->max_events = orig->max_events;
 415         evsel->tool_event = orig->tool_event;
 416         evsel->unit = orig->unit;
 417         evsel->scale = orig->scale;
 418         evsel->snapshot = orig->snapshot;
 419         evsel->per_pkg = orig->per_pkg;
 420         evsel->percore = orig->percore;
 421         evsel->precise_max = orig->precise_max;
 422         evsel->use_uncore_alias = orig->use_uncore_alias;
 423         evsel->is_libpfm_event = orig->is_libpfm_event;
 424
 425         evsel->exclude_GH = orig->exclude_GH;
 426         evsel->sample_read = orig->sample_read;
 427         evsel->auto_merge_stats = orig->auto_merge_stats;
 428         evsel->collect_stat = orig->collect_stat;
 429         evsel->weak_group = orig->weak_group;
 430
 431         if (evsel__copy_config_terms(evsel, orig) < 0)
 432                 goto out_err;
 433
 434         return evsel;
 435
 436 out_err:
 437         evsel__delete(evsel);
 438         return NULL;
 439 }
 440
 441 /*
 442  * Returns pointer with encoded error via <linux/err.h> interface.
 443  */
 444 struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx)
 445 {
 446         struct evsel *evsel = zalloc(perf_evsel__object.size);
 447         int err = -ENOMEM;
 448
 449         if (evsel == NULL) {
 450                 goto out_err;
 451         } else {
 452                 struct perf_event_attr attr = {
 453                         .type          = PERF_TYPE_TRACEPOINT,
 454                         .sample_type   = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME |
 455                                           PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD),
 456                 };
 457
 458                 if (asprintf(&evsel->name, "%s:%s", sys, name) < 0)
 459                         goto out_free;
 460
 461                 evsel->tp_format = trace_event__tp_format(sys, name);
 462                 if (IS_ERR(evsel->tp_format)) {
 463                         err = PTR_ERR(evsel->tp_format);
 464                         goto out_free;
 465                 }
 466
 467                 event_attr_init(&attr);
 468                 attr.config = evsel->tp_format->id;
 469                 attr.sample_period = 1;
 470                 evsel__init(evsel, &attr, idx);
 471         }
 472
 473         return evsel;
 474
 475 out_free:
 476         zfree(&evsel->name);
 477         free(evsel);
 478 out_err:
 479         return ERR_PTR(err);
 480 }
 481
 482 const char *evsel__hw_names[PERF_COUNT_HW_MAX] = {
 483         "cycles",
 484         "instructions",
 485         "cache-references",
 486         "cache-misses",
 487         "branches",
 488         "branch-misses",
 489         "bus-cycles",
 490         "stalled-cycles-frontend",
 491         "stalled-cycles-backend",
 492         "ref-cycles",
 493 };
 494
 495 static const char *__evsel__hw_name(u64 config)
 496 {
 497         if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config])
 498                 return evsel__hw_names[config];
 499
 500         return "unknown-hardware";
 501 }
 502
 503 static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size)
 504 {
 505         int colon = 0, r = 0;
 506         struct perf_event_attr *attr = &evsel->core.attr;
 507         bool exclude_guest_default = false;
 508
 509 #define MOD_PRINT(context, mod) do {                                    \
 510                 if (!attr->exclude_##context) {                         \
 511                         if (!colon) colon = ++r;                        \
 512                         r += scnprintf(bf + r, size - r, "%c", mod);    \
 513                 } } while(0)
 514
 515         if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) {
 516                 MOD_PRINT(kernel, 'k');
 517                 MOD_PRINT(user, 'u');
 518                 MOD_PRINT(hv, 'h');
 519                 exclude_guest_default = true;
 520         }
 521
 522         if (attr->precise_ip) {
 523                 if (!colon)
 524                         colon = ++r;
 525                 r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp");
 526                 exclude_guest_default = true;
 527         }
 528
 529         if (attr->exclude_host || attr->exclude_guest == exclude_guest_default) {
 530                 MOD_PRINT(host, 'H');
 531                 MOD_PRINT(guest, 'G');
 532         }
 533 #undef MOD_PRINT
 534         if (colon)
 535                 bf[colon - 1] = ':';
 536         return r;
 537 }
 538
 539 static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size)
 540 {
 541         int r = scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config));
 542         return r + evsel__add_modifiers(evsel, bf + r, size - r);
 543 }
 544
 545 const char *evsel__sw_names[PERF_COUNT_SW_MAX] = {
 546         "cpu-clock",
 547         "task-clock",
 548         "page-faults",
 549         "context-switches",
 550         "cpu-migrations",
 551         "minor-faults",
 552         "major-faults",
 553         "alignment-faults",
 554         "emulation-faults",
 555         "dummy",
 556 };
 557
 558 static const char *__evsel__sw_name(u64 config)
 559 {
 560         if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config])
 561                 return evsel__sw_names[config];
 562         return "unknown-software";
 563 }
 564
 565 static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size)
 566 {
 567         int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config));
 568         return r + evsel__add_modifiers(evsel, bf + r, size - r);
 569 }
 570
 571 static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type)
 572 {
 573         int r;
 574
 575         r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr);
 576
 577         if (type & HW_BREAKPOINT_R)
 578                 r += scnprintf(bf + r, size - r, "r");
 579
 580         if (type & HW_BREAKPOINT_W)
 581                 r += scnprintf(bf + r, size - r, "w");
 582
 583         if (type & HW_BREAKPOINT_X)
 584                 r += scnprintf(bf + r, size - r, "x");
 585
 586         return r;
 587 }
 588
 589 static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size)
 590 {
 591         struct perf_event_attr *attr = &evsel->core.attr;
 592         int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type);
 593         return r + evsel__add_modifiers(evsel, bf + r, size - r);
 594 }
 595
 596 const char *evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = {
 597  { "L1-dcache", "l1-d",         "l1d",          "L1-data",              },
 598  { "L1-icache", "l1-i",         "l1i",          "L1-instruction",       },
 599  { "LLC",       "L2",                                                   },
 600  { "dTLB",      "d-tlb",        "Data-TLB",                             },
 601  { "iTLB",      "i-tlb",        "Instruction-TLB",                      },
 602  { "branch",    "branches",     "bpu",          "btb",          "bpc",  },
 603  { "node",                                                              },
 604 };
 605
 606 const char *evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = {
 607  { "load",      "loads",        "read",                                 },
 608  { "store",     "stores",       "write",                                },
 609  { "prefetch",  "prefetches",   "speculative-read", "speculative-load", },
 610 };
 611
 612 const char *evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = {
 613  { "refs",      "Reference",    "ops",          "access",               },
 614  { "misses",    "miss",                                                 },
 615 };
 616
 617 #define C(x)            PERF_COUNT_HW_CACHE_##x
 618 #define CACHE_READ      (1 << C(OP_READ))
 619 #define CACHE_WRITE     (1 << C(OP_WRITE))
 620 #define CACHE_PREFETCH  (1 << C(OP_PREFETCH))
 621 #define COP(x)          (1 << x)
 622
 623 /*
 624  * cache operation stat
 625  * L1I : Read and prefetch only
 626  * ITLB and BPU : Read-only
 627  */
 628 static unsigned long evsel__hw_cache_stat[C(MAX)] = {
 629  [C(L1D)]       = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 630  [C(L1I)]       = (CACHE_READ | CACHE_PREFETCH),
 631  [C(LL)]        = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 632  [C(DTLB)]      = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 633  [C(ITLB)]      = (CACHE_READ),
 634  [C(BPU)]       = (CACHE_READ),
 635  [C(NODE)]      = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH),
 636 };
 637
 638 bool evsel__is_cache_op_valid(u8 type, u8 op)
 639 {
 640         if (evsel__hw_cache_stat[type] & COP(op))
 641                 return true;    /* valid */
 642         else
 643                 return false;   /* invalid */
 644 }
 645
 646 int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size)
 647 {
 648         if (result) {
 649                 return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0],
 650                                  evsel__hw_cache_op[op][0],
 651                                  evsel__hw_cache_result[result][0]);
 652         }
 653
 654         return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0],
 655                          evsel__hw_cache_op[op][1]);
 656 }
 657
 658 static int __evsel__hw_cache_name(u64 config, char *bf, size_t size)
 659 {
 660         u8 op, result, type = (config >>  0) & 0xff;
 661         const char *err = "unknown-ext-hardware-cache-type";
 662
 663         if (type >= PERF_COUNT_HW_CACHE_MAX)
 664                 goto out_err;
 665
 666         op = (config >>  8) & 0xff;
 667         err = "unknown-ext-hardware-cache-op";
 668         if (op >= PERF_COUNT_HW_CACHE_OP_MAX)
 669                 goto out_err;
 670
 671         result = (config >> 16) & 0xff;
 672         err = "unknown-ext-hardware-cache-result";
 673         if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 674                 goto out_err;
 675
 676         err = "invalid-cache";
 677         if (!evsel__is_cache_op_valid(type, op))
 678                 goto out_err;
 679
 680         return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size);
 681 out_err:
 682         return scnprintf(bf, size, "%s", err);
 683 }
 684
 685 static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size)
 686 {
 687         int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size);
 688         return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
 689 }
 690
 691 static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size)
 692 {
 693         int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config);
 694         return ret + evsel__add_modifiers(evsel, bf + ret, size - ret);
 695 }
 696
 697 static int evsel__tool_name(char *bf, size_t size)
 698 {
 699         int ret = scnprintf(bf, size, "duration_time");
 700         return ret;
 701 }
 702
 703 const char *evsel__name(struct evsel *evsel)
 704 {
 705         char bf[128];
 706
 707         if (!evsel)
 708                 goto out_unknown;
 709
 710         if (evsel->name)
 711                 return evsel->name;
 712
 713         switch (evsel->core.attr.type) {
 714         case PERF_TYPE_RAW:
 715                 evsel__raw_name(evsel, bf, sizeof(bf));
 716                 break;
 717
 718         case PERF_TYPE_HARDWARE:
 719                 evsel__hw_name(evsel, bf, sizeof(bf));
 720                 break;
 721
 722         case PERF_TYPE_HW_CACHE:
 723                 evsel__hw_cache_name(evsel, bf, sizeof(bf));
 724                 break;
 725
 726         case PERF_TYPE_SOFTWARE:
 727                 if (evsel->tool_event)
 728                         evsel__tool_name(bf, sizeof(bf));
 729                 else
 730                         evsel__sw_name(evsel, bf, sizeof(bf));
 731                 break;
 732
 733         case PERF_TYPE_TRACEPOINT:
 734                 scnprintf(bf, sizeof(bf), "%s", "unknown tracepoint");
 735                 break;
 736
 737         case PERF_TYPE_BREAKPOINT:
 738                 evsel__bp_name(evsel, bf, sizeof(bf));
 739                 break;
 740
 741         default:
 742                 scnprintf(bf, sizeof(bf), "unknown attr type: %d",
 743                           evsel->core.attr.type);
 744                 break;
 745         }
 746
 747         evsel->name = strdup(bf);
 748
 749         if (evsel->name)
 750                 return evsel->name;
 751 out_unknown:
 752         return "unknown";
 753 }
 754
 755 const char *evsel__group_name(struct evsel *evsel)
 756 {
 757         return evsel->group_name ?: "anon group";
 758 }
 759
 760 /*
 761  * Returns the group details for the specified leader,
 762  * with following rules.
 763  *
 764  *  For record -e '{cycles,instructions}'
 765  *    'anon group { cycles:u, instructions:u }'
 766  *
 767  *  For record -e 'cycles,instructions' and report --group
 768  *    'cycles:u, instructions:u'
 769  */
 770 int evsel__group_desc(struct evsel *evsel, char *buf, size_t size)
 771 {
 772         int ret = 0;
 773         struct evsel *pos;
 774         const char *group_name = evsel__group_name(evsel);
 775
 776         if (!evsel->forced_leader)
 777                 ret = scnprintf(buf, size, "%s { ", group_name);
 778
 779         ret += scnprintf(buf + ret, size - ret, "%s", evsel__name(evsel));
 780
 781         for_each_group_member(pos, evsel)
 782                 ret += scnprintf(buf + ret, size - ret, ", %s", evsel__name(pos));
 783
 784         if (!evsel->forced_leader)
 785                 ret += scnprintf(buf + ret, size - ret, " }");
 786
 787         return ret;
 788 }
 789
 790 static void __evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
 791                                       struct callchain_param *param)
 792 {
 793         bool function = evsel__is_function_event(evsel);
 794         struct perf_event_attr *attr = &evsel->core.attr;
 795
 796         evsel__set_sample_bit(evsel, CALLCHAIN);
 797
 798         attr->sample_max_stack = param->max_stack;
 799
 800         if (opts->kernel_callchains)
 801                 attr->exclude_callchain_user = 1;
 802         if (opts->user_callchains)
 803                 attr->exclude_callchain_kernel = 1;
 804         if (param->record_mode == CALLCHAIN_LBR) {
 805                 if (!opts->branch_stack) {
 806                         if (attr->exclude_user) {
 807                                 pr_warning("LBR callstack option is only available "
 808                                            "to get user callchain information. "
 809                                            "Falling back to framepointers.\n");
 810                         } else {
 811                                 evsel__set_sample_bit(evsel, BRANCH_STACK);
 812                                 attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER |
 813                                                         PERF_SAMPLE_BRANCH_CALL_STACK |
 814                                                         PERF_SAMPLE_BRANCH_NO_CYCLES |
 815                                                         PERF_SAMPLE_BRANCH_NO_FLAGS |
 816                                                         PERF_SAMPLE_BRANCH_HW_INDEX;
 817                         }
 818                 } else
 819                          pr_warning("Cannot use LBR callstack with branch stack. "
 820                                     "Falling back to framepointers.\n");
 821         }
 822
 823         if (param->record_mode == CALLCHAIN_DWARF) {
 824                 if (!function) {
 825                         evsel__set_sample_bit(evsel, REGS_USER);
 826                         evsel__set_sample_bit(evsel, STACK_USER);
 827                         if (opts->sample_user_regs && DWARF_MINIMAL_REGS != PERF_REGS_MASK) {
 828                                 attr->sample_regs_user |= DWARF_MINIMAL_REGS;
 829                                 pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, "
 830                                            "specifying a subset with --user-regs may render DWARF unwinding unreliable, "
 831                                            "so the minimal registers set (IP, SP) is explicitly forced.\n");
 832                         } else {
 833                                 attr->sample_regs_user |= PERF_REGS_MASK;
 834                         }
 835                         attr->sample_stack_user = param->dump_size;
 836                         attr->exclude_callchain_user = 1;
 837                 } else {
 838                         pr_info("Cannot use DWARF unwind for function trace event,"
 839                                 " falling back to framepointers.\n");
 840                 }
 841         }
 842
 843         if (function) {
 844                 pr_info("Disabling user space callchains for function trace event.\n");
 845                 attr->exclude_callchain_user = 1;
 846         }
 847 }
 848
 849 void evsel__config_callchain(struct evsel *evsel, struct record_opts *opts,
 850                              struct callchain_param *param)
 851 {
 852         if (param->enabled)
 853                 return __evsel__config_callchain(evsel, opts, param);
 854 }
 855
 856 static void evsel__reset_callgraph(struct evsel *evsel, struct callchain_param *param)
 857 {
 858         struct perf_event_attr *attr = &evsel->core.attr;
 859
 860         evsel__reset_sample_bit(evsel, CALLCHAIN);
 861         if (param->record_mode == CALLCHAIN_LBR) {
 862                 evsel__reset_sample_bit(evsel, BRANCH_STACK);
 863                 attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
 864                                               PERF_SAMPLE_BRANCH_CALL_STACK |
 865                                               PERF_SAMPLE_BRANCH_HW_INDEX);
 866         }
 867         if (param->record_mode == CALLCHAIN_DWARF) {
 868                 evsel__reset_sample_bit(evsel, REGS_USER);
 869                 evsel__reset_sample_bit(evsel, STACK_USER);
 870         }
 871 }
 872
 873 static void evsel__apply_config_terms(struct evsel *evsel,
 874                                       struct record_opts *opts, bool track)
 875 {
 876         struct evsel_config_term *term;
 877         struct list_head *config_terms = &evsel->config_terms;
 878         struct perf_event_attr *attr = &evsel->core.attr;
 879         /* callgraph default */
 880         struct callchain_param param = {
 881                 .record_mode = callchain_param.record_mode,
 882         };
 883         u32 dump_size = 0;
 884         int max_stack = 0;
 885         const char *callgraph_buf = NULL;
 886
 887         list_for_each_entry(term, config_terms, list) {
 888                 switch (term->type) {
 889                 case EVSEL__CONFIG_TERM_PERIOD:
 890                         if (!(term->weak && opts->user_interval != ULLONG_MAX)) {
 891                                 attr->sample_period = term->val.period;
 892                                 attr->freq = 0;
 893                                 evsel__reset_sample_bit(evsel, PERIOD);
 894                         }
 895                         break;
 896                 case EVSEL__CONFIG_TERM_FREQ:
 897                         if (!(term->weak && opts->user_freq != UINT_MAX)) {
 898                                 attr->sample_freq = term->val.freq;
 899                                 attr->freq = 1;
 900                                 evsel__set_sample_bit(evsel, PERIOD);
 901                         }
 902                         break;
 903                 case EVSEL__CONFIG_TERM_TIME:
 904                         if (term->val.time)
 905                                 evsel__set_sample_bit(evsel, TIME);
 906                         else
 907                                 evsel__reset_sample_bit(evsel, TIME);
 908                         break;
 909                 case EVSEL__CONFIG_TERM_CALLGRAPH:
 910                         callgraph_buf = term->val.str;
 911                         break;
 912                 case EVSEL__CONFIG_TERM_BRANCH:
 913                         if (term->val.str && strcmp(term->val.str, "no")) {
 914                                 evsel__set_sample_bit(evsel, BRANCH_STACK);
 915                                 parse_branch_str(term->val.str,
 916                                                  &attr->branch_sample_type);
 917                         } else
 918                                 evsel__reset_sample_bit(evsel, BRANCH_STACK);
 919                         break;
 920                 case EVSEL__CONFIG_TERM_STACK_USER:
 921                         dump_size = term->val.stack_user;
 922                         break;
 923                 case EVSEL__CONFIG_TERM_MAX_STACK:
 924                         max_stack = term->val.max_stack;
 925                         break;
 926                 case EVSEL__CONFIG_TERM_MAX_EVENTS:
 927                         evsel->max_events = term->val.max_events;
 928                         break;
 929                 case EVSEL__CONFIG_TERM_INHERIT:
 930                         /*
 931                          * attr->inherit should has already been set by
 932                          * evsel__config. If user explicitly set
 933                          * inherit using config terms, override global
 934                          * opt->no_inherit setting.
 935                          */
 936                         attr->inherit = term->val.inherit ? 1 : 0;
 937                         break;
 938                 case EVSEL__CONFIG_TERM_OVERWRITE:
 939                         attr->write_backward = term->val.overwrite ? 1 : 0;
 940                         break;
 941                 case EVSEL__CONFIG_TERM_DRV_CFG:
 942                         break;
 943                 case EVSEL__CONFIG_TERM_PERCORE:
 944                         break;
 945                 case EVSEL__CONFIG_TERM_AUX_OUTPUT:
 946                         attr->aux_output = term->val.aux_output ? 1 : 0;
 947                         break;
 948                 case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE:
 949                         /* Already applied by auxtrace */
 950                         break;
 951                 case EVSEL__CONFIG_TERM_CFG_CHG:
 952                         break;
 953                 default:
 954                         break;
 955                 }
 956         }
 957
 958         /* User explicitly set per-event callgraph, clear the old setting and reset. */
 959         if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) {
 960                 bool sample_address = false;
 961
 962                 if (max_stack) {
 963                         param.max_stack = max_stack;
 964                         if (callgraph_buf == NULL)
 965                                 callgraph_buf = "fp";
 966                 }
 967
 968                 /* parse callgraph parameters */
 969                 if (callgraph_buf != NULL) {
 970                         if (!strcmp(callgraph_buf, "no")) {
 971                                 param.enabled = false;
 972                                 param.record_mode = CALLCHAIN_NONE;
 973                         } else {
 974                                 param.enabled = true;
 975                                 if (parse_callchain_record(callgraph_buf, &param)) {
 976                                         pr_err("per-event callgraph setting for %s failed. "
 977                                                "Apply callgraph global setting for it\n",
 978                                                evsel->name);
 979                                         return;
 980                                 }
 981                                 if (param.record_mode == CALLCHAIN_DWARF)
 982                                         sample_address = true;
 983                         }
 984                 }
 985                 if (dump_size > 0) {
 986                         dump_size = round_up(dump_size, sizeof(u64));
 987                         param.dump_size = dump_size;
 988                 }
 989
 990                 /* If global callgraph set, clear it */
 991                 if (callchain_param.enabled)
 992                         evsel__reset_callgraph(evsel, &callchain_param);
 993
 994                 /* set perf-event callgraph */
 995                 if (param.enabled) {
 996                         if (sample_address) {
 997                                 evsel__set_sample_bit(evsel, ADDR);
 998                                 evsel__set_sample_bit(evsel, DATA_SRC);
 999                                 evsel->core.attr.mmap_data = track;
1000                         }
1001                         evsel__config_callchain(evsel, opts, &param);
1002                 }
1003         }
1004 }
1005
1006 struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type)
1007 {
1008         struct evsel_config_term *term, *found_term = NULL;
1009
1010         list_for_each_entry(term, &evsel->config_terms, list) {
1011                 if (term->type == type)
1012                         found_term = term;
1013         }
1014
1015         return found_term;
1016 }
1017
1018 void __weak arch_evsel__set_sample_weight(struct evsel *evsel)
1019 {
1020         evsel__set_sample_bit(evsel, WEIGHT);
1021 }
1022
1023 /*
1024  * The enable_on_exec/disabled value strategy:
1025  *
1026  *  1) For any type of traced program:
1027  *    - all independent events and group leaders are disabled
1028  *    - all group members are enabled
1029  *
1030  *     Group members are ruled by group leaders. They need to
1031  *     be enabled, because the group scheduling relies on that.
1032  *
1033  *  2) For traced programs executed by perf:
1034  *     - all independent events and group leaders have
1035  *       enable_on_exec set
1036  *     - we don't specifically enable or disable any event during
1037  *       the record command
1038  *
1039  *     Independent events and group leaders are initially disabled
1040  *     and get enabled by exec. Group members are ruled by group
1041  *     leaders as stated in 1).
1042  *
1043  *  3) For traced programs attached by perf (pid/tid):
1044  *     - we specifically enable or disable all events during
1045  *       the record command
1046  *
1047  *     When attaching events to already running traced we
1048  *     enable/disable events specifically, as there's no
1049  *     initial traced exec call.
1050  */
1051 void evsel__config(struct evsel *evsel, struct record_opts *opts,
1052                    struct callchain_param *callchain)
1053 {
1054         struct evsel *leader = evsel->leader;
1055         struct perf_event_attr *attr = &evsel->core.attr;
1056         int track = evsel->tracking;
1057         bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread;
1058
1059         attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1;
1060         attr->inherit       = !opts->no_inherit;
1061         attr->write_backward = opts->overwrite ? 1 : 0;
1062
1063         evsel__set_sample_bit(evsel, IP);
1064         evsel__set_sample_bit(evsel, TID);
1065
1066         if (evsel->sample_read) {
1067                 evsel__set_sample_bit(evsel, READ);
1068
1069                 /*
1070                  * We need ID even in case of single event, because
1071                  * PERF_SAMPLE_READ process ID specific data.
1072                  */
1073                 evsel__set_sample_id(evsel, false);
1074
1075                 /*
1076                  * Apply group format only if we belong to group
1077                  * with more than one members.
1078                  */
1079                 if (leader->core.nr_members > 1) {
1080                         attr->read_format |= PERF_FORMAT_GROUP;
1081                         attr->inherit = 0;
1082                 }
1083         }
1084
1085         /*
1086          * We default some events to have a default interval. But keep
1087          * it a weak assumption overridable by the user.
1088          */
1089         if (!attr->sample_period) {
1090                 if (opts->freq) {
1091                         attr->freq              = 1;
1092                         attr->sample_freq       = opts->freq;
1093                 } else {
1094                         attr->sample_period = opts->default_interval;
1095                 }
1096         }
1097         /*
1098          * If attr->freq was set (here or earlier), ask for period
1099          * to be sampled.
1100          */
1101         if (attr->freq)
1102                 evsel__set_sample_bit(evsel, PERIOD);
1103
1104         if (opts->no_samples)
1105                 attr->sample_freq = 0;
1106
1107         if (opts->inherit_stat) {
1108                 evsel->core.attr.read_format |=
1109                         PERF_FORMAT_TOTAL_TIME_ENABLED |
1110                         PERF_FORMAT_TOTAL_TIME_RUNNING |
1111                         PERF_FORMAT_ID;
1112                 attr->inherit_stat = 1;
1113         }
1114
1115         if (opts->sample_address) {
1116                 evsel__set_sample_bit(evsel, ADDR);
1117                 attr->mmap_data = track;
1118         }
1119
1120         /*
1121          * We don't allow user space callchains for  function trace
1122          * event, due to issues with page faults while tracing page
1123          * fault handler and its overall trickiness nature.
1124          */
1125         if (evsel__is_function_event(evsel))
1126                 evsel->core.attr.exclude_callchain_user = 1;
1127
1128         if (callchain && callchain->enabled && !evsel->no_aux_samples)
1129                 evsel__config_callchain(evsel, opts, callchain);
1130
1131         if (opts->sample_intr_regs && !evsel->no_aux_samples &&
1132             !evsel__is_dummy_event(evsel)) {
1133                 attr->sample_regs_intr = opts->sample_intr_regs;
1134                 evsel__set_sample_bit(evsel, REGS_INTR);
1135         }
1136
1137         if (opts->sample_user_regs && !evsel->no_aux_samples &&
1138             !evsel__is_dummy_event(evsel)) {
1139                 attr->sample_regs_user |= opts->sample_user_regs;
1140                 evsel__set_sample_bit(evsel, REGS_USER);
1141         }
1142
1143         if (target__has_cpu(&opts->target) || opts->sample_cpu)
1144                 evsel__set_sample_bit(evsel, CPU);
1145
1146         /*
1147          * When the user explicitly disabled time don't force it here.
1148          */
1149         if (opts->sample_time &&
1150             (!perf_missing_features.sample_id_all &&
1151             (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu ||
1152              opts->sample_time_set)))
1153                 evsel__set_sample_bit(evsel, TIME);
1154
1155         if (opts->raw_samples && !evsel->no_aux_samples) {
1156                 evsel__set_sample_bit(evsel, TIME);
1157                 evsel__set_sample_bit(evsel, RAW);
1158                 evsel__set_sample_bit(evsel, CPU);
1159         }
1160
1161         if (opts->sample_address)
1162                 evsel__set_sample_bit(evsel, DATA_SRC);
1163
1164         if (opts->sample_phys_addr)
1165                 evsel__set_sample_bit(evsel, PHYS_ADDR);
1166
1167         if (opts->no_buffering) {
1168                 attr->watermark = 0;
1169                 attr->wakeup_events = 1;
1170         }
1171         if (opts->branch_stack && !evsel->no_aux_samples) {
1172                 evsel__set_sample_bit(evsel, BRANCH_STACK);
1173                 attr->branch_sample_type = opts->branch_stack;
1174         }
1175
1176         if (opts->sample_weight)
1177                 arch_evsel__set_sample_weight(evsel);
1178
1179         attr->task     = track;
1180         attr->mmap     = track;
1181         attr->mmap2    = track && !perf_missing_features.mmap2;
1182         attr->comm     = track;
1183         attr->build_id = track && opts->build_id;
1184
1185         /*
1186          * ksymbol is tracked separately with text poke because it needs to be
1187          * system wide and enabled immediately.
1188          */
1189         if (!opts->text_poke)
1190                 attr->ksymbol = track && !perf_missing_features.ksymbol;
1191         attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf;
1192
1193         if (opts->record_namespaces)
1194                 attr->namespaces  = track;
1195
1196         if (opts->record_cgroup) {
1197                 attr->cgroup = track && !perf_missing_features.cgroup;
1198                 evsel__set_sample_bit(evsel, CGROUP);
1199         }
1200
1201         if (opts->sample_data_page_size)
1202                 evsel__set_sample_bit(evsel, DATA_PAGE_SIZE);
1203
1204         if (opts->sample_code_page_size)
1205                 evsel__set_sample_bit(evsel, CODE_PAGE_SIZE);
1206
1207         if (opts->record_switch_events)
1208                 attr->context_switch = track;
1209
1210         if (opts->sample_transaction)
1211                 evsel__set_sample_bit(evsel, TRANSACTION);
1212
1213         if (opts->running_time) {
1214                 evsel->core.attr.read_format |=
1215                         PERF_FORMAT_TOTAL_TIME_ENABLED |
1216                         PERF_FORMAT_TOTAL_TIME_RUNNING;
1217         }
1218
1219         /*
1220          * XXX see the function comment above
1221          *
1222          * Disabling only independent events or group leaders,
1223          * keeping group members enabled.
1224          */
1225         if (evsel__is_group_leader(evsel))
1226                 attr->disabled = 1;
1227
1228         /*
1229          * Setting enable_on_exec for independent events and
1230          * group leaders for traced executed by perf.
1231          */
1232         if (target__none(&opts->target) && evsel__is_group_leader(evsel) &&
1233             !opts->initial_delay)
1234                 attr->enable_on_exec = 1;
1235
1236         if (evsel->immediate) {
1237                 attr->disabled = 0;
1238                 attr->enable_on_exec = 0;
1239         }
1240
1241         clockid = opts->clockid;
1242         if (opts->use_clockid) {
1243                 attr->use_clockid = 1;
1244                 attr->clockid = opts->clockid;
1245         }
1246
1247         if (evsel->precise_max)
1248                 attr->precise_ip = 3;
1249
1250         if (opts->all_user) {
1251                 attr->exclude_kernel = 1;
1252                 attr->exclude_user   = 0;
1253         }
1254
1255         if (opts->all_kernel) {
1256                 attr->exclude_kernel = 0;
1257                 attr->exclude_user   = 1;
1258         }
1259
1260         if (evsel->core.own_cpus || evsel->unit)
1261                 evsel->core.attr.read_format |= PERF_FORMAT_ID;
1262
1263         /*
1264          * Apply event specific term settings,
1265          * it overloads any global configuration.
1266          */
1267         evsel__apply_config_terms(evsel, opts, track);
1268
1269         evsel->ignore_missing_thread = opts->ignore_missing_thread;
1270
1271         /* The --period option takes the precedence. */
1272         if (opts->period_set) {
1273                 if (opts->period)
1274                         evsel__set_sample_bit(evsel, PERIOD);
1275                 else
1276                         evsel__reset_sample_bit(evsel, PERIOD);
1277         }
1278
1279         /*
1280          * A dummy event never triggers any actual counter and therefore
1281          * cannot be used with branch_stack.
1282          *
1283          * For initial_delay, a dummy event is added implicitly.
1284          * The software event will trigger -EOPNOTSUPP error out,
1285          * if BRANCH_STACK bit is set.
1286          */
1287         if (evsel__is_dummy_event(evsel))
1288                 evsel__reset_sample_bit(evsel, BRANCH_STACK);
1289 }
1290
1291 int evsel__set_filter(struct evsel *evsel, const char *filter)
1292 {
1293         char *new_filter = strdup(filter);
1294
1295         if (new_filter != NULL) {
1296                 free(evsel->filter);
1297                 evsel->filter = new_filter;
1298                 return 0;
1299         }
1300
1301         return -1;
1302 }
1303
1304 static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter)
1305 {
1306         char *new_filter;
1307
1308         if (evsel->filter == NULL)
1309                 return evsel__set_filter(evsel, filter);
1310
1311         if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) {
1312                 free(evsel->filter);
1313                 evsel->filter = new_filter;
1314                 return 0;
1315         }
1316
1317         return -1;
1318 }
1319
1320 int evsel__append_tp_filter(struct evsel *evsel, const char *filter)
1321 {
1322         return evsel__append_filter(evsel, "(%s) && (%s)", filter);
1323 }
1324
1325 int evsel__append_addr_filter(struct evsel *evsel, const char *filter)
1326 {
1327         return evsel__append_filter(evsel, "%s,%s", filter);
1328 }
1329
1330 /* Caller has to clear disabled after going through all CPUs. */
1331 int evsel__enable_cpu(struct evsel *evsel, int cpu)
1332 {
1333         return perf_evsel__enable_cpu(&evsel->core, cpu);
1334 }
1335
1336 int evsel__enable(struct evsel *evsel)
1337 {
1338         int err = perf_evsel__enable(&evsel->core);
1339
1340         if (!err)
1341                 evsel->disabled = false;
1342         return err;
1343 }
1344
1345 /* Caller has to set disabled after going through all CPUs. */
1346 int evsel__disable_cpu(struct evsel *evsel, int cpu)
1347 {
1348         return perf_evsel__disable_cpu(&evsel->core, cpu);
1349 }
1350
1351 int evsel__disable(struct evsel *evsel)
1352 {
1353         int err = perf_evsel__disable(&evsel->core);
1354         /*
1355          * We mark it disabled here so that tools that disable a event can
1356          * ignore events after they disable it. I.e. the ring buffer may have
1357          * already a few more events queued up before the kernel got the stop
1358          * request.
1359          */
1360         if (!err)
1361                 evsel->disabled = true;
1362
1363         return err;
1364 }
1365
1366 static void evsel__free_config_terms(struct evsel *evsel)
1367 {
1368         struct evsel_config_term *term, *h;
1369
1370         list_for_each_entry_safe(term, h, &evsel->config_terms, list) {
1371                 list_del_init(&term->list);
1372                 if (term->free_str)
1373                         zfree(&term->val.str);
1374                 free(term);
1375         }
1376 }
1377
1378 void evsel__exit(struct evsel *evsel)
1379 {
1380         assert(list_empty(&evsel->core.node));
1381         assert(evsel->evlist == NULL);
1382         bpf_counter__destroy(evsel);
1383         evsel__free_counts(evsel);
1384         perf_evsel__free_fd(&evsel->core);
1385         perf_evsel__free_id(&evsel->core);
1386         evsel__free_config_terms(evsel);
1387         cgroup__put(evsel->cgrp);
1388         perf_cpu_map__put(evsel->core.cpus);
1389         perf_cpu_map__put(evsel->core.own_cpus);
1390         perf_thread_map__put(evsel->core.threads);
1391         zfree(&evsel->group_name);
1392         zfree(&evsel->name);
1393         zfree(&evsel->pmu_name);
1394         evsel__zero_per_pkg(evsel);
1395         hashmap__free(evsel->per_pkg_mask);
1396         evsel->per_pkg_mask = NULL;
1397         zfree(&evsel->metric_events);
1398         perf_evsel__object.fini(evsel);
1399 }
1400
1401 void evsel__delete(struct evsel *evsel)
1402 {
1403         evsel__exit(evsel);
1404         free(evsel);
1405 }
1406
1407 void evsel__compute_deltas(struct evsel *evsel, int cpu, int thread,
1408                            struct perf_counts_values *count)
1409 {
1410         struct perf_counts_values tmp;
1411
1412         if (!evsel->prev_raw_counts)
1413                 return;
1414
1415         if (cpu == -1) {
1416                 tmp = evsel->prev_raw_counts->aggr;
1417                 evsel->prev_raw_counts->aggr = *count;
1418         } else {
1419                 tmp = *perf_counts(evsel->prev_raw_counts, cpu, thread);
1420                 *perf_counts(evsel->prev_raw_counts, cpu, thread) = *count;
1421         }
1422
1423         count->val = count->val - tmp.val;
1424         count->ena = count->ena - tmp.ena;
1425         count->run = count->run - tmp.run;
1426 }
1427
1428 void perf_counts_values__scale(struct perf_counts_values *count,
1429                                bool scale, s8 *pscaled)
1430 {
1431         s8 scaled = 0;
1432
1433         if (scale) {
1434                 if (count->run == 0) {
1435                         scaled = -1;
1436                         count->val = 0;
1437                 } else if (count->run < count->ena) {
1438                         scaled = 1;
1439                         count->val = (u64)((double) count->val * count->ena / count->run);
1440                 }
1441         }
1442
1443         if (pscaled)
1444                 *pscaled = scaled;
1445 }
1446
1447 static int evsel__read_one(struct evsel *evsel, int cpu, int thread)
1448 {
1449         struct perf_counts_values *count = perf_counts(evsel->counts, cpu, thread);
1450
1451         return perf_evsel__read(&evsel->core, cpu, thread, count);
1452 }
1453
1454 static void evsel__set_count(struct evsel *counter, int cpu, int thread, u64 val, u64 ena, u64 run)
1455 {
1456         struct perf_counts_values *count;
1457
1458         count = perf_counts(counter->counts, cpu, thread);
1459
1460         count->val    = val;
1461         count->ena    = ena;
1462         count->run    = run;
1463
1464         perf_counts__set_loaded(counter->counts, cpu, thread, true);
1465 }
1466
1467 static int evsel__process_group_data(struct evsel *leader, int cpu, int thread, u64 *data)
1468 {
1469         u64 read_format = leader->core.attr.read_format;
1470         struct sample_read_value *v;
1471         u64 nr, ena = 0, run = 0, i;
1472
1473         nr = *data++;
1474
1475         if (nr != (u64) leader->core.nr_members)
1476                 return -EINVAL;
1477
1478         if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1479                 ena = *data++;
1480
1481         if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1482                 run = *data++;
1483
1484         v = (struct sample_read_value *) data;
1485
1486         evsel__set_count(leader, cpu, thread, v[0].value, ena, run);
1487
1488         for (i = 1; i < nr; i++) {
1489                 struct evsel *counter;
1490
1491                 counter = evlist__id2evsel(leader->evlist, v[i].id);
1492                 if (!counter)
1493                         return -EINVAL;
1494
1495                 evsel__set_count(counter, cpu, thread, v[i].value, ena, run);
1496         }
1497
1498         return 0;
1499 }
1500
1501 static int evsel__read_group(struct evsel *leader, int cpu, int thread)
1502 {
1503         struct perf_stat_evsel *ps = leader->stats;
1504         u64 read_format = leader->core.attr.read_format;
1505         int size = perf_evsel__read_size(&leader->core);
1506         u64 *data = ps->group_data;
1507
1508         if (!(read_format & PERF_FORMAT_ID))
1509                 return -EINVAL;
1510
1511         if (!evsel__is_group_leader(leader))
1512                 return -EINVAL;
1513
1514         if (!data) {
1515                 data = zalloc(size);
1516                 if (!data)
1517                         return -ENOMEM;
1518
1519                 ps->group_data = data;
1520         }
1521
1522         if (FD(leader, cpu, thread) < 0)
1523                 return -EINVAL;
1524
1525         if (readn(FD(leader, cpu, thread), data, size) <= 0)
1526                 return -errno;
1527
1528         return evsel__process_group_data(leader, cpu, thread, data);
1529 }
1530
1531 int evsel__read_counter(struct evsel *evsel, int cpu, int thread)
1532 {
1533         u64 read_format = evsel->core.attr.read_format;
1534
1535         if (read_format & PERF_FORMAT_GROUP)
1536                 return evsel__read_group(evsel, cpu, thread);
1537
1538         return evsel__read_one(evsel, cpu, thread);
1539 }
1540
1541 int __evsel__read_on_cpu(struct evsel *evsel, int cpu, int thread, bool scale)
1542 {
1543         struct perf_counts_values count;
1544         size_t nv = scale ? 3 : 1;
1545
1546         if (FD(evsel, cpu, thread) < 0)
1547                 return -EINVAL;
1548
1549         if (evsel->counts == NULL && evsel__alloc_counts(evsel, cpu + 1, thread + 1) < 0)
1550                 return -ENOMEM;
1551
1552         if (readn(FD(evsel, cpu, thread), &count, nv * sizeof(u64)) <= 0)
1553                 return -errno;
1554
1555         evsel__compute_deltas(evsel, cpu, thread, &count);
1556         perf_counts_values__scale(&count, scale, NULL);
1557         *perf_counts(evsel->counts, cpu, thread) = count;
1558         return 0;
1559 }
1560
1561 static int get_group_fd(struct evsel *evsel, int cpu, int thread)
1562 {
1563         struct evsel *leader = evsel->leader;
1564         int fd;
1565
1566         if (evsel__is_group_leader(evsel))
1567                 return -1;
1568
1569         /*
1570          * Leader must be already processed/open,
1571          * if not it's a bug.
1572          */
1573         BUG_ON(!leader->core.fd);
1574
1575         fd = FD(leader, cpu, thread);
1576         BUG_ON(fd == -1);
1577
1578         return fd;
1579 }
1580
1581 static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx)
1582 {
1583         for (int cpu = 0; cpu < nr_cpus; cpu++)
1584                 for (int thread = thread_idx; thread < nr_threads - 1; thread++)
1585                         FD(pos, cpu, thread) = FD(pos, cpu, thread + 1);
1586 }
1587
1588 static int update_fds(struct evsel *evsel,
1589                       int nr_cpus, int cpu_idx,
1590                       int nr_threads, int thread_idx)
1591 {
1592         struct evsel *pos;
1593
1594         if (cpu_idx >= nr_cpus || thread_idx >= nr_threads)
1595                 return -EINVAL;
1596
1597         evlist__for_each_entry(evsel->evlist, pos) {
1598                 nr_cpus = pos != evsel ? nr_cpus : cpu_idx;
1599
1600                 evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx);
1601
1602                 /*
1603                  * Since fds for next evsel has not been created,
1604                  * there is no need to iterate whole event list.
1605                  */
1606                 if (pos == evsel)
1607                         break;
1608         }
1609         return 0;
1610 }
1611
1612 static bool ignore_missing_thread(struct evsel *evsel,
1613                                   int nr_cpus, int cpu,
1614                                   struct perf_thread_map *threads,
1615                                   int thread, int err)
1616 {
1617         pid_t ignore_pid = perf_thread_map__pid(threads, thread);
1618
1619         if (!evsel->ignore_missing_thread)
1620                 return false;
1621
1622         /* The system wide setup does not work with threads. */
1623         if (evsel->core.system_wide)
1624                 return false;
1625
1626         /* The -ESRCH is perf event syscall errno for pid's not found. */
1627         if (err != -ESRCH)
1628                 return false;
1629
1630         /* If there's only one thread, let it fail. */
1631         if (threads->nr == 1)
1632                 return false;
1633
1634         /*
1635          * We should remove fd for missing_thread first
1636          * because thread_map__remove() will decrease threads->nr.
1637          */
1638         if (update_fds(evsel, nr_cpus, cpu, threads->nr, thread))
1639                 return false;
1640
1641         if (thread_map__remove(threads, thread))
1642                 return false;
1643
1644         pr_warning("WARNING: Ignored open failure for pid %d\n",
1645                    ignore_pid);
1646         return true;
1647 }
1648
1649 static int __open_attr__fprintf(FILE *fp, const char *name, const char *val,
1650                                 void *priv __maybe_unused)
1651 {
1652         return fprintf(fp, "  %-32s %s\n", name, val);
1653 }
1654
1655 static void display_attr(struct perf_event_attr *attr)
1656 {
1657         if (verbose >= 2 || debug_peo_args) {
1658                 fprintf(stderr, "%.60s\n", graph_dotted_line);
1659                 fprintf(stderr, "perf_event_attr:\n");
1660                 perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL);
1661                 fprintf(stderr, "%.60s\n", graph_dotted_line);
1662         }
1663 }
1664
1665 static int perf_event_open(struct evsel *evsel,
1666                            pid_t pid, int cpu, int group_fd,
1667                            unsigned long flags)
1668 {
1669         int precise_ip = evsel->core.attr.precise_ip;
1670         int fd;
1671
1672         while (1) {
1673                 pr_debug2_peo("sys_perf_event_open: pid %d  cpu %d  group_fd %d  flags %#lx",
1674                           pid, cpu, group_fd, flags);
1675
1676                 fd = sys_perf_event_open(&evsel->core.attr, pid, cpu, group_fd, flags);
1677                 if (fd >= 0)
1678                         break;
1679
1680                 /* Do not try less precise if not requested. */
1681                 if (!evsel->precise_max)
1682                         break;
1683
1684                 /*
1685                  * We tried all the precise_ip values, and it's
1686                  * still failing, so leave it to standard fallback.
1687                  */
1688                 if (!evsel->core.attr.precise_ip) {
1689                         evsel->core.attr.precise_ip = precise_ip;
1690                         break;
1691                 }
1692
1693                 pr_debug2_peo("\nsys_perf_event_open failed, error %d\n", -ENOTSUP);
1694                 evsel->core.attr.precise_ip--;
1695                 pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip);
1696                 display_attr(&evsel->core.attr);
1697         }
1698
1699         return fd;
1700 }
1701
1702 static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus,
1703                 struct perf_thread_map *threads,
1704                 int start_cpu, int end_cpu)
1705 {
1706         int cpu, thread, nthreads;
1707         unsigned long flags = PERF_FLAG_FD_CLOEXEC;
1708         int pid = -1, err, old_errno;
1709         enum { NO_CHANGE, SET_TO_MAX, INCREASED_MAX } set_rlimit = NO_CHANGE;
1710
1711         if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) ||
1712             (perf_missing_features.aux_output     && evsel->core.attr.aux_output))
1713                 return -EINVAL;
1714
1715         if (cpus == NULL) {
1716                 static struct perf_cpu_map *empty_cpu_map;
1717
1718                 if (empty_cpu_map == NULL) {
1719                         empty_cpu_map = perf_cpu_map__dummy_new();
1720                         if (empty_cpu_map == NULL)
1721                                 return -ENOMEM;
1722                 }
1723
1724                 cpus = empty_cpu_map;
1725         }
1726
1727         if (threads == NULL) {
1728                 static struct perf_thread_map *empty_thread_map;
1729
1730                 if (empty_thread_map == NULL) {
1731                         empty_thread_map = thread_map__new_by_tid(-1);
1732                         if (empty_thread_map == NULL)
1733                                 return -ENOMEM;
1734                 }
1735
1736                 threads = empty_thread_map;
1737         }
1738
1739         if (evsel->core.system_wide)
1740                 nthreads = 1;
1741         else
1742                 nthreads = threads->nr;
1743
1744         if (evsel->core.fd == NULL &&
1745             perf_evsel__alloc_fd(&evsel->core, cpus->nr, nthreads) < 0)
1746                 return -ENOMEM;
1747
1748         if (evsel->cgrp) {
1749                 flags |= PERF_FLAG_PID_CGROUP;
1750                 pid = evsel->cgrp->fd;
1751         }
1752
1753 fallback_missing_features:
1754         if (perf_missing_features.weight_struct) {
1755                 evsel__set_sample_bit(evsel, WEIGHT);
1756                 evsel__reset_sample_bit(evsel, WEIGHT_STRUCT);
1757         }
1758         if (perf_missing_features.clockid_wrong)
1759                 evsel->core.attr.clockid = CLOCK_MONOTONIC; /* should always work */
1760         if (perf_missing_features.clockid) {
1761                 evsel->core.attr.use_clockid = 0;
1762                 evsel->core.attr.clockid = 0;
1763         }
1764         if (perf_missing_features.cloexec)
1765                 flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC;
1766         if (perf_missing_features.mmap2)
1767                 evsel->core.attr.mmap2 = 0;
1768         if (perf_missing_features.exclude_guest)
1769                 evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0;
1770         if (perf_missing_features.lbr_flags)
1771                 evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS |
1772                                      PERF_SAMPLE_BRANCH_NO_CYCLES);
1773         if (perf_missing_features.group_read && evsel->core.attr.inherit)
1774                 evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID);
1775         if (perf_missing_features.ksymbol)
1776                 evsel->core.attr.ksymbol = 0;
1777         if (perf_missing_features.bpf)
1778                 evsel->core.attr.bpf_event = 0;
1779         if (perf_missing_features.branch_hw_idx)
1780                 evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX;
1781 retry_sample_id:
1782         if (perf_missing_features.sample_id_all)
1783                 evsel->core.attr.sample_id_all = 0;
1784
1785         display_attr(&evsel->core.attr);
1786
1787         for (cpu = start_cpu; cpu < end_cpu; cpu++) {
1788
1789                 for (thread = 0; thread < nthreads; thread++) {
1790                         int fd, group_fd;
1791
1792                         if (!evsel->cgrp && !evsel->core.system_wide)
1793                                 pid = perf_thread_map__pid(threads, thread);
1794
1795                         group_fd = get_group_fd(evsel, cpu, thread);
1796 retry_open:
1797                         test_attr__ready();
1798
1799                         fd = perf_event_open(evsel, pid, cpus->map[cpu],
1800                                              group_fd, flags);
1801
1802                         FD(evsel, cpu, thread) = fd;
1803
1804                         bpf_counter__install_pe(evsel, cpu, fd);
1805
1806                         if (unlikely(test_attr__enabled)) {
1807                                 test_attr__open(&evsel->core.attr, pid, cpus->map[cpu],
1808                                                 fd, group_fd, flags);
1809                         }
1810
1811                         if (fd < 0) {
1812                                 err = -errno;
1813
1814                                 if (ignore_missing_thread(evsel, cpus->nr, cpu, threads, thread, err)) {
1815                                         /*
1816                                          * We just removed 1 thread, so take a step
1817                                          * back on thread index and lower the upper
1818                                          * nthreads limit.
1819                                          */
1820                                         nthreads--;
1821                                         thread--;
1822
1823                                         /* ... and pretend like nothing have happened. */
1824                                         err = 0;
1825                                         continue;
1826                                 }
1827
1828                                 pr_debug2_peo("\nsys_perf_event_open failed, error %d\n",
1829                                           err);
1830                                 goto try_fallback;
1831                         }
1832
1833                         pr_debug2_peo(" = %d\n", fd);
1834
1835                         if (evsel->bpf_fd >= 0) {
1836                                 int evt_fd = fd;
1837                                 int bpf_fd = evsel->bpf_fd;
1838
1839                                 err = ioctl(evt_fd,
1840                                             PERF_EVENT_IOC_SET_BPF,
1841                                             bpf_fd);
1842                                 if (err && errno != EEXIST) {
1843                                         pr_err("failed to attach bpf fd %d: %s\n",
1844                                                bpf_fd, strerror(errno));
1845                                         err = -EINVAL;
1846                                         goto out_close;
1847                                 }
1848                         }
1849
1850                         set_rlimit = NO_CHANGE;
1851
1852                         /*
1853                          * If we succeeded but had to kill clockid, fail and
1854                          * have evsel__open_strerror() print us a nice error.
1855                          */
1856                         if (perf_missing_features.clockid ||
1857                             perf_missing_features.clockid_wrong) {
1858                                 err = -EINVAL;
1859                                 goto out_close;
1860                         }
1861                 }
1862         }
1863
1864         return 0;
1865
1866 try_fallback:
1867         /*
1868          * perf stat needs between 5 and 22 fds per CPU. When we run out
1869          * of them try to increase the limits.
1870          */
1871         if (err == -EMFILE && set_rlimit < INCREASED_MAX) {
1872                 struct rlimit l;
1873
1874                 old_errno = errno;
1875                 if (getrlimit(RLIMIT_NOFILE, &l) == 0) {
1876                         if (set_rlimit == NO_CHANGE)
1877                                 l.rlim_cur = l.rlim_max;
1878                         else {
1879                                 l.rlim_cur = l.rlim_max + 1000;
1880                                 l.rlim_max = l.rlim_cur;
1881                         }
1882                         if (setrlimit(RLIMIT_NOFILE, &l) == 0) {
1883                                 set_rlimit++;
1884                                 errno = old_errno;
1885                                 goto retry_open;
1886                         }
1887                 }
1888                 errno = old_errno;
1889         }
1890
1891         if (err != -EINVAL || cpu > 0 || thread > 0)
1892                 goto out_close;
1893
1894         /*
1895          * Must probe features in the order they were added to the
1896          * perf_event_attr interface.
1897          */
1898         if (!perf_missing_features.weight_struct &&
1899             (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) {
1900                 perf_missing_features.weight_struct = true;
1901                 pr_debug2("switching off weight struct support\n");
1902                 goto fallback_missing_features;
1903         } else if (!perf_missing_features.code_page_size &&
1904             (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)) {
1905                 perf_missing_features.code_page_size = true;
1906                 pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support, bailing out\n");
1907                 goto out_close;
1908         } else if (!perf_missing_features.data_page_size &&
1909             (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)) {
1910                 perf_missing_features.data_page_size = true;
1911                 pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support, bailing out\n");
1912                 goto out_close;
1913         } else if (!perf_missing_features.cgroup && evsel->core.attr.cgroup) {
1914                 perf_missing_features.cgroup = true;
1915                 pr_debug2_peo("Kernel has no cgroup sampling support, bailing out\n");
1916                 goto out_close;
1917         } else if (!perf_missing_features.branch_hw_idx &&
1918             (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX)) {
1919                 perf_missing_features.branch_hw_idx = true;
1920                 pr_debug2("switching off branch HW index support\n");
1921                 goto fallback_missing_features;
1922         } else if (!perf_missing_features.aux_output && evsel->core.attr.aux_output) {
1923                 perf_missing_features.aux_output = true;
1924                 pr_debug2_peo("Kernel has no attr.aux_output support, bailing out\n");
1925                 goto out_close;
1926         } else if (!perf_missing_features.bpf && evsel->core.attr.bpf_event) {
1927                 perf_missing_features.bpf = true;
1928                 pr_debug2_peo("switching off bpf_event\n");
1929                 goto fallback_missing_features;
1930         } else if (!perf_missing_features.ksymbol && evsel->core.attr.ksymbol) {
1931                 perf_missing_features.ksymbol = true;
1932                 pr_debug2_peo("switching off ksymbol\n");
1933                 goto fallback_missing_features;
1934         } else if (!perf_missing_features.write_backward && evsel->core.attr.write_backward) {
1935                 perf_missing_features.write_backward = true;
1936                 pr_debug2_peo("switching off write_backward\n");
1937                 goto out_close;
1938         } else if (!perf_missing_features.clockid_wrong && evsel->core.attr.use_clockid) {
1939                 perf_missing_features.clockid_wrong = true;
1940                 pr_debug2_peo("switching off clockid\n");
1941                 goto fallback_missing_features;
1942         } else if (!perf_missing_features.clockid && evsel->core.attr.use_clockid) {
1943                 perf_missing_features.clockid = true;
1944                 pr_debug2_peo("switching off use_clockid\n");
1945                 goto fallback_missing_features;
1946         } else if (!perf_missing_features.cloexec && (flags & PERF_FLAG_FD_CLOEXEC)) {
1947                 perf_missing_features.cloexec = true;
1948                 pr_debug2_peo("switching off cloexec flag\n");
1949                 goto fallback_missing_features;
1950         } else if (!perf_missing_features.mmap2 && evsel->core.attr.mmap2) {
1951                 perf_missing_features.mmap2 = true;
1952                 pr_debug2_peo("switching off mmap2\n");
1953                 goto fallback_missing_features;
1954         } else if (!perf_missing_features.exclude_guest &&
1955                    (evsel->core.attr.exclude_guest || evsel->core.attr.exclude_host)) {
1956                 perf_missing_features.exclude_guest = true;
1957                 pr_debug2_peo("switching off exclude_guest, exclude_host\n");
1958                 goto fallback_missing_features;
1959         } else if (!perf_missing_features.sample_id_all) {
1960                 perf_missing_features.sample_id_all = true;
1961                 pr_debug2_peo("switching off sample_id_all\n");
1962                 goto retry_sample_id;
1963         } else if (!perf_missing_features.lbr_flags &&
1964                         (evsel->core.attr.branch_sample_type &
1965                          (PERF_SAMPLE_BRANCH_NO_CYCLES |
1966                           PERF_SAMPLE_BRANCH_NO_FLAGS))) {
1967                 perf_missing_features.lbr_flags = true;
1968                 pr_debug2_peo("switching off branch sample type no (cycles/flags)\n");
1969                 goto fallback_missing_features;
1970         } else if (!perf_missing_features.group_read &&
1971                     evsel->core.attr.inherit &&
1972                    (evsel->core.attr.read_format & PERF_FORMAT_GROUP) &&
1973                    evsel__is_group_leader(evsel)) {
1974                 perf_missing_features.group_read = true;
1975                 pr_debug2_peo("switching off group read\n");
1976                 goto fallback_missing_features;
1977         }
1978 out_close:
1979         if (err)
1980                 threads->err_thread = thread;
1981
1982         old_errno = errno;
1983         do {
1984                 while (--thread >= 0) {
1985                         if (FD(evsel, cpu, thread) >= 0)
1986                                 close(FD(evsel, cpu, thread));
1987                         FD(evsel, cpu, thread) = -1;
1988                 }
1989                 thread = nthreads;
1990         } while (--cpu >= 0);
1991         errno = old_errno;
1992         return err;
1993 }
1994
1995 int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus,
1996                 struct perf_thread_map *threads)
1997 {
1998         return evsel__open_cpu(evsel, cpus, threads, 0, cpus ? cpus->nr : 1);
1999 }
2000
2001 void evsel__close(struct evsel *evsel)
2002 {
2003         perf_evsel__close(&evsel->core);
2004         perf_evsel__free_id(&evsel->core);
2005 }
2006
2007 int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu)
2008 {
2009         if (cpu == -1)
2010                 return evsel__open_cpu(evsel, cpus, NULL, 0,
2011                                         cpus ? cpus->nr : 1);
2012
2013         return evsel__open_cpu(evsel, cpus, NULL, cpu, cpu + 1);
2014 }
2015
2016 int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads)
2017 {
2018         return evsel__open(evsel, NULL, threads);
2019 }
2020
2021 static int perf_evsel__parse_id_sample(const struct evsel *evsel,
2022                                        const union perf_event *event,
2023                                        struct perf_sample *sample)
2024 {
2025         u64 type = evsel->core.attr.sample_type;
2026         const __u64 *array = event->sample.array;
2027         bool swapped = evsel->needs_swap;
2028         union u64_swap u;
2029
2030         array += ((event->header.size -
2031                    sizeof(event->header)) / sizeof(u64)) - 1;
2032
2033         if (type & PERF_SAMPLE_IDENTIFIER) {
2034                 sample->id = *array;
2035                 array--;
2036         }
2037
2038         if (type & PERF_SAMPLE_CPU) {
2039                 u.val64 = *array;
2040                 if (swapped) {
2041                         /* undo swap of u64, then swap on individual u32s */
2042                         u.val64 = bswap_64(u.val64);
2043                         u.val32[0] = bswap_32(u.val32[0]);
2044                 }
2045
2046                 sample->cpu = u.val32[0];
2047                 array--;
2048         }
2049
2050         if (type & PERF_SAMPLE_STREAM_ID) {
2051                 sample->stream_id = *array;
2052                 array--;
2053         }
2054
2055         if (type & PERF_SAMPLE_ID) {
2056                 sample->id = *array;
2057                 array--;
2058         }
2059
2060         if (type & PERF_SAMPLE_TIME) {
2061                 sample->time = *array;
2062                 array--;
2063         }
2064
2065         if (type & PERF_SAMPLE_TID) {
2066                 u.val64 = *array;
2067                 if (swapped) {
2068                         /* undo swap of u64, then swap on individual u32s */
2069                         u.val64 = bswap_64(u.val64);
2070                         u.val32[0] = bswap_32(u.val32[0]);
2071                         u.val32[1] = bswap_32(u.val32[1]);
2072                 }
2073
2074                 sample->pid = u.val32[0];
2075                 sample->tid = u.val32[1];
2076                 array--;
2077         }
2078
2079         return 0;
2080 }
2081
2082 static inline bool overflow(const void *endp, u16 max_size, const void *offset,
2083                             u64 size)
2084 {
2085         return size > max_size || offset + size > endp;
2086 }
2087
2088 #define OVERFLOW_CHECK(offset, size, max_size)                          \
2089         do {                                                            \
2090                 if (overflow(endp, (max_size), (offset), (size)))       \
2091                         return -EFAULT;                                 \
2092         } while (0)
2093
2094 #define OVERFLOW_CHECK_u64(offset) \
2095         OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64))
2096
2097 static int
2098 perf_event__check_size(union perf_event *event, unsigned int sample_size)
2099 {
2100         /*
2101          * The evsel's sample_size is based on PERF_SAMPLE_MASK which includes
2102          * up to PERF_SAMPLE_PERIOD.  After that overflow() must be used to
2103          * check the format does not go past the end of the event.
2104          */
2105         if (sample_size + sizeof(event->header) > event->header.size)
2106                 return -EFAULT;
2107
2108         return 0;
2109 }
2110
2111 void __weak arch_perf_parse_sample_weight(struct perf_sample *data,
2112                                           const __u64 *array,
2113                                           u64 type __maybe_unused)
2114 {
2115         data->weight = *array;
2116 }
2117
2118 int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
2119                         struct perf_sample *data)
2120 {
2121         u64 type = evsel->core.attr.sample_type;
2122         bool swapped = evsel->needs_swap;
2123         const __u64 *array;
2124         u16 max_size = event->header.size;
2125         const void *endp = (void *)event + max_size;
2126         u64 sz;
2127
2128         /*
2129          * used for cross-endian analysis. See git commit 65014ab3
2130          * for why this goofiness is needed.
2131          */
2132         union u64_swap u;
2133
2134         memset(data, 0, sizeof(*data));
2135         data->cpu = data->pid = data->tid = -1;
2136         data->stream_id = data->id = data->time = -1ULL;
2137         data->period = evsel->core.attr.sample_period;
2138         data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
2139         data->misc    = event->header.misc;
2140         data->id = -1ULL;
2141         data->data_src = PERF_MEM_DATA_SRC_NONE;
2142
2143         if (event->header.type != PERF_RECORD_SAMPLE) {
2144                 if (!evsel->core.attr.sample_id_all)
2145                         return 0;
2146                 return perf_evsel__parse_id_sample(evsel, event, data);
2147         }
2148
2149         array = event->sample.array;
2150
2151         if (perf_event__check_size(event, evsel->sample_size))
2152                 return -EFAULT;
2153
2154         if (type & PERF_SAMPLE_IDENTIFIER) {
2155                 data->id = *array;
2156                 array++;
2157         }
2158
2159         if (type & PERF_SAMPLE_IP) {
2160                 data->ip = *array;
2161                 array++;
2162         }
2163
2164         if (type & PERF_SAMPLE_TID) {
2165                 u.val64 = *array;
2166                 if (swapped) {
2167                         /* undo swap of u64, then swap on individual u32s */
2168                         u.val64 = bswap_64(u.val64);
2169                         u.val32[0] = bswap_32(u.val32[0]);
2170                         u.val32[1] = bswap_32(u.val32[1]);
2171                 }
2172
2173                 data->pid = u.val32[0];
2174                 data->tid = u.val32[1];
2175                 array++;
2176         }
2177
2178         if (type & PERF_SAMPLE_TIME) {
2179                 data->time = *array;
2180                 array++;
2181         }
2182
2183         if (type & PERF_SAMPLE_ADDR) {
2184                 data->addr = *array;
2185                 array++;
2186         }
2187
2188         if (type & PERF_SAMPLE_ID) {
2189                 data->id = *array;
2190                 array++;
2191         }
2192
2193         if (type & PERF_SAMPLE_STREAM_ID) {
2194                 data->stream_id = *array;
2195                 array++;
2196         }
2197
2198         if (type & PERF_SAMPLE_CPU) {
2199
2200                 u.val64 = *array;
2201                 if (swapped) {
2202                         /* undo swap of u64, then swap on individual u32s */
2203                         u.val64 = bswap_64(u.val64);
2204                         u.val32[0] = bswap_32(u.val32[0]);
2205                 }
2206
2207                 data->cpu = u.val32[0];
2208                 array++;
2209         }
2210
2211         if (type & PERF_SAMPLE_PERIOD) {
2212                 data->period = *array;
2213                 array++;
2214         }
2215
2216         if (type & PERF_SAMPLE_READ) {
2217                 u64 read_format = evsel->core.attr.read_format;
2218
2219                 OVERFLOW_CHECK_u64(array);
2220                 if (read_format & PERF_FORMAT_GROUP)
2221                         data->read.group.nr = *array;
2222                 else
2223                         data->read.one.value = *array;
2224
2225                 array++;
2226
2227                 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2228                         OVERFLOW_CHECK_u64(array);
2229                         data->read.time_enabled = *array;
2230                         array++;
2231                 }
2232
2233                 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2234                         OVERFLOW_CHECK_u64(array);
2235                         data->read.time_running = *array;
2236                         array++;
2237                 }
2238
2239                 /* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */
2240                 if (read_format & PERF_FORMAT_GROUP) {
2241                         const u64 max_group_nr = UINT64_MAX /
2242                                         sizeof(struct sample_read_value);
2243
2244                         if (data->read.group.nr > max_group_nr)
2245                                 return -EFAULT;
2246                         sz = data->read.group.nr *
2247                              sizeof(struct sample_read_value);
2248                         OVERFLOW_CHECK(array, sz, max_size);
2249                         data->read.group.values =
2250                                         (struct sample_read_value *)array;
2251                         array = (void *)array + sz;
2252                 } else {
2253                         OVERFLOW_CHECK_u64(array);
2254                         data->read.one.id = *array;
2255                         array++;
2256                 }
2257         }
2258
2259         if (type & PERF_SAMPLE_CALLCHAIN) {
2260                 const u64 max_callchain_nr = UINT64_MAX / sizeof(u64);
2261
2262                 OVERFLOW_CHECK_u64(array);
2263                 data->callchain = (struct ip_callchain *)array++;
2264                 if (data->callchain->nr > max_callchain_nr)
2265                         return -EFAULT;
2266                 sz = data->callchain->nr * sizeof(u64);
2267                 OVERFLOW_CHECK(array, sz, max_size);
2268                 array = (void *)array + sz;
2269         }
2270
2271         if (type & PERF_SAMPLE_RAW) {
2272                 OVERFLOW_CHECK_u64(array);
2273                 u.val64 = *array;
2274
2275                 /*
2276                  * Undo swap of u64, then swap on individual u32s,
2277                  * get the size of the raw area and undo all of the
2278                  * swap. The pevent interface handles endianness by
2279                  * itself.
2280                  */
2281                 if (swapped) {
2282                         u.val64 = bswap_64(u.val64);
2283                         u.val32[0] = bswap_32(u.val32[0]);
2284                         u.val32[1] = bswap_32(u.val32[1]);
2285                 }
2286                 data->raw_size = u.val32[0];
2287
2288                 /*
2289                  * The raw data is aligned on 64bits including the
2290                  * u32 size, so it's safe to use mem_bswap_64.
2291                  */
2292                 if (swapped)
2293                         mem_bswap_64((void *) array, data->raw_size);
2294
2295                 array = (void *)array + sizeof(u32);
2296
2297                 OVERFLOW_CHECK(array, data->raw_size, max_size);
2298                 data->raw_data = (void *)array;
2299                 array = (void *)array + data->raw_size;
2300         }
2301
2302         if (type & PERF_SAMPLE_BRANCH_STACK) {
2303                 const u64 max_branch_nr = UINT64_MAX /
2304                                           sizeof(struct branch_entry);
2305
2306                 OVERFLOW_CHECK_u64(array);
2307                 data->branch_stack = (struct branch_stack *)array++;
2308
2309                 if (data->branch_stack->nr > max_branch_nr)
2310                         return -EFAULT;
2311
2312                 sz = data->branch_stack->nr * sizeof(struct branch_entry);
2313                 if (evsel__has_branch_hw_idx(evsel))
2314                         sz += sizeof(u64);
2315                 else
2316                         data->no_hw_idx = true;
2317                 OVERFLOW_CHECK(array, sz, max_size);
2318                 array = (void *)array + sz;
2319         }
2320
2321         if (type & PERF_SAMPLE_REGS_USER) {
2322                 OVERFLOW_CHECK_u64(array);
2323                 data->user_regs.abi = *array;
2324                 array++;
2325
2326                 if (data->user_regs.abi) {
2327                         u64 mask = evsel->core.attr.sample_regs_user;
2328
2329                         sz = hweight64(mask) * sizeof(u64);
2330                         OVERFLOW_CHECK(array, sz, max_size);
2331                         data->user_regs.mask = mask;
2332                         data->user_regs.regs = (u64 *)array;
2333                         array = (void *)array + sz;
2334                 }
2335         }
2336
2337         if (type & PERF_SAMPLE_STACK_USER) {
2338                 OVERFLOW_CHECK_u64(array);
2339                 sz = *array++;
2340
2341                 data->user_stack.offset = ((char *)(array - 1)
2342                                           - (char *) event);
2343
2344                 if (!sz) {
2345                         data->user_stack.size = 0;
2346                 } else {
2347                         OVERFLOW_CHECK(array, sz, max_size);
2348                         data->user_stack.data = (char *)array;
2349                         array = (void *)array + sz;
2350                         OVERFLOW_CHECK_u64(array);
2351                         data->user_stack.size = *array++;
2352                         if (WARN_ONCE(data->user_stack.size > sz,
2353                                       "user stack dump failure\n"))
2354                                 return -EFAULT;
2355                 }
2356         }
2357
2358         if (type & PERF_SAMPLE_WEIGHT_TYPE) {
2359                 OVERFLOW_CHECK_u64(array);
2360                 arch_perf_parse_sample_weight(data, array, type);
2361                 array++;
2362         }
2363
2364         if (type & PERF_SAMPLE_DATA_SRC) {
2365                 OVERFLOW_CHECK_u64(array);
2366                 data->data_src = *array;
2367                 array++;
2368         }
2369
2370         if (type & PERF_SAMPLE_TRANSACTION) {
2371                 OVERFLOW_CHECK_u64(array);
2372                 data->transaction = *array;
2373                 array++;
2374         }
2375
2376         data->intr_regs.abi = PERF_SAMPLE_REGS_ABI_NONE;
2377         if (type & PERF_SAMPLE_REGS_INTR) {
2378                 OVERFLOW_CHECK_u64(array);
2379                 data->intr_regs.abi = *array;
2380                 array++;
2381
2382                 if (data->intr_regs.abi != PERF_SAMPLE_REGS_ABI_NONE) {
2383                         u64 mask = evsel->core.attr.sample_regs_intr;
2384
2385                         sz = hweight64(mask) * sizeof(u64);
2386                         OVERFLOW_CHECK(array, sz, max_size);
2387                         data->intr_regs.mask = mask;
2388                         data->intr_regs.regs = (u64 *)array;
2389                         array = (void *)array + sz;
2390                 }
2391         }
2392
2393         data->phys_addr = 0;
2394         if (type & PERF_SAMPLE_PHYS_ADDR) {
2395                 data->phys_addr = *array;
2396                 array++;
2397         }
2398
2399         data->cgroup = 0;
2400         if (type & PERF_SAMPLE_CGROUP) {
2401                 data->cgroup = *array;
2402                 array++;
2403         }
2404
2405         data->data_page_size = 0;
2406         if (type & PERF_SAMPLE_DATA_PAGE_SIZE) {
2407                 data->data_page_size = *array;
2408                 array++;
2409         }
2410
2411         data->code_page_size = 0;
2412         if (type & PERF_SAMPLE_CODE_PAGE_SIZE) {
2413                 data->code_page_size = *array;
2414                 array++;
2415         }
2416
2417         if (type & PERF_SAMPLE_AUX) {
2418                 OVERFLOW_CHECK_u64(array);
2419                 sz = *array++;
2420
2421                 OVERFLOW_CHECK(array, sz, max_size);
2422                 /* Undo swap of data */
2423                 if (swapped)
2424                         mem_bswap_64((char *)array, sz);
2425                 data->aux_sample.size = sz;
2426                 data->aux_sample.data = (char *)array;
2427                 array = (void *)array + sz;
2428         }
2429
2430         return 0;
2431 }
2432
2433 int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event,
2434                                   u64 *timestamp)
2435 {
2436         u64 type = evsel->core.attr.sample_type;
2437         const __u64 *array;
2438
2439         if (!(type & PERF_SAMPLE_TIME))
2440                 return -1;
2441
2442         if (event->header.type != PERF_RECORD_SAMPLE) {
2443                 struct perf_sample data = {
2444                         .time = -1ULL,
2445                 };
2446
2447                 if (!evsel->core.attr.sample_id_all)
2448                         return -1;
2449                 if (perf_evsel__parse_id_sample(evsel, event, &data))
2450                         return -1;
2451
2452                 *timestamp = data.time;
2453                 return 0;
2454         }
2455
2456         array = event->sample.array;
2457
2458         if (perf_event__check_size(event, evsel->sample_size))
2459                 return -EFAULT;
2460
2461         if (type & PERF_SAMPLE_IDENTIFIER)
2462                 array++;
2463
2464         if (type & PERF_SAMPLE_IP)
2465                 array++;
2466
2467         if (type & PERF_SAMPLE_TID)
2468                 array++;
2469
2470         if (type & PERF_SAMPLE_TIME)
2471                 *timestamp = *array;
2472
2473         return 0;
2474 }
2475
2476 struct tep_format_field *evsel__field(struct evsel *evsel, const char *name)
2477 {
2478         return tep_find_field(evsel->tp_format, name);
2479 }
2480
2481 void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name)
2482 {
2483         struct tep_format_field *field = evsel__field(evsel, name);
2484         int offset;
2485
2486         if (!field)
2487                 return NULL;
2488
2489         offset = field->offset;
2490
2491         if (field->flags & TEP_FIELD_IS_DYNAMIC) {
2492                 offset = *(int *)(sample->raw_data + field->offset);
2493                 offset &= 0xffff;
2494         }
2495
2496         return sample->raw_data + offset;
2497 }
2498
2499 u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample,
2500                          bool needs_swap)
2501 {
2502         u64 value;
2503         void *ptr = sample->raw_data + field->offset;
2504
2505         switch (field->size) {
2506         case 1:
2507                 return *(u8 *)ptr;
2508         case 2:
2509                 value = *(u16 *)ptr;
2510                 break;
2511         case 4:
2512                 value = *(u32 *)ptr;
2513                 break;
2514         case 8:
2515                 memcpy(&value, ptr, sizeof(u64));
2516                 break;
2517         default:
2518                 return 0;
2519         }
2520
2521         if (!needs_swap)
2522                 return value;
2523
2524         switch (field->size) {
2525         case 2:
2526                 return bswap_16(value);
2527         case 4:
2528                 return bswap_32(value);
2529         case 8:
2530                 return bswap_64(value);
2531         default:
2532                 return 0;
2533         }
2534
2535         return 0;
2536 }
2537
2538 u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name)
2539 {
2540         struct tep_format_field *field = evsel__field(evsel, name);
2541
2542         if (!field)
2543                 return 0;
2544
2545         return field ? format_field__intval(field, sample, evsel->needs_swap) : 0;
2546 }
2547
2548 bool evsel__fallback(struct evsel *evsel, int err, char *msg, size_t msgsize)
2549 {
2550         int paranoid;
2551
2552         if ((err == ENOENT || err == ENXIO || err == ENODEV) &&
2553             evsel->core.attr.type   == PERF_TYPE_HARDWARE &&
2554             evsel->core.attr.config == PERF_COUNT_HW_CPU_CYCLES) {
2555                 /*
2556                  * If it's cycles then fall back to hrtimer based
2557                  * cpu-clock-tick sw counter, which is always available even if
2558                  * no PMU support.
2559                  *
2560                  * PPC returns ENXIO until 2.6.37 (behavior changed with commit
2561                  * b0a873e).
2562                  */
2563                 scnprintf(msg, msgsize, "%s",
2564 "The cycles event is not supported, trying to fall back to cpu-clock-ticks");
2565
2566                 evsel->core.attr.type   = PERF_TYPE_SOFTWARE;
2567                 evsel->core.attr.config = PERF_COUNT_SW_CPU_CLOCK;
2568
2569                 zfree(&evsel->name);
2570                 return true;
2571         } else if (err == EACCES && !evsel->core.attr.exclude_kernel &&
2572                    (paranoid = perf_event_paranoid()) > 1) {
2573                 const char *name = evsel__name(evsel);
2574                 char *new_name;
2575                 const char *sep = ":";
2576
2577                 /* If event has exclude user then don't exclude kernel. */
2578                 if (evsel->core.attr.exclude_user)
2579                         return false;
2580
2581                 /* Is there already the separator in the name. */
2582                 if (strchr(name, '/') ||
2583                     (strchr(name, ':') && !evsel->is_libpfm_event))
2584                         sep = "";
2585
2586                 if (asprintf(&new_name, "%s%su", name, sep) < 0)
2587                         return false;
2588
2589                 if (evsel->name)
2590                         free(evsel->name);
2591                 evsel->name = new_name;
2592                 scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying "
2593                           "to fall back to excluding kernel and hypervisor "
2594                           " samples", paranoid);
2595                 evsel->core.attr.exclude_kernel = 1;
2596                 evsel->core.attr.exclude_hv     = 1;
2597
2598                 return true;
2599         }
2600
2601         return false;
2602 }
2603
2604 static bool find_process(const char *name)
2605 {
2606         size_t len = strlen(name);
2607         DIR *dir;
2608         struct dirent *d;
2609         int ret = -1;
2610
2611         dir = opendir(procfs__mountpoint());
2612         if (!dir)
2613                 return false;
2614
2615         /* Walk through the directory. */
2616         while (ret && (d = readdir(dir)) != NULL) {
2617                 char path[PATH_MAX];
2618                 char *data;
2619                 size_t size;
2620
2621                 if ((d->d_type != DT_DIR) ||
2622                      !strcmp(".", d->d_name) ||
2623                      !strcmp("..", d->d_name))
2624                         continue;
2625
2626                 scnprintf(path, sizeof(path), "%s/%s/comm",
2627                           procfs__mountpoint(), d->d_name);
2628
2629                 if (filename__read_str(path, &data, &size))
2630                         continue;
2631
2632                 ret = strncmp(name, data, len);
2633                 free(data);
2634         }
2635
2636         closedir(dir);
2637         return ret ? false : true;
2638 }
2639
2640 int evsel__open_strerror(struct evsel *evsel, struct target *target,
2641                          int err, char *msg, size_t size)
2642 {
2643         char sbuf[STRERR_BUFSIZE];
2644         int printed = 0, enforced = 0;
2645
2646         switch (err) {
2647         case EPERM:
2648         case EACCES:
2649                 printed += scnprintf(msg + printed, size - printed,
2650                         "Access to performance monitoring and observability operations is limited.\n");
2651
2652                 if (!sysfs__read_int("fs/selinux/enforce", &enforced)) {
2653                         if (enforced) {
2654                                 printed += scnprintf(msg + printed, size - printed,
2655                                         "Enforced MAC policy settings (SELinux) can limit access to performance\n"
2656                                         "monitoring and observability operations. Inspect system audit records for\n"
2657                                         "more perf_event access control information and adjusting the policy.\n");
2658                         }
2659                 }
2660
2661                 if (err == EPERM)
2662                         printed += scnprintf(msg, size,
2663                                 "No permission to enable %s event.\n\n", evsel__name(evsel));
2664
2665                 return scnprintf(msg + printed, size - printed,
2666                  "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n"
2667                  "access to performance monitoring and observability operations for processes\n"
2668                  "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n"
2669                  "More information can be found at 'Perf events and tool security' document:\n"
2670                  "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n"
2671                  "perf_event_paranoid setting is %d:\n"
2672                  "  -1: Allow use of (almost) all events by all users\n"
2673                  "      Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n"
2674                  ">= 0: Disallow raw and ftrace function tracepoint access\n"
2675                  ">= 1: Disallow CPU event access\n"
2676                  ">= 2: Disallow kernel profiling\n"
2677                  "To make the adjusted perf_event_paranoid setting permanent preserve it\n"
2678                  "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = <setting>)",
2679                  perf_event_paranoid());
2680         case ENOENT:
2681                 return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel));
2682         case EMFILE:
2683                 return scnprintf(msg, size, "%s",
2684                          "Too many events are opened.\n"
2685                          "Probably the maximum number of open file descriptors has been reached.\n"
2686                          "Hint: Try again after reducing the number of events.\n"
2687                          "Hint: Try increasing the limit with 'ulimit -n <limit>'");
2688         case ENOMEM:
2689                 if (evsel__has_callchain(evsel) &&
2690                     access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0)
2691                         return scnprintf(msg, size,
2692                                          "Not enough memory to setup event with callchain.\n"
2693                                          "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n"
2694                                          "Hint: Current value: %d", sysctl__max_stack());
2695                 break;
2696         case ENODEV:
2697                 if (target->cpu_list)
2698                         return scnprintf(msg, size, "%s",
2699          "No such device - did you specify an out-of-range profile CPU?");
2700                 break;
2701         case EOPNOTSUPP:
2702                 if (evsel->core.attr.aux_output)
2703                         return scnprintf(msg, size,
2704         "%s: PMU Hardware doesn't support 'aux_output' feature",
2705                                          evsel__name(evsel));
2706                 if (evsel->core.attr.sample_period != 0)
2707                         return scnprintf(msg, size,
2708         "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'",
2709                                          evsel__name(evsel));
2710                 if (evsel->core.attr.precise_ip)
2711                         return scnprintf(msg, size, "%s",
2712         "\'precise\' request may not be supported. Try removing 'p' modifier.");
2713 #if defined(__i386__) || defined(__x86_64__)
2714                 if (evsel->core.attr.type == PERF_TYPE_HARDWARE)
2715                         return scnprintf(msg, size, "%s",
2716         "No hardware sampling interrupt available.\n");
2717 #endif
2718                 break;
2719         case EBUSY:
2720                 if (find_process("oprofiled"))
2721                         return scnprintf(msg, size,
2722         "The PMU counters are busy/taken by another profiler.\n"
2723         "We found oprofile daemon running, please stop it and try again.");
2724                 break;
2725         case EINVAL:
2726                 if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size)
2727                         return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel.");
2728                 if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size)
2729                         return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel.");
2730                 if (evsel->core.attr.write_backward && perf_missing_features.write_backward)
2731                         return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel.");
2732                 if (perf_missing_features.clockid)
2733                         return scnprintf(msg, size, "clockid feature not supported.");
2734                 if (perf_missing_features.clockid_wrong)
2735                         return scnprintf(msg, size, "wrong clockid (%d).", clockid);
2736                 if (perf_missing_features.aux_output)
2737                         return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel.");
2738                 break;
2739         case ENODATA:
2740                 return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. "
2741                                  "Please add an auxiliary event in front of the load latency event.");
2742         default:
2743                 break;
2744         }
2745
2746         return scnprintf(msg, size,
2747         "The sys_perf_event_open() syscall returned with %d (%s) for event (%s).\n"
2748         "/bin/dmesg | grep -i perf may provide additional information.\n",
2749                          err, str_error_r(err, sbuf, sizeof(sbuf)), evsel__name(evsel));
2750 }
2751
2752 struct perf_env *evsel__env(struct evsel *evsel)
2753 {
2754         if (evsel && evsel->evlist)
2755                 return evsel->evlist->env;
2756         return &perf_env;
2757 }
2758
2759 static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist)
2760 {
2761         int cpu, thread;
2762
2763         for (cpu = 0; cpu < xyarray__max_x(evsel->core.fd); cpu++) {
2764                 for (thread = 0; thread < xyarray__max_y(evsel->core.fd);
2765                      thread++) {
2766                         int fd = FD(evsel, cpu, thread);
2767
2768                         if (perf_evlist__id_add_fd(&evlist->core, &evsel->core,
2769                                                    cpu, thread, fd) < 0)
2770                                 return -1;
2771                 }
2772         }
2773
2774         return 0;
2775 }
2776
2777 int evsel__store_ids(struct evsel *evsel, struct evlist *evlist)
2778 {
2779         struct perf_cpu_map *cpus = evsel->core.cpus;
2780         struct perf_thread_map *threads = evsel->core.threads;
2781
2782         if (perf_evsel__alloc_id(&evsel->core, cpus->nr, threads->nr))
2783                 return -ENOMEM;
2784
2785         return store_evsel_ids(evsel, evlist);
2786 }
2787
2788 void evsel__zero_per_pkg(struct evsel *evsel)
2789 {
2790         struct hashmap_entry *cur;
2791         size_t bkt;
2792
2793         if (evsel->per_pkg_mask) {
2794                 hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt)
2795                         free((char *)cur->key);
2796
2797                 hashmap__clear(evsel->per_pkg_mask);
2798         }
2799 }