tools/perf/util/bpf_counter.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /* Copyright (c) 2019 Facebook */
   4
   5 #include <assert.h>
   6 #include <limits.h>
   7 #include <unistd.h>
   8 #include <sys/file.h>
   9 #include <sys/time.h>
  10 #include <sys/resource.h>
  11 #include <linux/err.h>
  12 #include <linux/zalloc.h>
  13 #include <bpf/bpf.h>
  14 #include <bpf/btf.h>
  15 #include <bpf/libbpf.h>
  16 #include <api/fs/fs.h>
  17 #include <perf/bpf_perf.h>
  18
  19 #include "bpf_counter.h"
  20 #include "counts.h"
  21 #include "debug.h"
  22 #include "evsel.h"
  23 #include "evlist.h"
  24 #include "target.h"
  25 #include "cpumap.h"
  26 #include "thread_map.h"
  27
  28 #include "bpf_skel/bpf_prog_profiler.skel.h"
  29 #include "bpf_skel/bperf_u.h"
  30 #include "bpf_skel/bperf_leader.skel.h"
  31 #include "bpf_skel/bperf_follower.skel.h"
  32
  33 #define ATTR_MAP_SIZE 16
  34
  35 static inline void *u64_to_ptr(__u64 ptr)
  36 {
  37         return (void *)(unsigned long)ptr;
  38 }
  39
  40 static void set_max_rlimit(void)
  41 {
  42         struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY };
  43
  44         setrlimit(RLIMIT_MEMLOCK, &rinf);
  45 }
  46
  47 static struct bpf_counter *bpf_counter_alloc(void)
  48 {
  49         struct bpf_counter *counter;
  50
  51         counter = zalloc(sizeof(*counter));
  52         if (counter)
  53                 INIT_LIST_HEAD(&counter->list);
  54         return counter;
  55 }
  56
  57 static int bpf_program_profiler__destroy(struct evsel *evsel)
  58 {
  59         struct bpf_counter *counter, *tmp;
  60
  61         list_for_each_entry_safe(counter, tmp,
  62                                  &evsel->bpf_counter_list, list) {
  63                 list_del_init(&counter->list);
  64                 bpf_prog_profiler_bpf__destroy(counter->skel);
  65                 free(counter);
  66         }
  67         assert(list_empty(&evsel->bpf_counter_list));
  68
  69         return 0;
  70 }
  71
  72 static char *bpf_target_prog_name(int tgt_fd)
  73 {
  74         struct bpf_prog_info_linear *info_linear;
  75         struct bpf_func_info *func_info;
  76         const struct btf_type *t;
  77         char *name = NULL;
  78         struct btf *btf;
  79
  80         info_linear = bpf_program__get_prog_info_linear(
  81                 tgt_fd, 1UL << BPF_PROG_INFO_FUNC_INFO);
  82         if (IS_ERR_OR_NULL(info_linear)) {
  83                 pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd);
  84                 return NULL;
  85         }
  86
  87         if (info_linear->info.btf_id == 0 ||
  88             btf__get_from_id(info_linear->info.btf_id, &btf)) {
  89                 pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd);
  90                 goto out;
  91         }
  92
  93         func_info = u64_to_ptr(info_linear->info.func_info);
  94         t = btf__type_by_id(btf, func_info[0].type_id);
  95         if (!t) {
  96                 pr_debug("btf %d doesn't have type %d\n",
  97                          info_linear->info.btf_id, func_info[0].type_id);
  98                 goto out;
  99         }
 100         name = strdup(btf__name_by_offset(btf, t->name_off));
 101 out:
 102         free(info_linear);
 103         return name;
 104 }
 105
 106 static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id)
 107 {
 108         struct bpf_prog_profiler_bpf *skel;
 109         struct bpf_counter *counter;
 110         struct bpf_program *prog;
 111         char *prog_name;
 112         int prog_fd;
 113         int err;
 114
 115         prog_fd = bpf_prog_get_fd_by_id(prog_id);
 116         if (prog_fd < 0) {
 117                 pr_err("Failed to open fd for bpf prog %u\n", prog_id);
 118                 return -1;
 119         }
 120         counter = bpf_counter_alloc();
 121         if (!counter) {
 122                 close(prog_fd);
 123                 return -1;
 124         }
 125
 126         skel = bpf_prog_profiler_bpf__open();
 127         if (!skel) {
 128                 pr_err("Failed to open bpf skeleton\n");
 129                 goto err_out;
 130         }
 131
 132         skel->rodata->num_cpu = evsel__nr_cpus(evsel);
 133
 134         bpf_map__resize(skel->maps.events, evsel__nr_cpus(evsel));
 135         bpf_map__resize(skel->maps.fentry_readings, 1);
 136         bpf_map__resize(skel->maps.accum_readings, 1);
 137
 138         prog_name = bpf_target_prog_name(prog_fd);
 139         if (!prog_name) {
 140                 pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id);
 141                 goto err_out;
 142         }
 143
 144         bpf_object__for_each_program(prog, skel->obj) {
 145                 err = bpf_program__set_attach_target(prog, prog_fd, prog_name);
 146                 if (err) {
 147                         pr_err("bpf_program__set_attach_target failed.\n"
 148                                "Does bpf prog %u have BTF?\n", prog_id);
 149                         goto err_out;
 150                 }
 151         }
 152         set_max_rlimit();
 153         err = bpf_prog_profiler_bpf__load(skel);
 154         if (err) {
 155                 pr_err("bpf_prog_profiler_bpf__load failed\n");
 156                 goto err_out;
 157         }
 158
 159         assert(skel != NULL);
 160         counter->skel = skel;
 161         list_add(&counter->list, &evsel->bpf_counter_list);
 162         close(prog_fd);
 163         return 0;
 164 err_out:
 165         bpf_prog_profiler_bpf__destroy(skel);
 166         free(counter);
 167         close(prog_fd);
 168         return -1;
 169 }
 170
 171 static int bpf_program_profiler__load(struct evsel *evsel, struct target *target)
 172 {
 173         char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p;
 174         u32 prog_id;
 175         int ret;
 176
 177         bpf_str_ = bpf_str = strdup(target->bpf_str);
 178         if (!bpf_str)
 179                 return -1;
 180
 181         while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) {
 182                 prog_id = strtoul(tok, &p, 10);
 183                 if (prog_id == 0 || prog_id == UINT_MAX ||
 184                     (*p != '\0' && *p != ',')) {
 185                         pr_err("Failed to parse bpf prog ids %s\n",
 186                                target->bpf_str);
 187                         return -1;
 188                 }
 189
 190                 ret = bpf_program_profiler_load_one(evsel, prog_id);
 191                 if (ret) {
 192                         bpf_program_profiler__destroy(evsel);
 193                         free(bpf_str_);
 194                         return -1;
 195                 }
 196                 bpf_str = NULL;
 197         }
 198         free(bpf_str_);
 199         return 0;
 200 }
 201
 202 static int bpf_program_profiler__enable(struct evsel *evsel)
 203 {
 204         struct bpf_counter *counter;
 205         int ret;
 206
 207         list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
 208                 assert(counter->skel != NULL);
 209                 ret = bpf_prog_profiler_bpf__attach(counter->skel);
 210                 if (ret) {
 211                         bpf_program_profiler__destroy(evsel);
 212                         return ret;
 213                 }
 214         }
 215         return 0;
 216 }
 217
 218 static int bpf_program_profiler__disable(struct evsel *evsel)
 219 {
 220         struct bpf_counter *counter;
 221
 222         list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
 223                 assert(counter->skel != NULL);
 224                 bpf_prog_profiler_bpf__detach(counter->skel);
 225         }
 226         return 0;
 227 }
 228
 229 static int bpf_program_profiler__read(struct evsel *evsel)
 230 {
 231         // perf_cpu_map uses /sys/devices/system/cpu/online
 232         int num_cpu = evsel__nr_cpus(evsel);
 233         // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible
 234         // Sometimes possible > online, like on a Ryzen 3900X that has 24
 235         // threads but its possible showed 0-31 -acme
 236         int num_cpu_bpf = libbpf_num_possible_cpus();
 237         struct bpf_perf_event_value values[num_cpu_bpf];
 238         struct bpf_counter *counter;
 239         int reading_map_fd;
 240         __u32 key = 0;
 241         int err, cpu;
 242
 243         if (list_empty(&evsel->bpf_counter_list))
 244                 return -EAGAIN;
 245
 246         for (cpu = 0; cpu < num_cpu; cpu++) {
 247                 perf_counts(evsel->counts, cpu, 0)->val = 0;
 248                 perf_counts(evsel->counts, cpu, 0)->ena = 0;
 249                 perf_counts(evsel->counts, cpu, 0)->run = 0;
 250         }
 251         list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
 252                 struct bpf_prog_profiler_bpf *skel = counter->skel;
 253
 254                 assert(skel != NULL);
 255                 reading_map_fd = bpf_map__fd(skel->maps.accum_readings);
 256
 257                 err = bpf_map_lookup_elem(reading_map_fd, &key, values);
 258                 if (err) {
 259                         pr_err("failed to read value\n");
 260                         return err;
 261                 }
 262
 263                 for (cpu = 0; cpu < num_cpu; cpu++) {
 264                         perf_counts(evsel->counts, cpu, 0)->val += values[cpu].counter;
 265                         perf_counts(evsel->counts, cpu, 0)->ena += values[cpu].enabled;
 266                         perf_counts(evsel->counts, cpu, 0)->run += values[cpu].running;
 267                 }
 268         }
 269         return 0;
 270 }
 271
 272 static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu,
 273                                             int fd)
 274 {
 275         struct bpf_prog_profiler_bpf *skel;
 276         struct bpf_counter *counter;
 277         int ret;
 278
 279         list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
 280                 skel = counter->skel;
 281                 assert(skel != NULL);
 282
 283                 ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events),
 284                                           &cpu, &fd, BPF_ANY);
 285                 if (ret)
 286                         return ret;
 287         }
 288         return 0;
 289 }
 290
 291 struct bpf_counter_ops bpf_program_profiler_ops = {
 292         .load       = bpf_program_profiler__load,
 293         .enable     = bpf_program_profiler__enable,
 294         .disable    = bpf_program_profiler__disable,
 295         .read       = bpf_program_profiler__read,
 296         .destroy    = bpf_program_profiler__destroy,
 297         .install_pe = bpf_program_profiler__install_pe,
 298 };
 299
 300 static __u32 bpf_link_get_id(int fd)
 301 {
 302         struct bpf_link_info link_info = {0};
 303         __u32 link_info_len = sizeof(link_info);
 304
 305         bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len);
 306         return link_info.id;
 307 }
 308
 309 static __u32 bpf_link_get_prog_id(int fd)
 310 {
 311         struct bpf_link_info link_info = {0};
 312         __u32 link_info_len = sizeof(link_info);
 313
 314         bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len);
 315         return link_info.prog_id;
 316 }
 317
 318 static __u32 bpf_map_get_id(int fd)
 319 {
 320         struct bpf_map_info map_info = {0};
 321         __u32 map_info_len = sizeof(map_info);
 322
 323         bpf_obj_get_info_by_fd(fd, &map_info, &map_info_len);
 324         return map_info.id;
 325 }
 326
 327 static bool bperf_attr_map_compatible(int attr_map_fd)
 328 {
 329         struct bpf_map_info map_info = {0};
 330         __u32 map_info_len = sizeof(map_info);
 331         int err;
 332
 333         err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len);
 334
 335         if (err)
 336                 return false;
 337         return (map_info.key_size == sizeof(struct perf_event_attr)) &&
 338                 (map_info.value_size == sizeof(struct perf_event_attr_map_entry));
 339 }
 340
 341 static int bperf_lock_attr_map(struct target *target)
 342 {
 343         char path[PATH_MAX];
 344         int map_fd, err;
 345
 346         if (target->attr_map) {
 347                 scnprintf(path, PATH_MAX, "%s", target->attr_map);
 348         } else {
 349                 scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(),
 350                           BPF_PERF_DEFAULT_ATTR_MAP_PATH);
 351         }
 352
 353         if (access(path, F_OK)) {
 354                 map_fd = bpf_create_map(BPF_MAP_TYPE_HASH,
 355                                         sizeof(struct perf_event_attr),
 356                                         sizeof(struct perf_event_attr_map_entry),
 357                                         ATTR_MAP_SIZE, 0);
 358                 if (map_fd < 0)
 359                         return -1;
 360
 361                 err = bpf_obj_pin(map_fd, path);
 362                 if (err) {
 363                         /* someone pinned the map in parallel? */
 364                         close(map_fd);
 365                         map_fd = bpf_obj_get(path);
 366                         if (map_fd < 0)
 367                                 return -1;
 368                 }
 369         } else {
 370                 map_fd = bpf_obj_get(path);
 371                 if (map_fd < 0)
 372                         return -1;
 373         }
 374
 375         if (!bperf_attr_map_compatible(map_fd)) {
 376                 close(map_fd);
 377                 return -1;
 378
 379         }
 380         err = flock(map_fd, LOCK_EX);
 381         if (err) {
 382                 close(map_fd);
 383                 return -1;
 384         }
 385         return map_fd;
 386 }
 387
 388 /* trigger the leader program on a cpu */
 389 static int bperf_trigger_reading(int prog_fd, int cpu)
 390 {
 391         DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts,
 392                             .ctx_in = NULL,
 393                             .ctx_size_in = 0,
 394                             .flags = BPF_F_TEST_RUN_ON_CPU,
 395                             .cpu = cpu,
 396                             .retval = 0,
 397                 );
 398
 399         return bpf_prog_test_run_opts(prog_fd, &opts);
 400 }
 401
 402 static int bperf_check_target(struct evsel *evsel,
 403                               struct target *target,
 404                               enum bperf_filter_type *filter_type,
 405                               __u32 *filter_entry_cnt)
 406 {
 407         if (evsel->leader->core.nr_members > 1) {
 408                 pr_err("bpf managed perf events do not yet support groups.\n");
 409                 return -1;
 410         }
 411
 412         /* determine filter type based on target */
 413         if (target->system_wide) {
 414                 *filter_type = BPERF_FILTER_GLOBAL;
 415                 *filter_entry_cnt = 1;
 416         } else if (target->cpu_list) {
 417                 *filter_type = BPERF_FILTER_CPU;
 418                 *filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel));
 419         } else if (target->tid) {
 420                 *filter_type = BPERF_FILTER_PID;
 421                 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
 422         } else if (target->pid || evsel->evlist->workload.pid != -1) {
 423                 *filter_type = BPERF_FILTER_TGID;
 424                 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
 425         } else {
 426                 pr_err("bpf managed perf events do not yet support these targets.\n");
 427                 return -1;
 428         }
 429
 430         return 0;
 431 }
 432
 433 static  struct perf_cpu_map *all_cpu_map;
 434
 435 static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd,
 436                                        struct perf_event_attr_map_entry *entry)
 437 {
 438         struct bperf_leader_bpf *skel = bperf_leader_bpf__open();
 439         int link_fd, diff_map_fd, err;
 440         struct bpf_link *link = NULL;
 441
 442         if (!skel) {
 443                 pr_err("Failed to open leader skeleton\n");
 444                 return -1;
 445         }
 446
 447         bpf_map__resize(skel->maps.events, libbpf_num_possible_cpus());
 448         err = bperf_leader_bpf__load(skel);
 449         if (err) {
 450                 pr_err("Failed to load leader skeleton\n");
 451                 goto out;
 452         }
 453
 454         link = bpf_program__attach(skel->progs.on_switch);
 455         if (IS_ERR(link)) {
 456                 pr_err("Failed to attach leader program\n");
 457                 err = PTR_ERR(link);
 458                 goto out;
 459         }
 460
 461         link_fd = bpf_link__fd(link);
 462         diff_map_fd = bpf_map__fd(skel->maps.diff_readings);
 463         entry->link_id = bpf_link_get_id(link_fd);
 464         entry->diff_map_id = bpf_map_get_id(diff_map_fd);
 465         err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY);
 466         assert(err == 0);
 467
 468         evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id);
 469         assert(evsel->bperf_leader_link_fd >= 0);
 470
 471         /*
 472          * save leader_skel for install_pe, which is called within
 473          * following evsel__open_per_cpu call
 474          */
 475         evsel->leader_skel = skel;
 476         evsel__open_per_cpu(evsel, all_cpu_map, -1);
 477
 478 out:
 479         bperf_leader_bpf__destroy(skel);
 480         bpf_link__destroy(link);
 481         return err;
 482 }
 483
 484 static int bperf__load(struct evsel *evsel, struct target *target)
 485 {
 486         struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff};
 487         int attr_map_fd, diff_map_fd = -1, err;
 488         enum bperf_filter_type filter_type;
 489         __u32 filter_entry_cnt, i;
 490
 491         if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt))
 492                 return -1;
 493
 494         if (!all_cpu_map) {
 495                 all_cpu_map = perf_cpu_map__new(NULL);
 496                 if (!all_cpu_map)
 497                         return -1;
 498         }
 499
 500         evsel->bperf_leader_prog_fd = -1;
 501         evsel->bperf_leader_link_fd = -1;
 502
 503         /*
 504          * Step 1: hold a fd on the leader program and the bpf_link, if
 505          * the program is not already gone, reload the program.
 506          * Use flock() to ensure exclusive access to the perf_event_attr
 507          * map.
 508          */
 509         attr_map_fd = bperf_lock_attr_map(target);
 510         if (attr_map_fd < 0) {
 511                 pr_err("Failed to lock perf_event_attr map\n");
 512                 return -1;
 513         }
 514
 515         err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry);
 516         if (err) {
 517                 err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY);
 518                 if (err)
 519                         goto out;
 520         }
 521
 522         evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id);
 523         if (evsel->bperf_leader_link_fd < 0 &&
 524             bperf_reload_leader_program(evsel, attr_map_fd, &entry)) {
 525                 err = -1;
 526                 goto out;
 527         }
 528         /*
 529          * The bpf_link holds reference to the leader program, and the
 530          * leader program holds reference to the maps. Therefore, if
 531          * link_id is valid, diff_map_id should also be valid.
 532          */
 533         evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id(
 534                 bpf_link_get_prog_id(evsel->bperf_leader_link_fd));
 535         assert(evsel->bperf_leader_prog_fd >= 0);
 536
 537         diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id);
 538         assert(diff_map_fd >= 0);
 539
 540         /*
 541          * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check
 542          * whether the kernel support it
 543          */
 544         err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0);
 545         if (err) {
 546                 pr_err("The kernel does not support test_run for raw_tp BPF programs.\n"
 547                        "Therefore, --use-bpf might show inaccurate readings\n");
 548                 goto out;
 549         }
 550
 551         /* Step 2: load the follower skeleton */
 552         evsel->follower_skel = bperf_follower_bpf__open();
 553         if (!evsel->follower_skel) {
 554                 err = -1;
 555                 pr_err("Failed to open follower skeleton\n");
 556                 goto out;
 557         }
 558
 559         /* attach fexit program to the leader program */
 560         bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX,
 561                                        evsel->bperf_leader_prog_fd, "on_switch");
 562
 563         /* connect to leader diff_reading map */
 564         bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd);
 565
 566         /* set up reading map */
 567         bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings,
 568                                  filter_entry_cnt);
 569         /* set up follower filter based on target */
 570         bpf_map__set_max_entries(evsel->follower_skel->maps.filter,
 571                                  filter_entry_cnt);
 572         err = bperf_follower_bpf__load(evsel->follower_skel);
 573         if (err) {
 574                 pr_err("Failed to load follower skeleton\n");
 575                 bperf_follower_bpf__destroy(evsel->follower_skel);
 576                 evsel->follower_skel = NULL;
 577                 goto out;
 578         }
 579
 580         for (i = 0; i < filter_entry_cnt; i++) {
 581                 int filter_map_fd;
 582                 __u32 key;
 583
 584                 if (filter_type == BPERF_FILTER_PID ||
 585                     filter_type == BPERF_FILTER_TGID)
 586                         key = evsel->core.threads->map[i].pid;
 587                 else if (filter_type == BPERF_FILTER_CPU)
 588                         key = evsel->core.cpus->map[i];
 589                 else
 590                         break;
 591
 592                 filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter);
 593                 bpf_map_update_elem(filter_map_fd, &key, &i, BPF_ANY);
 594         }
 595
 596         evsel->follower_skel->bss->type = filter_type;
 597
 598         err = bperf_follower_bpf__attach(evsel->follower_skel);
 599
 600 out:
 601         if (err && evsel->bperf_leader_link_fd >= 0)
 602                 close(evsel->bperf_leader_link_fd);
 603         if (err && evsel->bperf_leader_prog_fd >= 0)
 604                 close(evsel->bperf_leader_prog_fd);
 605         if (diff_map_fd >= 0)
 606                 close(diff_map_fd);
 607
 608         flock(attr_map_fd, LOCK_UN);
 609         close(attr_map_fd);
 610
 611         return err;
 612 }
 613
 614 static int bperf__install_pe(struct evsel *evsel, int cpu, int fd)
 615 {
 616         struct bperf_leader_bpf *skel = evsel->leader_skel;
 617
 618         return bpf_map_update_elem(bpf_map__fd(skel->maps.events),
 619                                    &cpu, &fd, BPF_ANY);
 620 }
 621
 622 /*
 623  * trigger the leader prog on each cpu, so the accum_reading map could get
 624  * the latest readings.
 625  */
 626 static int bperf_sync_counters(struct evsel *evsel)
 627 {
 628         int num_cpu, i, cpu;
 629
 630         num_cpu = all_cpu_map->nr;
 631         for (i = 0; i < num_cpu; i++) {
 632                 cpu = all_cpu_map->map[i];
 633                 bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu);
 634         }
 635         return 0;
 636 }
 637
 638 static int bperf__enable(struct evsel *evsel)
 639 {
 640         evsel->follower_skel->bss->enabled = 1;
 641         return 0;
 642 }
 643
 644 static int bperf__disable(struct evsel *evsel)
 645 {
 646         evsel->follower_skel->bss->enabled = 0;
 647         return 0;
 648 }
 649
 650 static int bperf__read(struct evsel *evsel)
 651 {
 652         struct bperf_follower_bpf *skel = evsel->follower_skel;
 653         __u32 num_cpu_bpf = cpu__max_cpu();
 654         struct bpf_perf_event_value values[num_cpu_bpf];
 655         int reading_map_fd, err = 0;
 656         __u32 i, j, num_cpu;
 657
 658         bperf_sync_counters(evsel);
 659         reading_map_fd = bpf_map__fd(skel->maps.accum_readings);
 660
 661         for (i = 0; i < bpf_map__max_entries(skel->maps.accum_readings); i++) {
 662                 __u32 cpu;
 663
 664                 err = bpf_map_lookup_elem(reading_map_fd, &i, values);
 665                 if (err)
 666                         goto out;
 667                 switch (evsel->follower_skel->bss->type) {
 668                 case BPERF_FILTER_GLOBAL:
 669                         assert(i == 0);
 670
 671                         num_cpu = all_cpu_map->nr;
 672                         for (j = 0; j < num_cpu; j++) {
 673                                 cpu = all_cpu_map->map[j];
 674                                 perf_counts(evsel->counts, cpu, 0)->val = values[cpu].counter;
 675                                 perf_counts(evsel->counts, cpu, 0)->ena = values[cpu].enabled;
 676                                 perf_counts(evsel->counts, cpu, 0)->run = values[cpu].running;
 677                         }
 678                         break;
 679                 case BPERF_FILTER_CPU:
 680                         cpu = evsel->core.cpus->map[i];
 681                         perf_counts(evsel->counts, i, 0)->val = values[cpu].counter;
 682                         perf_counts(evsel->counts, i, 0)->ena = values[cpu].enabled;
 683                         perf_counts(evsel->counts, i, 0)->run = values[cpu].running;
 684                         break;
 685                 case BPERF_FILTER_PID:
 686                 case BPERF_FILTER_TGID:
 687                         perf_counts(evsel->counts, 0, i)->val = 0;
 688                         perf_counts(evsel->counts, 0, i)->ena = 0;
 689                         perf_counts(evsel->counts, 0, i)->run = 0;
 690
 691                         for (cpu = 0; cpu < num_cpu_bpf; cpu++) {
 692                                 perf_counts(evsel->counts, 0, i)->val += values[cpu].counter;
 693                                 perf_counts(evsel->counts, 0, i)->ena += values[cpu].enabled;
 694                                 perf_counts(evsel->counts, 0, i)->run += values[cpu].running;
 695                         }
 696                         break;
 697                 default:
 698                         break;
 699                 }
 700         }
 701 out:
 702         return err;
 703 }
 704
 705 static int bperf__destroy(struct evsel *evsel)
 706 {
 707         bperf_follower_bpf__destroy(evsel->follower_skel);
 708         close(evsel->bperf_leader_prog_fd);
 709         close(evsel->bperf_leader_link_fd);
 710         return 0;
 711 }
 712
 713 /*
 714  * bperf: share hardware PMCs with BPF
 715  *
 716  * perf uses performance monitoring counters (PMC) to monitor system
 717  * performance. The PMCs are limited hardware resources. For example,
 718  * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu.
 719  *
 720  * Modern data center systems use these PMCs in many different ways:
 721  * system level monitoring, (maybe nested) container level monitoring, per
 722  * process monitoring, profiling (in sample mode), etc. In some cases,
 723  * there are more active perf_events than available hardware PMCs. To allow
 724  * all perf_events to have a chance to run, it is necessary to do expensive
 725  * time multiplexing of events.
 726  *
 727  * On the other hand, many monitoring tools count the common metrics
 728  * (cycles, instructions). It is a waste to have multiple tools create
 729  * multiple perf_events of "cycles" and occupy multiple PMCs.
 730  *
 731  * bperf tries to reduce such wastes by allowing multiple perf_events of
 732  * "cycles" or "instructions" (at different scopes) to share PMUs. Instead
 733  * of having each perf-stat session to read its own perf_events, bperf uses
 734  * BPF programs to read the perf_events and aggregate readings to BPF maps.
 735  * Then, the perf-stat session(s) reads the values from these BPF maps.
 736  *
 737  *                                ||
 738  *       shared progs and maps <- || -> per session progs and maps
 739  *                                ||
 740  *   ---------------              ||
 741  *   | perf_events |              ||
 742  *   ---------------       fexit  ||      -----------------
 743  *          |             --------||----> | follower prog |
 744  *       --------------- /        || ---  -----------------
 745  * cs -> | leader prog |/         ||/        |         |
 746  *   --> ---------------         /||  --------------  ------------------
 747  *  /       |         |         / ||  | filter map |  | accum_readings |
 748  * /  ------------  ------------  ||  --------------  ------------------
 749  * |  | prev map |  | diff map |  ||                        |
 750  * |  ------------  ------------  ||                        |
 751  *  \                             ||                        |
 752  * = \ ==================================================== | ============
 753  *    \                                                    /   user space
 754  *     \                                                  /
 755  *      \                                                /
 756  *    BPF_PROG_TEST_RUN                    BPF_MAP_LOOKUP_ELEM
 757  *        \                                            /
 758  *         \                                          /
 759  *          \------  perf-stat ----------------------/
 760  *
 761  * The figure above shows the architecture of bperf. Note that the figure
 762  * is divided into 3 regions: shared progs and maps (top left), per session
 763  * progs and maps (top right), and user space (bottom).
 764  *
 765  * The leader prog is triggered on each context switch (cs). The leader
 766  * prog reads perf_events and stores the difference (current_reading -
 767  * previous_reading) to the diff map. For the same metric, e.g. "cycles",
 768  * multiple perf-stat sessions share the same leader prog.
 769  *
 770  * Each perf-stat session creates a follower prog as fexit program to the
 771  * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38)
 772  * follower progs to the same leader prog. The follower prog checks current
 773  * task and processor ID to decide whether to add the value from the diff
 774  * map to its accumulated reading map (accum_readings).
 775  *
 776  * Finally, perf-stat user space reads the value from accum_reading map.
 777  *
 778  * Besides context switch, it is also necessary to trigger the leader prog
 779  * before perf-stat reads the value. Otherwise, the accum_reading map may
 780  * not have the latest reading from the perf_events. This is achieved by
 781  * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU.
 782  *
 783  * Comment before the definition of struct perf_event_attr_map_entry
 784  * describes how different sessions of perf-stat share information about
 785  * the leader prog.
 786  */
 787
 788 struct bpf_counter_ops bperf_ops = {
 789         .load       = bperf__load,
 790         .enable     = bperf__enable,
 791         .disable    = bperf__disable,
 792         .read       = bperf__read,
 793         .install_pe = bperf__install_pe,
 794         .destroy    = bperf__destroy,
 795 };
 796
 797 static inline bool bpf_counter_skip(struct evsel *evsel)
 798 {
 799         return list_empty(&evsel->bpf_counter_list) &&
 800                 evsel->follower_skel == NULL;
 801 }
 802
 803 int bpf_counter__install_pe(struct evsel *evsel, int cpu, int fd)
 804 {
 805         if (bpf_counter_skip(evsel))
 806                 return 0;
 807         return evsel->bpf_counter_ops->install_pe(evsel, cpu, fd);
 808 }
 809
 810 int bpf_counter__load(struct evsel *evsel, struct target *target)
 811 {
 812         if (target->bpf_str)
 813                 evsel->bpf_counter_ops = &bpf_program_profiler_ops;
 814         else if (target->use_bpf || evsel->bpf_counter ||
 815                  evsel__match_bpf_counter_events(evsel->name))
 816                 evsel->bpf_counter_ops = &bperf_ops;
 817
 818         if (evsel->bpf_counter_ops)
 819                 return evsel->bpf_counter_ops->load(evsel, target);
 820         return 0;
 821 }
 822
 823 int bpf_counter__enable(struct evsel *evsel)
 824 {
 825         if (bpf_counter_skip(evsel))
 826                 return 0;
 827         return evsel->bpf_counter_ops->enable(evsel);
 828 }
 829
 830 int bpf_counter__disable(struct evsel *evsel)
 831 {
 832         if (bpf_counter_skip(evsel))
 833                 return 0;
 834         return evsel->bpf_counter_ops->disable(evsel);
 835 }
 836
 837 int bpf_counter__read(struct evsel *evsel)
 838 {
 839         if (bpf_counter_skip(evsel))
 840                 return -EAGAIN;
 841         return evsel->bpf_counter_ops->read(evsel);
 842 }
 843
 844 void bpf_counter__destroy(struct evsel *evsel)
 845 {
 846         if (bpf_counter_skip(evsel))
 847                 return;
 848         evsel->bpf_counter_ops->destroy(evsel);
 849         evsel->bpf_counter_ops = NULL;
 850 }