tools/perf/util/bpf_counter_cgroup.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /* Copyright (c) 2021 Facebook */
   4 /* Copyright (c) 2021 Google */
   5
   6 #include <assert.h>
   7 #include <limits.h>
   8 #include <unistd.h>
   9 #include <sys/file.h>
  10 #include <sys/time.h>
  11 #include <sys/resource.h>
  12 #include <linux/err.h>
  13 #include <linux/zalloc.h>
  14 #include <linux/perf_event.h>
  15 #include <api/fs/fs.h>
  16 #include <perf/bpf_perf.h>
  17
  18 #include "affinity.h"
  19 #include "bpf_counter.h"
  20 #include "cgroup.h"
  21 #include "counts.h"
  22 #include "debug.h"
  23 #include "evsel.h"
  24 #include "evlist.h"
  25 #include "target.h"
  26 #include "cpumap.h"
  27 #include "thread_map.h"
  28
  29 #include "bpf_skel/bperf_cgroup.skel.h"
  30
  31 static struct perf_event_attr cgrp_switch_attr = {
  32         .type = PERF_TYPE_SOFTWARE,
  33         .config = PERF_COUNT_SW_CGROUP_SWITCHES,
  34         .size = sizeof(cgrp_switch_attr),
  35         .sample_period = 1,
  36         .disabled = 1,
  37 };
  38
  39 static struct evsel *cgrp_switch;
  40 static struct bperf_cgroup_bpf *skel;
  41
  42 #define FD(evt, cpu) (*(int *)xyarray__entry(evt->core.fd, cpu, 0))
  43
  44 static int bperf_load_program(struct evlist *evlist)
  45 {
  46         struct bpf_link *link;
  47         struct evsel *evsel;
  48         struct cgroup *cgrp, *leader_cgrp;
  49         __u32 i, cpu;
  50         __u32 nr_cpus = evlist->core.all_cpus->nr;
  51         int total_cpus = cpu__max_cpu().cpu;
  52         int map_size, map_fd;
  53         int prog_fd, err;
  54
  55         skel = bperf_cgroup_bpf__open();
  56         if (!skel) {
  57                 pr_err("Failed to open cgroup skeleton\n");
  58                 return -1;
  59         }
  60
  61         skel->rodata->num_cpus = total_cpus;
  62         skel->rodata->num_events = evlist->core.nr_entries / nr_cgroups;
  63
  64         BUG_ON(evlist->core.nr_entries % nr_cgroups != 0);
  65
  66         /* we need one copy of events per cpu for reading */
  67         map_size = total_cpus * evlist->core.nr_entries / nr_cgroups;
  68         bpf_map__set_max_entries(skel->maps.events, map_size);
  69         bpf_map__set_max_entries(skel->maps.cgrp_idx, nr_cgroups);
  70         /* previous result is saved in a per-cpu array */
  71         map_size = evlist->core.nr_entries / nr_cgroups;
  72         bpf_map__set_max_entries(skel->maps.prev_readings, map_size);
  73         /* cgroup result needs all events (per-cpu) */
  74         map_size = evlist->core.nr_entries;
  75         bpf_map__set_max_entries(skel->maps.cgrp_readings, map_size);
  76
  77         set_max_rlimit();
  78
  79         err = bperf_cgroup_bpf__load(skel);
  80         if (err) {
  81                 pr_err("Failed to load cgroup skeleton\n");
  82                 goto out;
  83         }
  84
  85         if (cgroup_is_v2("perf_event") > 0)
  86                 skel->bss->use_cgroup_v2 = 1;
  87
  88         err = -1;
  89
  90         cgrp_switch = evsel__new(&cgrp_switch_attr);
  91         if (evsel__open_per_cpu(cgrp_switch, evlist->core.all_cpus, -1) < 0) {
  92                 pr_err("Failed to open cgroup switches event\n");
  93                 goto out;
  94         }
  95
  96         for (i = 0; i < nr_cpus; i++) {
  97                 link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch,
  98                                                       FD(cgrp_switch, i));
  99                 if (IS_ERR(link)) {
 100                         pr_err("Failed to attach cgroup program\n");
 101                         err = PTR_ERR(link);
 102                         goto out;
 103                 }
 104         }
 105
 106         /*
 107          * Update cgrp_idx map from cgroup-id to event index.
 108          */
 109         cgrp = NULL;
 110         i = 0;
 111
 112         evlist__for_each_entry(evlist, evsel) {
 113                 if (cgrp == NULL || evsel->cgrp == leader_cgrp) {
 114                         leader_cgrp = evsel->cgrp;
 115                         evsel->cgrp = NULL;
 116
 117                         /* open single copy of the events w/o cgroup */
 118                         err = evsel__open_per_cpu(evsel, evlist->core.all_cpus, -1);
 119                         if (err) {
 120                                 pr_err("Failed to open first cgroup events\n");
 121                                 goto out;
 122                         }
 123
 124                         map_fd = bpf_map__fd(skel->maps.events);
 125                         for (cpu = 0; cpu < nr_cpus; cpu++) {
 126                                 int fd = FD(evsel, cpu);
 127                                 __u32 idx = evsel->core.idx * total_cpus +
 128                                         evlist->core.all_cpus->map[cpu].cpu;
 129
 130                                 err = bpf_map_update_elem(map_fd, &idx, &fd,
 131                                                           BPF_ANY);
 132                                 if (err < 0) {
 133                                         pr_err("Failed to update perf_event fd\n");
 134                                         goto out;
 135                                 }
 136                         }
 137
 138                         evsel->cgrp = leader_cgrp;
 139                 }
 140                 evsel->supported = true;
 141
 142                 if (evsel->cgrp == cgrp)
 143                         continue;
 144
 145                 cgrp = evsel->cgrp;
 146
 147                 if (read_cgroup_id(cgrp) < 0) {
 148                         pr_err("Failed to get cgroup id\n");
 149                         err = -1;
 150                         goto out;
 151                 }
 152
 153                 map_fd = bpf_map__fd(skel->maps.cgrp_idx);
 154                 err = bpf_map_update_elem(map_fd, &cgrp->id, &i, BPF_ANY);
 155                 if (err < 0) {
 156                         pr_err("Failed to update cgroup index map\n");
 157                         goto out;
 158                 }
 159
 160                 i++;
 161         }
 162
 163         /*
 164          * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check
 165          * whether the kernel support it
 166          */
 167         prog_fd = bpf_program__fd(skel->progs.trigger_read);
 168         err = bperf_trigger_reading(prog_fd, 0);
 169         if (err) {
 170                 pr_warning("The kernel does not support test_run for raw_tp BPF programs.\n"
 171                            "Therefore, --for-each-cgroup might show inaccurate readings\n");
 172                 err = 0;
 173         }
 174
 175 out:
 176         return err;
 177 }
 178
 179 static int bperf_cgrp__load(struct evsel *evsel,
 180                             struct target *target __maybe_unused)
 181 {
 182         static bool bperf_loaded = false;
 183
 184         evsel->bperf_leader_prog_fd = -1;
 185         evsel->bperf_leader_link_fd = -1;
 186
 187         if (!bperf_loaded && bperf_load_program(evsel->evlist))
 188                 return -1;
 189
 190         bperf_loaded = true;
 191         /* just to bypass bpf_counter_skip() */
 192         evsel->follower_skel = (struct bperf_follower_bpf *)skel;
 193
 194         return 0;
 195 }
 196
 197 static int bperf_cgrp__install_pe(struct evsel *evsel __maybe_unused,
 198                                   int cpu __maybe_unused, int fd __maybe_unused)
 199 {
 200         /* nothing to do */
 201         return 0;
 202 }
 203
 204 /*
 205  * trigger the leader prog on each cpu, so the cgrp_reading map could get
 206  * the latest results.
 207  */
 208 static int bperf_cgrp__sync_counters(struct evlist *evlist)
 209 {
 210         int i, cpu;
 211         int nr_cpus = evlist->core.all_cpus->nr;
 212         int prog_fd = bpf_program__fd(skel->progs.trigger_read);
 213
 214         for (i = 0; i < nr_cpus; i++) {
 215                 cpu = evlist->core.all_cpus->map[i].cpu;
 216                 bperf_trigger_reading(prog_fd, cpu);
 217         }
 218
 219         return 0;
 220 }
 221
 222 static int bperf_cgrp__enable(struct evsel *evsel)
 223 {
 224         if (evsel->core.idx)
 225                 return 0;
 226
 227         bperf_cgrp__sync_counters(evsel->evlist);
 228
 229         skel->bss->enabled = 1;
 230         return 0;
 231 }
 232
 233 static int bperf_cgrp__disable(struct evsel *evsel)
 234 {
 235         if (evsel->core.idx)
 236                 return 0;
 237
 238         bperf_cgrp__sync_counters(evsel->evlist);
 239
 240         skel->bss->enabled = 0;
 241         return 0;
 242 }
 243
 244 static int bperf_cgrp__read(struct evsel *evsel)
 245 {
 246         struct evlist *evlist = evsel->evlist;
 247         int i, cpu, nr_cpus = evlist->core.all_cpus->nr;
 248         int total_cpus = cpu__max_cpu().cpu;
 249         struct perf_counts_values *counts;
 250         struct bpf_perf_event_value *values;
 251         int reading_map_fd, err = 0;
 252         __u32 idx;
 253
 254         if (evsel->core.idx)
 255                 return 0;
 256
 257         bperf_cgrp__sync_counters(evsel->evlist);
 258
 259         values = calloc(total_cpus, sizeof(*values));
 260         if (values == NULL)
 261                 return -ENOMEM;
 262
 263         reading_map_fd = bpf_map__fd(skel->maps.cgrp_readings);
 264
 265         evlist__for_each_entry(evlist, evsel) {
 266                 idx = evsel->core.idx;
 267                 err = bpf_map_lookup_elem(reading_map_fd, &idx, values);
 268                 if (err) {
 269                         pr_err("bpf map lookup falied: idx=%u, event=%s, cgrp=%s\n",
 270                                idx, evsel__name(evsel), evsel->cgrp->name);
 271                         goto out;
 272                 }
 273
 274                 for (i = 0; i < nr_cpus; i++) {
 275                         cpu = evlist->core.all_cpus->map[i].cpu;
 276
 277                         counts = perf_counts(evsel->counts, i, 0);
 278                         counts->val = values[cpu].counter;
 279                         counts->ena = values[cpu].enabled;
 280                         counts->run = values[cpu].running;
 281                 }
 282         }
 283
 284 out:
 285         free(values);
 286         return err;
 287 }
 288
 289 static int bperf_cgrp__destroy(struct evsel *evsel)
 290 {
 291         if (evsel->core.idx)
 292                 return 0;
 293
 294         bperf_cgroup_bpf__destroy(skel);
 295         evsel__delete(cgrp_switch);  // it'll destroy on_switch progs too
 296
 297         return 0;
 298 }
 299
 300 struct bpf_counter_ops bperf_cgrp_ops = {
 301         .load       = bperf_cgrp__load,
 302         .enable     = bperf_cgrp__enable,
 303         .disable    = bperf_cgrp__disable,
 304         .read       = bperf_cgrp__read,
 305         .install_pe = bperf_cgrp__install_pe,
 306         .destroy    = bperf_cgrp__destroy,
 307 };