tools/perf/util/bpf-loader.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * bpf-loader.c
   4  *
   5  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   6  * Copyright (C) 2015 Huawei Inc.
   7  */
   8
   9 #include <linux/bpf.h>
  10 #include <bpf/libbpf.h>
  11 #include <bpf/bpf.h>
  12 #include <linux/err.h>
  13 #include <linux/kernel.h>
  14 #include <linux/string.h>
  15 #include <linux/zalloc.h>
  16 #include <errno.h>
  17 #include <stdlib.h>
  18 #include "debug.h"
  19 #include "evlist.h"
  20 #include "bpf-loader.h"
  21 #include "bpf-prologue.h"
  22 #include "probe-event.h"
  23 #include "probe-finder.h" // for MAX_PROBES
  24 #include "parse-events.h"
  25 #include "strfilter.h"
  26 #include "util.h"
  27 #include "llvm-utils.h"
  28 #include "c++/clang-c.h"
  29
  30 #include <internal/xyarray.h>
  31
  32 /* temporarily disable libbpf deprecation warnings */
  33 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
  34
  35 static int libbpf_perf_print(enum libbpf_print_level level __attribute__((unused)),
  36                               const char *fmt, va_list args)
  37 {
  38         return veprintf(1, verbose, pr_fmt(fmt), args);
  39 }
  40
  41 struct bpf_prog_priv {
  42         bool is_tp;
  43         char *sys_name;
  44         char *evt_name;
  45         struct perf_probe_event pev;
  46         bool need_prologue;
  47         struct bpf_insn *insns_buf;
  48         int nr_types;
  49         int *type_mapping;
  50 };
  51
  52 static bool libbpf_initialized;
  53
  54 struct bpf_object *
  55 bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
  56 {
  57         LIBBPF_OPTS(bpf_object_open_opts, opts, .object_name = name);
  58         struct bpf_object *obj;
  59
  60         if (!libbpf_initialized) {
  61                 libbpf_set_print(libbpf_perf_print);
  62                 libbpf_initialized = true;
  63         }
  64
  65         obj = bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
  66         if (IS_ERR_OR_NULL(obj)) {
  67                 pr_debug("bpf: failed to load buffer\n");
  68                 return ERR_PTR(-EINVAL);
  69         }
  70
  71         return obj;
  72 }
  73
  74 struct bpf_object *bpf__prepare_load(const char *filename, bool source)
  75 {
  76         LIBBPF_OPTS(bpf_object_open_opts, opts, .object_name = filename);
  77         struct bpf_object *obj;
  78
  79         if (!libbpf_initialized) {
  80                 libbpf_set_print(libbpf_perf_print);
  81                 libbpf_initialized = true;
  82         }
  83
  84         if (source) {
  85                 int err;
  86                 void *obj_buf;
  87                 size_t obj_buf_sz;
  88
  89                 perf_clang__init();
  90                 err = perf_clang__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  91                 perf_clang__cleanup();
  92                 if (err) {
  93                         pr_debug("bpf: builtin compilation failed: %d, try external compiler\n", err);
  94                         err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  95                         if (err)
  96                                 return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
  97                 } else
  98                         pr_debug("bpf: successful builtin compilation\n");
  99                 obj = bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
 100
 101                 if (!IS_ERR_OR_NULL(obj) && llvm_param.dump_obj)
 102                         llvm__dump_obj(filename, obj_buf, obj_buf_sz);
 103
 104                 free(obj_buf);
 105         } else
 106                 obj = bpf_object__open(filename);
 107
 108         if (IS_ERR_OR_NULL(obj)) {
 109                 pr_debug("bpf: failed to load %s\n", filename);
 110                 return obj;
 111         }
 112
 113         return obj;
 114 }
 115
 116 void bpf__clear(void)
 117 {
 118         struct bpf_object *obj, *tmp;
 119
 120         bpf_object__for_each_safe(obj, tmp) {
 121                 bpf__unprobe(obj);
 122                 bpf_object__close(obj);
 123         }
 124 }
 125
 126 static void
 127 clear_prog_priv(struct bpf_program *prog __maybe_unused,
 128                 void *_priv)
 129 {
 130         struct bpf_prog_priv *priv = _priv;
 131
 132         cleanup_perf_probe_events(&priv->pev, 1);
 133         zfree(&priv->insns_buf);
 134         zfree(&priv->type_mapping);
 135         zfree(&priv->sys_name);
 136         zfree(&priv->evt_name);
 137         free(priv);
 138 }
 139
 140 static int
 141 prog_config__exec(const char *value, struct perf_probe_event *pev)
 142 {
 143         pev->uprobes = true;
 144         pev->target = strdup(value);
 145         if (!pev->target)
 146                 return -ENOMEM;
 147         return 0;
 148 }
 149
 150 static int
 151 prog_config__module(const char *value, struct perf_probe_event *pev)
 152 {
 153         pev->uprobes = false;
 154         pev->target = strdup(value);
 155         if (!pev->target)
 156                 return -ENOMEM;
 157         return 0;
 158 }
 159
 160 static int
 161 prog_config__bool(const char *value, bool *pbool, bool invert)
 162 {
 163         int err;
 164         bool bool_value;
 165
 166         if (!pbool)
 167                 return -EINVAL;
 168
 169         err = strtobool(value, &bool_value);
 170         if (err)
 171                 return err;
 172
 173         *pbool = invert ? !bool_value : bool_value;
 174         return 0;
 175 }
 176
 177 static int
 178 prog_config__inlines(const char *value,
 179                      struct perf_probe_event *pev __maybe_unused)
 180 {
 181         return prog_config__bool(value, &probe_conf.no_inlines, true);
 182 }
 183
 184 static int
 185 prog_config__force(const char *value,
 186                    struct perf_probe_event *pev __maybe_unused)
 187 {
 188         return prog_config__bool(value, &probe_conf.force_add, false);
 189 }
 190
 191 static struct {
 192         const char *key;
 193         const char *usage;
 194         const char *desc;
 195         int (*func)(const char *, struct perf_probe_event *);
 196 } bpf_prog_config_terms[] = {
 197         {
 198                 .key    = "exec",
 199                 .usage  = "exec=<full path of file>",
 200                 .desc   = "Set uprobe target",
 201                 .func   = prog_config__exec,
 202         },
 203         {
 204                 .key    = "module",
 205                 .usage  = "module=<module name>    ",
 206                 .desc   = "Set kprobe module",
 207                 .func   = prog_config__module,
 208         },
 209         {
 210                 .key    = "inlines",
 211                 .usage  = "inlines=[yes|no]        ",
 212                 .desc   = "Probe at inline symbol",
 213                 .func   = prog_config__inlines,
 214         },
 215         {
 216                 .key    = "force",
 217                 .usage  = "force=[yes|no]          ",
 218                 .desc   = "Forcibly add events with existing name",
 219                 .func   = prog_config__force,
 220         },
 221 };
 222
 223 static int
 224 do_prog_config(const char *key, const char *value,
 225                struct perf_probe_event *pev)
 226 {
 227         unsigned int i;
 228
 229         pr_debug("config bpf program: %s=%s\n", key, value);
 230         for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
 231                 if (strcmp(key, bpf_prog_config_terms[i].key) == 0)
 232                         return bpf_prog_config_terms[i].func(value, pev);
 233
 234         pr_debug("BPF: ERROR: invalid program config option: %s=%s\n",
 235                  key, value);
 236
 237         pr_debug("\nHint: Valid options are:\n");
 238         for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
 239                 pr_debug("\t%s:\t%s\n", bpf_prog_config_terms[i].usage,
 240                          bpf_prog_config_terms[i].desc);
 241         pr_debug("\n");
 242
 243         return -BPF_LOADER_ERRNO__PROGCONF_TERM;
 244 }
 245
 246 static const char *
 247 parse_prog_config_kvpair(const char *config_str, struct perf_probe_event *pev)
 248 {
 249         char *text = strdup(config_str);
 250         char *sep, *line;
 251         const char *main_str = NULL;
 252         int err = 0;
 253
 254         if (!text) {
 255                 pr_debug("Not enough memory: dup config_str failed\n");
 256                 return ERR_PTR(-ENOMEM);
 257         }
 258
 259         line = text;
 260         while ((sep = strchr(line, ';'))) {
 261                 char *equ;
 262
 263                 *sep = '\0';
 264                 equ = strchr(line, '=');
 265                 if (!equ) {
 266                         pr_warning("WARNING: invalid config in BPF object: %s\n",
 267                                    line);
 268                         pr_warning("\tShould be 'key=value'.\n");
 269                         goto nextline;
 270                 }
 271                 *equ = '\0';
 272
 273                 err = do_prog_config(line, equ + 1, pev);
 274                 if (err)
 275                         break;
 276 nextline:
 277                 line = sep + 1;
 278         }
 279
 280         if (!err)
 281                 main_str = config_str + (line - text);
 282         free(text);
 283
 284         return err ? ERR_PTR(err) : main_str;
 285 }
 286
 287 static int
 288 parse_prog_config(const char *config_str, const char **p_main_str,
 289                   bool *is_tp, struct perf_probe_event *pev)
 290 {
 291         int err;
 292         const char *main_str = parse_prog_config_kvpair(config_str, pev);
 293
 294         if (IS_ERR(main_str))
 295                 return PTR_ERR(main_str);
 296
 297         *p_main_str = main_str;
 298         if (!strchr(main_str, '=')) {
 299                 /* Is a tracepoint event? */
 300                 const char *s = strchr(main_str, ':');
 301
 302                 if (!s) {
 303                         pr_debug("bpf: '%s' is not a valid tracepoint\n",
 304                                  config_str);
 305                         return -BPF_LOADER_ERRNO__CONFIG;
 306                 }
 307
 308                 *is_tp = true;
 309                 return 0;
 310         }
 311
 312         *is_tp = false;
 313         err = parse_perf_probe_command(main_str, pev);
 314         if (err < 0) {
 315                 pr_debug("bpf: '%s' is not a valid config string\n",
 316                          config_str);
 317                 /* parse failed, don't need clear pev. */
 318                 return -BPF_LOADER_ERRNO__CONFIG;
 319         }
 320         return 0;
 321 }
 322
 323 static int
 324 config_bpf_program(struct bpf_program *prog)
 325 {
 326         struct perf_probe_event *pev = NULL;
 327         struct bpf_prog_priv *priv = NULL;
 328         const char *config_str, *main_str;
 329         bool is_tp = false;
 330         int err;
 331
 332         /* Initialize per-program probing setting */
 333         probe_conf.no_inlines = false;
 334         probe_conf.force_add = false;
 335
 336         priv = calloc(sizeof(*priv), 1);
 337         if (!priv) {
 338                 pr_debug("bpf: failed to alloc priv\n");
 339                 return -ENOMEM;
 340         }
 341         pev = &priv->pev;
 342
 343         config_str = bpf_program__section_name(prog);
 344         pr_debug("bpf: config program '%s'\n", config_str);
 345         err = parse_prog_config(config_str, &main_str, &is_tp, pev);
 346         if (err)
 347                 goto errout;
 348
 349         if (is_tp) {
 350                 char *s = strchr(main_str, ':');
 351
 352                 priv->is_tp = true;
 353                 priv->sys_name = strndup(main_str, s - main_str);
 354                 priv->evt_name = strdup(s + 1);
 355                 goto set_priv;
 356         }
 357
 358         if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
 359                 pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
 360                          config_str, PERF_BPF_PROBE_GROUP);
 361                 err = -BPF_LOADER_ERRNO__GROUP;
 362                 goto errout;
 363         } else if (!pev->group)
 364                 pev->group = strdup(PERF_BPF_PROBE_GROUP);
 365
 366         if (!pev->group) {
 367                 pr_debug("bpf: strdup failed\n");
 368                 err = -ENOMEM;
 369                 goto errout;
 370         }
 371
 372         if (!pev->event) {
 373                 pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
 374                          config_str);
 375                 err = -BPF_LOADER_ERRNO__EVENTNAME;
 376                 goto errout;
 377         }
 378         pr_debug("bpf: config '%s' is ok\n", config_str);
 379
 380 set_priv:
 381         err = bpf_program__set_priv(prog, priv, clear_prog_priv);
 382         if (err) {
 383                 pr_debug("Failed to set priv for program '%s'\n", config_str);
 384                 goto errout;
 385         }
 386
 387         return 0;
 388
 389 errout:
 390         if (pev)
 391                 clear_perf_probe_event(pev);
 392         free(priv);
 393         return err;
 394 }
 395
 396 static int bpf__prepare_probe(void)
 397 {
 398         static int err = 0;
 399         static bool initialized = false;
 400
 401         /*
 402          * Make err static, so if init failed the first, bpf__prepare_probe()
 403          * fails each time without calling init_probe_symbol_maps multiple
 404          * times.
 405          */
 406         if (initialized)
 407                 return err;
 408
 409         initialized = true;
 410         err = init_probe_symbol_maps(false);
 411         if (err < 0)
 412                 pr_debug("Failed to init_probe_symbol_maps\n");
 413         probe_conf.max_probes = MAX_PROBES;
 414         return err;
 415 }
 416
 417 static int
 418 preproc_gen_prologue(struct bpf_program *prog, int n,
 419                      struct bpf_insn *orig_insns, int orig_insns_cnt,
 420                      struct bpf_prog_prep_result *res)
 421 {
 422         struct bpf_prog_priv *priv = bpf_program__priv(prog);
 423         struct probe_trace_event *tev;
 424         struct perf_probe_event *pev;
 425         struct bpf_insn *buf;
 426         size_t prologue_cnt = 0;
 427         int i, err;
 428
 429         if (IS_ERR_OR_NULL(priv) || priv->is_tp)
 430                 goto errout;
 431
 432         pev = &priv->pev;
 433
 434         if (n < 0 || n >= priv->nr_types)
 435                 goto errout;
 436
 437         /* Find a tev belongs to that type */
 438         for (i = 0; i < pev->ntevs; i++) {
 439                 if (priv->type_mapping[i] == n)
 440                         break;
 441         }
 442
 443         if (i >= pev->ntevs) {
 444                 pr_debug("Internal error: prologue type %d not found\n", n);
 445                 return -BPF_LOADER_ERRNO__PROLOGUE;
 446         }
 447
 448         tev = &pev->tevs[i];
 449
 450         buf = priv->insns_buf;
 451         err = bpf__gen_prologue(tev->args, tev->nargs,
 452                                 buf, &prologue_cnt,
 453                                 BPF_MAXINSNS - orig_insns_cnt);
 454         if (err) {
 455                 const char *title;
 456
 457                 title = bpf_program__section_name(prog);
 458                 pr_debug("Failed to generate prologue for program %s\n",
 459                          title);
 460                 return err;
 461         }
 462
 463         memcpy(&buf[prologue_cnt], orig_insns,
 464                sizeof(struct bpf_insn) * orig_insns_cnt);
 465
 466         res->new_insn_ptr = buf;
 467         res->new_insn_cnt = prologue_cnt + orig_insns_cnt;
 468         res->pfd = NULL;
 469         return 0;
 470
 471 errout:
 472         pr_debug("Internal error in preproc_gen_prologue\n");
 473         return -BPF_LOADER_ERRNO__PROLOGUE;
 474 }
 475
 476 /*
 477  * compare_tev_args is reflexive, transitive and antisymmetric.
 478  * I can proof it but this margin is too narrow to contain.
 479  */
 480 static int compare_tev_args(const void *ptev1, const void *ptev2)
 481 {
 482         int i, ret;
 483         const struct probe_trace_event *tev1 =
 484                 *(const struct probe_trace_event **)ptev1;
 485         const struct probe_trace_event *tev2 =
 486                 *(const struct probe_trace_event **)ptev2;
 487
 488         ret = tev2->nargs - tev1->nargs;
 489         if (ret)
 490                 return ret;
 491
 492         for (i = 0; i < tev1->nargs; i++) {
 493                 struct probe_trace_arg *arg1, *arg2;
 494                 struct probe_trace_arg_ref *ref1, *ref2;
 495
 496                 arg1 = &tev1->args[i];
 497                 arg2 = &tev2->args[i];
 498
 499                 ret = strcmp(arg1->value, arg2->value);
 500                 if (ret)
 501                         return ret;
 502
 503                 ref1 = arg1->ref;
 504                 ref2 = arg2->ref;
 505
 506                 while (ref1 && ref2) {
 507                         ret = ref2->offset - ref1->offset;
 508                         if (ret)
 509                                 return ret;
 510
 511                         ref1 = ref1->next;
 512                         ref2 = ref2->next;
 513                 }
 514
 515                 if (ref1 || ref2)
 516                         return ref2 ? 1 : -1;
 517         }
 518
 519         return 0;
 520 }
 521
 522 /*
 523  * Assign a type number to each tevs in a pev.
 524  * mapping is an array with same slots as tevs in that pev.
 525  * nr_types will be set to number of types.
 526  */
 527 static int map_prologue(struct perf_probe_event *pev, int *mapping,
 528                         int *nr_types)
 529 {
 530         int i, type = 0;
 531         struct probe_trace_event **ptevs;
 532
 533         size_t array_sz = sizeof(*ptevs) * pev->ntevs;
 534
 535         ptevs = malloc(array_sz);
 536         if (!ptevs) {
 537                 pr_debug("Not enough memory: alloc ptevs failed\n");
 538                 return -ENOMEM;
 539         }
 540
 541         pr_debug("In map_prologue, ntevs=%d\n", pev->ntevs);
 542         for (i = 0; i < pev->ntevs; i++)
 543                 ptevs[i] = &pev->tevs[i];
 544
 545         qsort(ptevs, pev->ntevs, sizeof(*ptevs),
 546               compare_tev_args);
 547
 548         for (i = 0; i < pev->ntevs; i++) {
 549                 int n;
 550
 551                 n = ptevs[i] - pev->tevs;
 552                 if (i == 0) {
 553                         mapping[n] = type;
 554                         pr_debug("mapping[%d]=%d\n", n, type);
 555                         continue;
 556                 }
 557
 558                 if (compare_tev_args(ptevs + i, ptevs + i - 1) == 0)
 559                         mapping[n] = type;
 560                 else
 561                         mapping[n] = ++type;
 562
 563                 pr_debug("mapping[%d]=%d\n", n, mapping[n]);
 564         }
 565         free(ptevs);
 566         *nr_types = type + 1;
 567
 568         return 0;
 569 }
 570
 571 static int hook_load_preprocessor(struct bpf_program *prog)
 572 {
 573         struct bpf_prog_priv *priv = bpf_program__priv(prog);
 574         struct perf_probe_event *pev;
 575         bool need_prologue = false;
 576         int err, i;
 577
 578         if (IS_ERR_OR_NULL(priv)) {
 579                 pr_debug("Internal error when hook preprocessor\n");
 580                 return -BPF_LOADER_ERRNO__INTERNAL;
 581         }
 582
 583         if (priv->is_tp) {
 584                 priv->need_prologue = false;
 585                 return 0;
 586         }
 587
 588         pev = &priv->pev;
 589         for (i = 0; i < pev->ntevs; i++) {
 590                 struct probe_trace_event *tev = &pev->tevs[i];
 591
 592                 if (tev->nargs > 0) {
 593                         need_prologue = true;
 594                         break;
 595                 }
 596         }
 597
 598         /*
 599          * Since all tevs don't have argument, we don't need generate
 600          * prologue.
 601          */
 602         if (!need_prologue) {
 603                 priv->need_prologue = false;
 604                 return 0;
 605         }
 606
 607         priv->need_prologue = true;
 608         priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS);
 609         if (!priv->insns_buf) {
 610                 pr_debug("Not enough memory: alloc insns_buf failed\n");
 611                 return -ENOMEM;
 612         }
 613
 614         priv->type_mapping = malloc(sizeof(int) * pev->ntevs);
 615         if (!priv->type_mapping) {
 616                 pr_debug("Not enough memory: alloc type_mapping failed\n");
 617                 return -ENOMEM;
 618         }
 619         memset(priv->type_mapping, -1,
 620                sizeof(int) * pev->ntevs);
 621
 622         err = map_prologue(pev, priv->type_mapping, &priv->nr_types);
 623         if (err)
 624                 return err;
 625
 626         err = bpf_program__set_prep(prog, priv->nr_types,
 627                                     preproc_gen_prologue);
 628         return err;
 629 }
 630
 631 int bpf__probe(struct bpf_object *obj)
 632 {
 633         int err = 0;
 634         struct bpf_program *prog;
 635         struct bpf_prog_priv *priv;
 636         struct perf_probe_event *pev;
 637
 638         err = bpf__prepare_probe();
 639         if (err) {
 640                 pr_debug("bpf__prepare_probe failed\n");
 641                 return err;
 642         }
 643
 644         bpf_object__for_each_program(prog, obj) {
 645                 err = config_bpf_program(prog);
 646                 if (err)
 647                         goto out;
 648
 649                 priv = bpf_program__priv(prog);
 650                 if (IS_ERR_OR_NULL(priv)) {
 651                         if (!priv)
 652                                 err = -BPF_LOADER_ERRNO__INTERNAL;
 653                         else
 654                                 err = PTR_ERR(priv);
 655                         goto out;
 656                 }
 657
 658                 if (priv->is_tp) {
 659                         bpf_program__set_type(prog, BPF_PROG_TYPE_TRACEPOINT);
 660                         continue;
 661                 }
 662
 663                 bpf_program__set_type(prog, BPF_PROG_TYPE_KPROBE);
 664                 pev = &priv->pev;
 665
 666                 err = convert_perf_probe_events(pev, 1);
 667                 if (err < 0) {
 668                         pr_debug("bpf_probe: failed to convert perf probe events\n");
 669                         goto out;
 670                 }
 671
 672                 err = apply_perf_probe_events(pev, 1);
 673                 if (err < 0) {
 674                         pr_debug("bpf_probe: failed to apply perf probe events\n");
 675                         goto out;
 676                 }
 677
 678                 /*
 679                  * After probing, let's consider prologue, which
 680                  * adds program fetcher to BPF programs.
 681                  *
 682                  * hook_load_preprocessor() hooks pre-processor
 683                  * to bpf_program, let it generate prologue
 684                  * dynamically during loading.
 685                  */
 686                 err = hook_load_preprocessor(prog);
 687                 if (err)
 688                         goto out;
 689         }
 690 out:
 691         return err < 0 ? err : 0;
 692 }
 693
 694 #define EVENTS_WRITE_BUFSIZE  4096
 695 int bpf__unprobe(struct bpf_object *obj)
 696 {
 697         int err, ret = 0;
 698         struct bpf_program *prog;
 699
 700         bpf_object__for_each_program(prog, obj) {
 701                 struct bpf_prog_priv *priv = bpf_program__priv(prog);
 702                 int i;
 703
 704                 if (IS_ERR_OR_NULL(priv) || priv->is_tp)
 705                         continue;
 706
 707                 for (i = 0; i < priv->pev.ntevs; i++) {
 708                         struct probe_trace_event *tev = &priv->pev.tevs[i];
 709                         char name_buf[EVENTS_WRITE_BUFSIZE];
 710                         struct strfilter *delfilter;
 711
 712                         snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
 713                                  "%s:%s", tev->group, tev->event);
 714                         name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
 715
 716                         delfilter = strfilter__new(name_buf, NULL);
 717                         if (!delfilter) {
 718                                 pr_debug("Failed to create filter for unprobing\n");
 719                                 ret = -ENOMEM;
 720                                 continue;
 721                         }
 722
 723                         err = del_perf_probe_events(delfilter);
 724                         strfilter__delete(delfilter);
 725                         if (err) {
 726                                 pr_debug("Failed to delete %s\n", name_buf);
 727                                 ret = err;
 728                                 continue;
 729                         }
 730                 }
 731         }
 732         return ret;
 733 }
 734
 735 int bpf__load(struct bpf_object *obj)
 736 {
 737         int err;
 738
 739         err = bpf_object__load(obj);
 740         if (err) {
 741                 char bf[128];
 742                 libbpf_strerror(err, bf, sizeof(bf));
 743                 pr_debug("bpf: load objects failed: err=%d: (%s)\n", err, bf);
 744                 return err;
 745         }
 746         return 0;
 747 }
 748
 749 int bpf__foreach_event(struct bpf_object *obj,
 750                        bpf_prog_iter_callback_t func,
 751                        void *arg)
 752 {
 753         struct bpf_program *prog;
 754         int err;
 755
 756         bpf_object__for_each_program(prog, obj) {
 757                 struct bpf_prog_priv *priv = bpf_program__priv(prog);
 758                 struct probe_trace_event *tev;
 759                 struct perf_probe_event *pev;
 760                 int i, fd;
 761
 762                 if (IS_ERR_OR_NULL(priv)) {
 763                         pr_debug("bpf: failed to get private field\n");
 764                         return -BPF_LOADER_ERRNO__INTERNAL;
 765                 }
 766
 767                 if (priv->is_tp) {
 768                         fd = bpf_program__fd(prog);
 769                         err = (*func)(priv->sys_name, priv->evt_name, fd, obj, arg);
 770                         if (err) {
 771                                 pr_debug("bpf: tracepoint call back failed, stop iterate\n");
 772                                 return err;
 773                         }
 774                         continue;
 775                 }
 776
 777                 pev = &priv->pev;
 778                 for (i = 0; i < pev->ntevs; i++) {
 779                         tev = &pev->tevs[i];
 780
 781                         if (priv->need_prologue) {
 782                                 int type = priv->type_mapping[i];
 783
 784                                 fd = bpf_program__nth_fd(prog, type);
 785                         } else {
 786                                 fd = bpf_program__fd(prog);
 787                         }
 788
 789                         if (fd < 0) {
 790                                 pr_debug("bpf: failed to get file descriptor\n");
 791                                 return fd;
 792                         }
 793
 794                         err = (*func)(tev->group, tev->event, fd, obj, arg);
 795                         if (err) {
 796                                 pr_debug("bpf: call back failed, stop iterate\n");
 797                                 return err;
 798                         }
 799                 }
 800         }
 801         return 0;
 802 }
 803
 804 enum bpf_map_op_type {
 805         BPF_MAP_OP_SET_VALUE,
 806         BPF_MAP_OP_SET_EVSEL,
 807 };
 808
 809 enum bpf_map_key_type {
 810         BPF_MAP_KEY_ALL,
 811         BPF_MAP_KEY_RANGES,
 812 };
 813
 814 struct bpf_map_op {
 815         struct list_head list;
 816         enum bpf_map_op_type op_type;
 817         enum bpf_map_key_type key_type;
 818         union {
 819                 struct parse_events_array array;
 820         } k;
 821         union {
 822                 u64 value;
 823                 struct evsel *evsel;
 824         } v;
 825 };
 826
 827 struct bpf_map_priv {
 828         struct list_head ops_list;
 829 };
 830
 831 static void
 832 bpf_map_op__delete(struct bpf_map_op *op)
 833 {
 834         if (!list_empty(&op->list))
 835                 list_del_init(&op->list);
 836         if (op->key_type == BPF_MAP_KEY_RANGES)
 837                 parse_events__clear_array(&op->k.array);
 838         free(op);
 839 }
 840
 841 static void
 842 bpf_map_priv__purge(struct bpf_map_priv *priv)
 843 {
 844         struct bpf_map_op *pos, *n;
 845
 846         list_for_each_entry_safe(pos, n, &priv->ops_list, list) {
 847                 list_del_init(&pos->list);
 848                 bpf_map_op__delete(pos);
 849         }
 850 }
 851
 852 static void
 853 bpf_map_priv__clear(struct bpf_map *map __maybe_unused,
 854                     void *_priv)
 855 {
 856         struct bpf_map_priv *priv = _priv;
 857
 858         bpf_map_priv__purge(priv);
 859         free(priv);
 860 }
 861
 862 static int
 863 bpf_map_op_setkey(struct bpf_map_op *op, struct parse_events_term *term)
 864 {
 865         op->key_type = BPF_MAP_KEY_ALL;
 866         if (!term)
 867                 return 0;
 868
 869         if (term->array.nr_ranges) {
 870                 size_t memsz = term->array.nr_ranges *
 871                                 sizeof(op->k.array.ranges[0]);
 872
 873                 op->k.array.ranges = memdup(term->array.ranges, memsz);
 874                 if (!op->k.array.ranges) {
 875                         pr_debug("Not enough memory to alloc indices for map\n");
 876                         return -ENOMEM;
 877                 }
 878                 op->key_type = BPF_MAP_KEY_RANGES;
 879                 op->k.array.nr_ranges = term->array.nr_ranges;
 880         }
 881         return 0;
 882 }
 883
 884 static struct bpf_map_op *
 885 bpf_map_op__new(struct parse_events_term *term)
 886 {
 887         struct bpf_map_op *op;
 888         int err;
 889
 890         op = zalloc(sizeof(*op));
 891         if (!op) {
 892                 pr_debug("Failed to alloc bpf_map_op\n");
 893                 return ERR_PTR(-ENOMEM);
 894         }
 895         INIT_LIST_HEAD(&op->list);
 896
 897         err = bpf_map_op_setkey(op, term);
 898         if (err) {
 899                 free(op);
 900                 return ERR_PTR(err);
 901         }
 902         return op;
 903 }
 904
 905 static struct bpf_map_op *
 906 bpf_map_op__clone(struct bpf_map_op *op)
 907 {
 908         struct bpf_map_op *newop;
 909
 910         newop = memdup(op, sizeof(*op));
 911         if (!newop) {
 912                 pr_debug("Failed to alloc bpf_map_op\n");
 913                 return NULL;
 914         }
 915
 916         INIT_LIST_HEAD(&newop->list);
 917         if (op->key_type == BPF_MAP_KEY_RANGES) {
 918                 size_t memsz = op->k.array.nr_ranges *
 919                                sizeof(op->k.array.ranges[0]);
 920
 921                 newop->k.array.ranges = memdup(op->k.array.ranges, memsz);
 922                 if (!newop->k.array.ranges) {
 923                         pr_debug("Failed to alloc indices for map\n");
 924                         free(newop);
 925                         return NULL;
 926                 }
 927         }
 928
 929         return newop;
 930 }
 931
 932 static struct bpf_map_priv *
 933 bpf_map_priv__clone(struct bpf_map_priv *priv)
 934 {
 935         struct bpf_map_priv *newpriv;
 936         struct bpf_map_op *pos, *newop;
 937
 938         newpriv = zalloc(sizeof(*newpriv));
 939         if (!newpriv) {
 940                 pr_debug("Not enough memory to alloc map private\n");
 941                 return NULL;
 942         }
 943         INIT_LIST_HEAD(&newpriv->ops_list);
 944
 945         list_for_each_entry(pos, &priv->ops_list, list) {
 946                 newop = bpf_map_op__clone(pos);
 947                 if (!newop) {
 948                         bpf_map_priv__purge(newpriv);
 949                         return NULL;
 950                 }
 951                 list_add_tail(&newop->list, &newpriv->ops_list);
 952         }
 953
 954         return newpriv;
 955 }
 956
 957 static int
 958 bpf_map__add_op(struct bpf_map *map, struct bpf_map_op *op)
 959 {
 960         const char *map_name = bpf_map__name(map);
 961         struct bpf_map_priv *priv = bpf_map__priv(map);
 962
 963         if (IS_ERR(priv)) {
 964                 pr_debug("Failed to get private from map %s\n", map_name);
 965                 return PTR_ERR(priv);
 966         }
 967
 968         if (!priv) {
 969                 priv = zalloc(sizeof(*priv));
 970                 if (!priv) {
 971                         pr_debug("Not enough memory to alloc map private\n");
 972                         return -ENOMEM;
 973                 }
 974                 INIT_LIST_HEAD(&priv->ops_list);
 975
 976                 if (bpf_map__set_priv(map, priv, bpf_map_priv__clear)) {
 977                         free(priv);
 978                         return -BPF_LOADER_ERRNO__INTERNAL;
 979                 }
 980         }
 981
 982         list_add_tail(&op->list, &priv->ops_list);
 983         return 0;
 984 }
 985
 986 static struct bpf_map_op *
 987 bpf_map__add_newop(struct bpf_map *map, struct parse_events_term *term)
 988 {
 989         struct bpf_map_op *op;
 990         int err;
 991
 992         op = bpf_map_op__new(term);
 993         if (IS_ERR(op))
 994                 return op;
 995
 996         err = bpf_map__add_op(map, op);
 997         if (err) {
 998                 bpf_map_op__delete(op);
 999                 return ERR_PTR(err);
1000         }
1001         return op;
1002 }
1003
1004 static int
1005 __bpf_map__config_value(struct bpf_map *map,
1006                         struct parse_events_term *term)
1007 {
1008         struct bpf_map_op *op;
1009         const char *map_name = bpf_map__name(map);
1010
1011         if (!map) {
1012                 pr_debug("Map '%s' is invalid\n", map_name);
1013                 return -BPF_LOADER_ERRNO__INTERNAL;
1014         }
1015
1016         if (bpf_map__type(map) != BPF_MAP_TYPE_ARRAY) {
1017                 pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n",
1018                          map_name);
1019                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1020         }
1021         if (bpf_map__key_size(map) < sizeof(unsigned int)) {
1022                 pr_debug("Map %s has incorrect key size\n", map_name);
1023                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE;
1024         }
1025         switch (bpf_map__value_size(map)) {
1026         case 1:
1027         case 2:
1028         case 4:
1029         case 8:
1030                 break;
1031         default:
1032                 pr_debug("Map %s has incorrect value size\n", map_name);
1033                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
1034         }
1035
1036         op = bpf_map__add_newop(map, term);
1037         if (IS_ERR(op))
1038                 return PTR_ERR(op);
1039         op->op_type = BPF_MAP_OP_SET_VALUE;
1040         op->v.value = term->val.num;
1041         return 0;
1042 }
1043
1044 static int
1045 bpf_map__config_value(struct bpf_map *map,
1046                       struct parse_events_term *term,
1047                       struct evlist *evlist __maybe_unused)
1048 {
1049         if (!term->err_val) {
1050                 pr_debug("Config value not set\n");
1051                 return -BPF_LOADER_ERRNO__OBJCONF_CONF;
1052         }
1053
1054         if (term->type_val != PARSE_EVENTS__TERM_TYPE_NUM) {
1055                 pr_debug("ERROR: wrong value type for 'value'\n");
1056                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
1057         }
1058
1059         return __bpf_map__config_value(map, term);
1060 }
1061
1062 static int
1063 __bpf_map__config_event(struct bpf_map *map,
1064                         struct parse_events_term *term,
1065                         struct evlist *evlist)
1066 {
1067         struct bpf_map_op *op;
1068         const char *map_name = bpf_map__name(map);
1069         struct evsel *evsel = evlist__find_evsel_by_str(evlist, term->val.str);
1070
1071         if (!evsel) {
1072                 pr_debug("Event (for '%s') '%s' doesn't exist\n",
1073                          map_name, term->val.str);
1074                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_NOEVT;
1075         }
1076
1077         if (!map) {
1078                 pr_debug("Map '%s' is invalid\n", map_name);
1079                 return PTR_ERR(map);
1080         }
1081
1082         /*
1083          * No need to check key_size and value_size:
1084          * kernel has already checked them.
1085          */
1086         if (bpf_map__type(map) != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
1087                 pr_debug("Map %s type is not BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
1088                          map_name);
1089                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1090         }
1091
1092         op = bpf_map__add_newop(map, term);
1093         if (IS_ERR(op))
1094                 return PTR_ERR(op);
1095         op->op_type = BPF_MAP_OP_SET_EVSEL;
1096         op->v.evsel = evsel;
1097         return 0;
1098 }
1099
1100 static int
1101 bpf_map__config_event(struct bpf_map *map,
1102                       struct parse_events_term *term,
1103                       struct evlist *evlist)
1104 {
1105         if (!term->err_val) {
1106                 pr_debug("Config value not set\n");
1107                 return -BPF_LOADER_ERRNO__OBJCONF_CONF;
1108         }
1109
1110         if (term->type_val != PARSE_EVENTS__TERM_TYPE_STR) {
1111                 pr_debug("ERROR: wrong value type for 'event'\n");
1112                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
1113         }
1114
1115         return __bpf_map__config_event(map, term, evlist);
1116 }
1117
1118 struct bpf_obj_config__map_func {
1119         const char *config_opt;
1120         int (*config_func)(struct bpf_map *, struct parse_events_term *,
1121                            struct evlist *);
1122 };
1123
1124 struct bpf_obj_config__map_func bpf_obj_config__map_funcs[] = {
1125         {"value", bpf_map__config_value},
1126         {"event", bpf_map__config_event},
1127 };
1128
1129 static int
1130 config_map_indices_range_check(struct parse_events_term *term,
1131                                struct bpf_map *map,
1132                                const char *map_name)
1133 {
1134         struct parse_events_array *array = &term->array;
1135         unsigned int i;
1136
1137         if (!array->nr_ranges)
1138                 return 0;
1139         if (!array->ranges) {
1140                 pr_debug("ERROR: map %s: array->nr_ranges is %d but range array is NULL\n",
1141                          map_name, (int)array->nr_ranges);
1142                 return -BPF_LOADER_ERRNO__INTERNAL;
1143         }
1144
1145         if (!map) {
1146                 pr_debug("Map '%s' is invalid\n", map_name);
1147                 return -BPF_LOADER_ERRNO__INTERNAL;
1148         }
1149
1150         for (i = 0; i < array->nr_ranges; i++) {
1151                 unsigned int start = array->ranges[i].start;
1152                 size_t length = array->ranges[i].length;
1153                 unsigned int idx = start + length - 1;
1154
1155                 if (idx >= bpf_map__max_entries(map)) {
1156                         pr_debug("ERROR: index %d too large\n", idx);
1157                         return -BPF_LOADER_ERRNO__OBJCONF_MAP_IDX2BIG;
1158                 }
1159         }
1160         return 0;
1161 }
1162
1163 static int
1164 bpf__obj_config_map(struct bpf_object *obj,
1165                     struct parse_events_term *term,
1166                     struct evlist *evlist,
1167                     int *key_scan_pos)
1168 {
1169         /* key is "map:<mapname>.<config opt>" */
1170         char *map_name = strdup(term->config + sizeof("map:") - 1);
1171         struct bpf_map *map;
1172         int err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
1173         char *map_opt;
1174         size_t i;
1175
1176         if (!map_name)
1177                 return -ENOMEM;
1178
1179         map_opt = strchr(map_name, '.');
1180         if (!map_opt) {
1181                 pr_debug("ERROR: Invalid map config: %s\n", map_name);
1182                 goto out;
1183         }
1184
1185         *map_opt++ = '\0';
1186         if (*map_opt == '\0') {
1187                 pr_debug("ERROR: Invalid map option: %s\n", term->config);
1188                 goto out;
1189         }
1190
1191         map = bpf_object__find_map_by_name(obj, map_name);
1192         if (!map) {
1193                 pr_debug("ERROR: Map %s doesn't exist\n", map_name);
1194                 err = -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST;
1195                 goto out;
1196         }
1197
1198         *key_scan_pos += strlen(map_opt);
1199         err = config_map_indices_range_check(term, map, map_name);
1200         if (err)
1201                 goto out;
1202         *key_scan_pos -= strlen(map_opt);
1203
1204         for (i = 0; i < ARRAY_SIZE(bpf_obj_config__map_funcs); i++) {
1205                 struct bpf_obj_config__map_func *func =
1206                                 &bpf_obj_config__map_funcs[i];
1207
1208                 if (strcmp(map_opt, func->config_opt) == 0) {
1209                         err = func->config_func(map, term, evlist);
1210                         goto out;
1211                 }
1212         }
1213
1214         pr_debug("ERROR: Invalid map config option '%s'\n", map_opt);
1215         err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT;
1216 out:
1217         if (!err)
1218                 *key_scan_pos += strlen(map_opt);
1219
1220         free(map_name);
1221         return err;
1222 }
1223
1224 int bpf__config_obj(struct bpf_object *obj,
1225                     struct parse_events_term *term,
1226                     struct evlist *evlist,
1227                     int *error_pos)
1228 {
1229         int key_scan_pos = 0;
1230         int err;
1231
1232         if (!obj || !term || !term->config)
1233                 return -EINVAL;
1234
1235         if (strstarts(term->config, "map:")) {
1236                 key_scan_pos = sizeof("map:") - 1;
1237                 err = bpf__obj_config_map(obj, term, evlist, &key_scan_pos);
1238                 goto out;
1239         }
1240         err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
1241 out:
1242         if (error_pos)
1243                 *error_pos = key_scan_pos;
1244         return err;
1245
1246 }
1247
1248 typedef int (*map_config_func_t)(const char *name, int map_fd,
1249                                  const struct bpf_map *map,
1250                                  struct bpf_map_op *op,
1251                                  void *pkey, void *arg);
1252
1253 static int
1254 foreach_key_array_all(map_config_func_t func,
1255                       void *arg, const char *name,
1256                       int map_fd, const struct bpf_map *map,
1257                       struct bpf_map_op *op)
1258 {
1259         unsigned int i;
1260         int err;
1261
1262         for (i = 0; i < bpf_map__max_entries(map); i++) {
1263                 err = func(name, map_fd, map, op, &i, arg);
1264                 if (err) {
1265                         pr_debug("ERROR: failed to insert value to %s[%u]\n",
1266                                  name, i);
1267                         return err;
1268                 }
1269         }
1270         return 0;
1271 }
1272
1273 static int
1274 foreach_key_array_ranges(map_config_func_t func, void *arg,
1275                          const char *name, int map_fd,
1276                          const struct bpf_map *map,
1277                          struct bpf_map_op *op)
1278 {
1279         unsigned int i, j;
1280         int err;
1281
1282         for (i = 0; i < op->k.array.nr_ranges; i++) {
1283                 unsigned int start = op->k.array.ranges[i].start;
1284                 size_t length = op->k.array.ranges[i].length;
1285
1286                 for (j = 0; j < length; j++) {
1287                         unsigned int idx = start + j;
1288
1289                         err = func(name, map_fd, map, op, &idx, arg);
1290                         if (err) {
1291                                 pr_debug("ERROR: failed to insert value to %s[%u]\n",
1292                                          name, idx);
1293                                 return err;
1294                         }
1295                 }
1296         }
1297         return 0;
1298 }
1299
1300 static int
1301 bpf_map_config_foreach_key(struct bpf_map *map,
1302                            map_config_func_t func,
1303                            void *arg)
1304 {
1305         int err, map_fd, type;
1306         struct bpf_map_op *op;
1307         const char *name = bpf_map__name(map);
1308         struct bpf_map_priv *priv = bpf_map__priv(map);
1309
1310         if (IS_ERR(priv)) {
1311                 pr_debug("ERROR: failed to get private from map %s\n", name);
1312                 return -BPF_LOADER_ERRNO__INTERNAL;
1313         }
1314         if (!priv || list_empty(&priv->ops_list)) {
1315                 pr_debug("INFO: nothing to config for map %s\n", name);
1316                 return 0;
1317         }
1318
1319         if (!map) {
1320                 pr_debug("Map '%s' is invalid\n", name);
1321                 return -BPF_LOADER_ERRNO__INTERNAL;
1322         }
1323         map_fd = bpf_map__fd(map);
1324         if (map_fd < 0) {
1325                 pr_debug("ERROR: failed to get fd from map %s\n", name);
1326                 return map_fd;
1327         }
1328
1329         type = bpf_map__type(map);
1330         list_for_each_entry(op, &priv->ops_list, list) {
1331                 switch (type) {
1332                 case BPF_MAP_TYPE_ARRAY:
1333                 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1334                         switch (op->key_type) {
1335                         case BPF_MAP_KEY_ALL:
1336                                 err = foreach_key_array_all(func, arg, name,
1337                                                             map_fd, map, op);
1338                                 break;
1339                         case BPF_MAP_KEY_RANGES:
1340                                 err = foreach_key_array_ranges(func, arg, name,
1341                                                                map_fd, map, op);
1342                                 break;
1343                         default:
1344                                 pr_debug("ERROR: keytype for map '%s' invalid\n",
1345                                          name);
1346                                 return -BPF_LOADER_ERRNO__INTERNAL;
1347                         }
1348                         if (err)
1349                                 return err;
1350                         break;
1351                 default:
1352                         pr_debug("ERROR: type of '%s' incorrect\n", name);
1353                         return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1354                 }
1355         }
1356
1357         return 0;
1358 }
1359
1360 static int
1361 apply_config_value_for_key(int map_fd, void *pkey,
1362                            size_t val_size, u64 val)
1363 {
1364         int err = 0;
1365
1366         switch (val_size) {
1367         case 1: {
1368                 u8 _val = (u8)(val);
1369                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1370                 break;
1371         }
1372         case 2: {
1373                 u16 _val = (u16)(val);
1374                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1375                 break;
1376         }
1377         case 4: {
1378                 u32 _val = (u32)(val);
1379                 err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1380                 break;
1381         }
1382         case 8: {
1383                 err = bpf_map_update_elem(map_fd, pkey, &val, BPF_ANY);
1384                 break;
1385         }
1386         default:
1387                 pr_debug("ERROR: invalid value size\n");
1388                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
1389         }
1390         if (err && errno)
1391                 err = -errno;
1392         return err;
1393 }
1394
1395 static int
1396 apply_config_evsel_for_key(const char *name, int map_fd, void *pkey,
1397                            struct evsel *evsel)
1398 {
1399         struct xyarray *xy = evsel->core.fd;
1400         struct perf_event_attr *attr;
1401         unsigned int key, events;
1402         bool check_pass = false;
1403         int *evt_fd;
1404         int err;
1405
1406         if (!xy) {
1407                 pr_debug("ERROR: evsel not ready for map %s\n", name);
1408                 return -BPF_LOADER_ERRNO__INTERNAL;
1409         }
1410
1411         if (xy->row_size / xy->entry_size != 1) {
1412                 pr_debug("ERROR: Dimension of target event is incorrect for map %s\n",
1413                          name);
1414                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM;
1415         }
1416
1417         attr = &evsel->core.attr;
1418         if (attr->inherit) {
1419                 pr_debug("ERROR: Can't put inherit event into map %s\n", name);
1420                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH;
1421         }
1422
1423         if (evsel__is_bpf_output(evsel))
1424                 check_pass = true;
1425         if (attr->type == PERF_TYPE_RAW)
1426                 check_pass = true;
1427         if (attr->type == PERF_TYPE_HARDWARE)
1428                 check_pass = true;
1429         if (!check_pass) {
1430                 pr_debug("ERROR: Event type is wrong for map %s\n", name);
1431                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE;
1432         }
1433
1434         events = xy->entries / (xy->row_size / xy->entry_size);
1435         key = *((unsigned int *)pkey);
1436         if (key >= events) {
1437                 pr_debug("ERROR: there is no event %d for map %s\n",
1438                          key, name);
1439                 return -BPF_LOADER_ERRNO__OBJCONF_MAP_MAPSIZE;
1440         }
1441         evt_fd = xyarray__entry(xy, key, 0);
1442         err = bpf_map_update_elem(map_fd, pkey, evt_fd, BPF_ANY);
1443         if (err && errno)
1444                 err = -errno;
1445         return err;
1446 }
1447
1448 static int
1449 apply_obj_config_map_for_key(const char *name, int map_fd,
1450                              const struct bpf_map *map,
1451                              struct bpf_map_op *op,
1452                              void *pkey, void *arg __maybe_unused)
1453 {
1454         int err;
1455
1456         switch (op->op_type) {
1457         case BPF_MAP_OP_SET_VALUE:
1458                 err = apply_config_value_for_key(map_fd, pkey,
1459                                                  bpf_map__value_size(map),
1460                                                  op->v.value);
1461                 break;
1462         case BPF_MAP_OP_SET_EVSEL:
1463                 err = apply_config_evsel_for_key(name, map_fd, pkey,
1464                                                  op->v.evsel);
1465                 break;
1466         default:
1467                 pr_debug("ERROR: unknown value type for '%s'\n", name);
1468                 err = -BPF_LOADER_ERRNO__INTERNAL;
1469         }
1470         return err;
1471 }
1472
1473 static int
1474 apply_obj_config_map(struct bpf_map *map)
1475 {
1476         return bpf_map_config_foreach_key(map,
1477                                           apply_obj_config_map_for_key,
1478                                           NULL);
1479 }
1480
1481 static int
1482 apply_obj_config_object(struct bpf_object *obj)
1483 {
1484         struct bpf_map *map;
1485         int err;
1486
1487         bpf_object__for_each_map(map, obj) {
1488                 err = apply_obj_config_map(map);
1489                 if (err)
1490                         return err;
1491         }
1492         return 0;
1493 }
1494
1495 int bpf__apply_obj_config(void)
1496 {
1497         struct bpf_object *obj, *tmp;
1498         int err;
1499
1500         bpf_object__for_each_safe(obj, tmp) {
1501                 err = apply_obj_config_object(obj);
1502                 if (err)
1503                         return err;
1504         }
1505
1506         return 0;
1507 }
1508
1509 #define bpf__for_each_map(pos, obj, objtmp)     \
1510         bpf_object__for_each_safe(obj, objtmp)  \
1511                 bpf_object__for_each_map(pos, obj)
1512
1513 #define bpf__for_each_map_named(pos, obj, objtmp, name) \
1514         bpf__for_each_map(pos, obj, objtmp)             \
1515                 if (bpf_map__name(pos) &&               \
1516                         (strcmp(name,                   \
1517                                 bpf_map__name(pos)) == 0))
1518
1519 struct evsel *bpf__setup_output_event(struct evlist *evlist, const char *name)
1520 {
1521         struct bpf_map_priv *tmpl_priv = NULL;
1522         struct bpf_object *obj, *tmp;
1523         struct evsel *evsel = NULL;
1524         struct bpf_map *map;
1525         int err;
1526         bool need_init = false;
1527
1528         bpf__for_each_map_named(map, obj, tmp, name) {
1529                 struct bpf_map_priv *priv = bpf_map__priv(map);
1530
1531                 if (IS_ERR(priv))
1532                         return ERR_PTR(-BPF_LOADER_ERRNO__INTERNAL);
1533
1534                 /*
1535                  * No need to check map type: type should have been
1536                  * verified by kernel.
1537                  */
1538                 if (!need_init && !priv)
1539                         need_init = !priv;
1540                 if (!tmpl_priv && priv)
1541                         tmpl_priv = priv;
1542         }
1543
1544         if (!need_init)
1545                 return NULL;
1546
1547         if (!tmpl_priv) {
1548                 char *event_definition = NULL;
1549
1550                 if (asprintf(&event_definition, "bpf-output/no-inherit=1,name=%s/", name) < 0)
1551                         return ERR_PTR(-ENOMEM);
1552
1553                 err = parse_events(evlist, event_definition, NULL);
1554                 free(event_definition);
1555
1556                 if (err) {
1557                         pr_debug("ERROR: failed to create the \"%s\" bpf-output event\n", name);
1558                         return ERR_PTR(-err);
1559                 }
1560
1561                 evsel = evlist__last(evlist);
1562         }
1563
1564         bpf__for_each_map_named(map, obj, tmp, name) {
1565                 struct bpf_map_priv *priv = bpf_map__priv(map);
1566
1567                 if (IS_ERR(priv))
1568                         return ERR_PTR(-BPF_LOADER_ERRNO__INTERNAL);
1569                 if (priv)
1570                         continue;
1571
1572                 if (tmpl_priv) {
1573                         priv = bpf_map_priv__clone(tmpl_priv);
1574                         if (!priv)
1575                                 return ERR_PTR(-ENOMEM);
1576
1577                         err = bpf_map__set_priv(map, priv, bpf_map_priv__clear);
1578                         if (err) {
1579                                 bpf_map_priv__clear(map, priv);
1580                                 return ERR_PTR(err);
1581                         }
1582                 } else if (evsel) {
1583                         struct bpf_map_op *op;
1584
1585                         op = bpf_map__add_newop(map, NULL);
1586                         if (IS_ERR(op))
1587                                 return ERR_CAST(op);
1588                         op->op_type = BPF_MAP_OP_SET_EVSEL;
1589                         op->v.evsel = evsel;
1590                 }
1591         }
1592
1593         return evsel;
1594 }
1595
1596 int bpf__setup_stdout(struct evlist *evlist)
1597 {
1598         struct evsel *evsel = bpf__setup_output_event(evlist, "__bpf_stdout__");
1599         return PTR_ERR_OR_ZERO(evsel);
1600 }
1601
1602 #define ERRNO_OFFSET(e)         ((e) - __BPF_LOADER_ERRNO__START)
1603 #define ERRCODE_OFFSET(c)       ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
1604 #define NR_ERRNO        (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
1605
1606 static const char *bpf_loader_strerror_table[NR_ERRNO] = {
1607         [ERRCODE_OFFSET(CONFIG)]        = "Invalid config string",
1608         [ERRCODE_OFFSET(GROUP)]         = "Invalid group name",
1609         [ERRCODE_OFFSET(EVENTNAME)]     = "No event name found in config string",
1610         [ERRCODE_OFFSET(INTERNAL)]      = "BPF loader internal error",
1611         [ERRCODE_OFFSET(COMPILE)]       = "Error when compiling BPF scriptlet",
1612         [ERRCODE_OFFSET(PROGCONF_TERM)] = "Invalid program config term in config string",
1613         [ERRCODE_OFFSET(PROLOGUE)]      = "Failed to generate prologue",
1614         [ERRCODE_OFFSET(PROLOGUE2BIG)]  = "Prologue too big for program",
1615         [ERRCODE_OFFSET(PROLOGUEOOB)]   = "Offset out of bound for prologue",
1616         [ERRCODE_OFFSET(OBJCONF_OPT)]   = "Invalid object config option",
1617         [ERRCODE_OFFSET(OBJCONF_CONF)]  = "Config value not set (missing '=')",
1618         [ERRCODE_OFFSET(OBJCONF_MAP_OPT)]       = "Invalid object map config option",
1619         [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST)]  = "Target map doesn't exist",
1620         [ERRCODE_OFFSET(OBJCONF_MAP_VALUE)]     = "Incorrect value type for map",
1621         [ERRCODE_OFFSET(OBJCONF_MAP_TYPE)]      = "Incorrect map type",
1622         [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE)]   = "Incorrect map key size",
1623         [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE)] = "Incorrect map value size",
1624         [ERRCODE_OFFSET(OBJCONF_MAP_NOEVT)]     = "Event not found for map setting",
1625         [ERRCODE_OFFSET(OBJCONF_MAP_MAPSIZE)]   = "Invalid map size for event setting",
1626         [ERRCODE_OFFSET(OBJCONF_MAP_EVTDIM)]    = "Event dimension too large",
1627         [ERRCODE_OFFSET(OBJCONF_MAP_EVTINH)]    = "Doesn't support inherit event",
1628         [ERRCODE_OFFSET(OBJCONF_MAP_EVTTYPE)]   = "Wrong event type for map",
1629         [ERRCODE_OFFSET(OBJCONF_MAP_IDX2BIG)]   = "Index too large",
1630 };
1631
1632 static int
1633 bpf_loader_strerror(int err, char *buf, size_t size)
1634 {
1635         char sbuf[STRERR_BUFSIZE];
1636         const char *msg;
1637
1638         if (!buf || !size)
1639                 return -1;
1640
1641         err = err > 0 ? err : -err;
1642
1643         if (err >= __LIBBPF_ERRNO__START)
1644                 return libbpf_strerror(err, buf, size);
1645
1646         if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
1647                 msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
1648                 snprintf(buf, size, "%s", msg);
1649                 buf[size - 1] = '\0';
1650                 return 0;
1651         }
1652
1653         if (err >= __BPF_LOADER_ERRNO__END)
1654                 snprintf(buf, size, "Unknown bpf loader error %d", err);
1655         else
1656                 snprintf(buf, size, "%s",
1657                          str_error_r(err, sbuf, sizeof(sbuf)));
1658
1659         buf[size - 1] = '\0';
1660         return -1;
1661 }
1662
1663 #define bpf__strerror_head(err, buf, size) \
1664         char sbuf[STRERR_BUFSIZE], *emsg;\
1665         if (!size)\
1666                 return 0;\
1667         if (err < 0)\
1668                 err = -err;\
1669         bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
1670         emsg = sbuf;\
1671         switch (err) {\
1672         default:\
1673                 scnprintf(buf, size, "%s", emsg);\
1674                 break;
1675
1676 #define bpf__strerror_entry(val, fmt...)\
1677         case val: {\
1678                 scnprintf(buf, size, fmt);\
1679                 break;\
1680         }
1681
1682 #define bpf__strerror_end(buf, size)\
1683         }\
1684         buf[size - 1] = '\0';
1685
1686 int bpf__strerror_prepare_load(const char *filename, bool source,
1687                                int err, char *buf, size_t size)
1688 {
1689         size_t n;
1690         int ret;
1691
1692         n = snprintf(buf, size, "Failed to load %s%s: ",
1693                          filename, source ? " from source" : "");
1694         if (n >= size) {
1695                 buf[size - 1] = '\0';
1696                 return 0;
1697         }
1698         buf += n;
1699         size -= n;
1700
1701         ret = bpf_loader_strerror(err, buf, size);
1702         buf[size - 1] = '\0';
1703         return ret;
1704 }
1705
1706 int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
1707                         int err, char *buf, size_t size)
1708 {
1709         bpf__strerror_head(err, buf, size);
1710         case BPF_LOADER_ERRNO__PROGCONF_TERM: {
1711                 scnprintf(buf, size, "%s (add -v to see detail)", emsg);
1712                 break;
1713         }
1714         bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
1715         bpf__strerror_entry(EACCES, "You need to be root");
1716         bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
1717         bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
1718         bpf__strerror_end(buf, size);
1719         return 0;
1720 }
1721
1722 int bpf__strerror_load(struct bpf_object *obj,
1723                        int err, char *buf, size_t size)
1724 {
1725         bpf__strerror_head(err, buf, size);
1726         case LIBBPF_ERRNO__KVER: {
1727                 unsigned int obj_kver = bpf_object__kversion(obj);
1728                 unsigned int real_kver;
1729
1730                 if (fetch_kernel_version(&real_kver, NULL, 0)) {
1731                         scnprintf(buf, size, "Unable to fetch kernel version");
1732                         break;
1733                 }
1734
1735                 if (obj_kver != real_kver) {
1736                         scnprintf(buf, size,
1737                                   "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
1738                                   KVER_PARAM(obj_kver),
1739                                   KVER_PARAM(real_kver));
1740                         break;
1741                 }
1742
1743                 scnprintf(buf, size, "Failed to load program for unknown reason");
1744                 break;
1745         }
1746         bpf__strerror_end(buf, size);
1747         return 0;
1748 }
1749
1750 int bpf__strerror_config_obj(struct bpf_object *obj __maybe_unused,
1751                              struct parse_events_term *term __maybe_unused,
1752                              struct evlist *evlist __maybe_unused,
1753                              int *error_pos __maybe_unused, int err,
1754                              char *buf, size_t size)
1755 {
1756         bpf__strerror_head(err, buf, size);
1757         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE,
1758                             "Can't use this config term with this map type");
1759         bpf__strerror_end(buf, size);
1760         return 0;
1761 }
1762
1763 int bpf__strerror_apply_obj_config(int err, char *buf, size_t size)
1764 {
1765         bpf__strerror_head(err, buf, size);
1766         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM,
1767                             "Cannot set event to BPF map in multi-thread tracing");
1768         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH,
1769                             "%s (Hint: use -i to turn off inherit)", emsg);
1770         bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE,
1771                             "Can only put raw, hardware and BPF output event into a BPF map");
1772         bpf__strerror_end(buf, size);
1773         return 0;
1774 }
1775
1776 int bpf__strerror_setup_output_event(struct evlist *evlist __maybe_unused,
1777                                      int err, char *buf, size_t size)
1778 {
1779         bpf__strerror_head(err, buf, size);
1780         bpf__strerror_end(buf, size);
1781         return 0;
1782 }