tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  * Copyright (C) 2019 Isovalent, Inc.
  11  */
  12
  13 #ifndef _GNU_SOURCE
  14 #define _GNU_SOURCE
  15 #endif
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <stdarg.h>
  19 #include <libgen.h>
  20 #include <inttypes.h>
  21 #include <string.h>
  22 #include <unistd.h>
  23 #include <endian.h>
  24 #include <fcntl.h>
  25 #include <errno.h>
  26 #include <asm/unistd.h>
  27 #include <linux/err.h>
  28 #include <linux/kernel.h>
  29 #include <linux/bpf.h>
  30 #include <linux/btf.h>
  31 #include <linux/filter.h>
  32 #include <linux/list.h>
  33 #include <linux/limits.h>
  34 #include <linux/perf_event.h>
  35 #include <linux/ring_buffer.h>
  36 #include <linux/version.h>
  37 #include <sys/epoll.h>
  38 #include <sys/ioctl.h>
  39 #include <sys/mman.h>
  40 #include <sys/stat.h>
  41 #include <sys/types.h>
  42 #include <sys/vfs.h>
  43 #include <sys/utsname.h>
  44 #include <tools/libc_compat.h>
  45 #include <libelf.h>
  46 #include <gelf.h>
  47
  48 #include "libbpf.h"
  49 #include "bpf.h"
  50 #include "btf.h"
  51 #include "str_error.h"
  52 #include "libbpf_internal.h"
  53 #include "hashmap.h"
  54
  55 #ifndef EM_BPF
  56 #define EM_BPF 247
  57 #endif
  58
  59 #ifndef BPF_FS_MAGIC
  60 #define BPF_FS_MAGIC            0xcafe4a11
  61 #endif
  62
  63 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
  64  * compilation if user enables corresponding warning. Disable it explicitly.
  65  */
  66 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
  67
  68 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  69
  70 static int __base_pr(enum libbpf_print_level level, const char *format,
  71                      va_list args)
  72 {
  73         if (level == LIBBPF_DEBUG)
  74                 return 0;
  75
  76         return vfprintf(stderr, format, args);
  77 }
  78
  79 static libbpf_print_fn_t __libbpf_pr = __base_pr;
  80
  81 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
  82 {
  83         libbpf_print_fn_t old_print_fn = __libbpf_pr;
  84
  85         __libbpf_pr = fn;
  86         return old_print_fn;
  87 }
  88
  89 __printf(2, 3)
  90 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
  91 {
  92         va_list args;
  93
  94         if (!__libbpf_pr)
  95                 return;
  96
  97         va_start(args, format);
  98         __libbpf_pr(level, format, args);
  99         va_end(args);
 100 }
 101
 102 #define STRERR_BUFSIZE  128
 103
 104 /* Copied from tools/perf/util/util.h */
 105 #ifndef zfree
 106 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 107 #endif
 108
 109 #ifndef zclose
 110 # define zclose(fd) ({                  \
 111         int ___err = 0;                 \
 112         if ((fd) >= 0)                  \
 113                 ___err = close((fd));   \
 114         fd = -1;                        \
 115         ___err; })
 116 #endif
 117
 118 #ifdef HAVE_LIBELF_MMAP_SUPPORT
 119 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
 120 #else
 121 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
 122 #endif
 123
 124 static inline __u64 ptr_to_u64(const void *ptr)
 125 {
 126         return (__u64) (unsigned long) ptr;
 127 }
 128
 129 struct bpf_capabilities {
 130         /* v4.14: kernel support for program & map names. */
 131         __u32 name:1;
 132         /* v5.2: kernel support for global data sections. */
 133         __u32 global_data:1;
 134         /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
 135         __u32 btf_func:1;
 136         /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
 137         __u32 btf_datasec:1;
 138         /* BPF_F_MMAPABLE is supported for arrays */
 139         __u32 array_mmap:1;
 140 };
 141
 142 /*
 143  * bpf_prog should be a better name but it has been used in
 144  * linux/filter.h.
 145  */
 146 struct bpf_program {
 147         /* Index in elf obj file, for relocation use. */
 148         int idx;
 149         char *name;
 150         int prog_ifindex;
 151         char *section_name;
 152         /* section_name with / replaced by _; makes recursive pinning
 153          * in bpf_object__pin_programs easier
 154          */
 155         char *pin_name;
 156         struct bpf_insn *insns;
 157         size_t insns_cnt, main_prog_cnt;
 158         enum bpf_prog_type type;
 159
 160         struct reloc_desc {
 161                 enum {
 162                         RELO_LD64,
 163                         RELO_CALL,
 164                         RELO_DATA,
 165                 } type;
 166                 int insn_idx;
 167                 int map_idx;
 168                 int sym_off;
 169         } *reloc_desc;
 170         int nr_reloc;
 171         int log_level;
 172
 173         struct {
 174                 int nr;
 175                 int *fds;
 176         } instances;
 177         bpf_program_prep_t preprocessor;
 178
 179         struct bpf_object *obj;
 180         void *priv;
 181         bpf_program_clear_priv_t clear_priv;
 182
 183         enum bpf_attach_type expected_attach_type;
 184         __u32 attach_btf_id;
 185         __u32 attach_prog_fd;
 186         void *func_info;
 187         __u32 func_info_rec_size;
 188         __u32 func_info_cnt;
 189
 190         struct bpf_capabilities *caps;
 191
 192         void *line_info;
 193         __u32 line_info_rec_size;
 194         __u32 line_info_cnt;
 195         __u32 prog_flags;
 196 };
 197
 198 enum libbpf_map_type {
 199         LIBBPF_MAP_UNSPEC,
 200         LIBBPF_MAP_DATA,
 201         LIBBPF_MAP_BSS,
 202         LIBBPF_MAP_RODATA,
 203 };
 204
 205 static const char * const libbpf_type_to_btf_name[] = {
 206         [LIBBPF_MAP_DATA]       = ".data",
 207         [LIBBPF_MAP_BSS]        = ".bss",
 208         [LIBBPF_MAP_RODATA]     = ".rodata",
 209 };
 210
 211 struct bpf_map {
 212         char *name;
 213         int fd;
 214         int sec_idx;
 215         size_t sec_offset;
 216         int map_ifindex;
 217         int inner_map_fd;
 218         struct bpf_map_def def;
 219         __u32 btf_key_type_id;
 220         __u32 btf_value_type_id;
 221         void *priv;
 222         bpf_map_clear_priv_t clear_priv;
 223         enum libbpf_map_type libbpf_type;
 224         void *mmaped;
 225         char *pin_path;
 226         bool pinned;
 227         bool reused;
 228 };
 229
 230 static LIST_HEAD(bpf_objects_list);
 231
 232 struct bpf_object {
 233         char name[BPF_OBJ_NAME_LEN];
 234         char license[64];
 235         __u32 kern_version;
 236
 237         struct bpf_program *programs;
 238         size_t nr_programs;
 239         struct bpf_map *maps;
 240         size_t nr_maps;
 241         size_t maps_cap;
 242
 243         bool loaded;
 244         bool has_pseudo_calls;
 245         bool relaxed_core_relocs;
 246
 247         /*
 248          * Information when doing elf related work. Only valid if fd
 249          * is valid.
 250          */
 251         struct {
 252                 int fd;
 253                 const void *obj_buf;
 254                 size_t obj_buf_sz;
 255                 Elf *elf;
 256                 GElf_Ehdr ehdr;
 257                 Elf_Data *symbols;
 258                 Elf_Data *data;
 259                 Elf_Data *rodata;
 260                 Elf_Data *bss;
 261                 size_t strtabidx;
 262                 struct {
 263                         GElf_Shdr shdr;
 264                         Elf_Data *data;
 265                 } *reloc_sects;
 266                 int nr_reloc_sects;
 267                 int maps_shndx;
 268                 int btf_maps_shndx;
 269                 int text_shndx;
 270                 int data_shndx;
 271                 int rodata_shndx;
 272                 int bss_shndx;
 273         } efile;
 274         /*
 275          * All loaded bpf_object is linked in a list, which is
 276          * hidden to caller. bpf_objects__<func> handlers deal with
 277          * all objects.
 278          */
 279         struct list_head list;
 280
 281         struct btf *btf;
 282         struct btf_ext *btf_ext;
 283
 284         void *priv;
 285         bpf_object_clear_priv_t clear_priv;
 286
 287         struct bpf_capabilities caps;
 288
 289         char path[];
 290 };
 291 #define obj_elf_valid(o)        ((o)->efile.elf)
 292
 293 void bpf_program__unload(struct bpf_program *prog)
 294 {
 295         int i;
 296
 297         if (!prog)
 298                 return;
 299
 300         /*
 301          * If the object is opened but the program was never loaded,
 302          * it is possible that prog->instances.nr == -1.
 303          */
 304         if (prog->instances.nr > 0) {
 305                 for (i = 0; i < prog->instances.nr; i++)
 306                         zclose(prog->instances.fds[i]);
 307         } else if (prog->instances.nr != -1) {
 308                 pr_warn("Internal error: instances.nr is %d\n",
 309                         prog->instances.nr);
 310         }
 311
 312         prog->instances.nr = -1;
 313         zfree(&prog->instances.fds);
 314
 315         zfree(&prog->func_info);
 316         zfree(&prog->line_info);
 317 }
 318
 319 static void bpf_program__exit(struct bpf_program *prog)
 320 {
 321         if (!prog)
 322                 return;
 323
 324         if (prog->clear_priv)
 325                 prog->clear_priv(prog, prog->priv);
 326
 327         prog->priv = NULL;
 328         prog->clear_priv = NULL;
 329
 330         bpf_program__unload(prog);
 331         zfree(&prog->name);
 332         zfree(&prog->section_name);
 333         zfree(&prog->pin_name);
 334         zfree(&prog->insns);
 335         zfree(&prog->reloc_desc);
 336
 337         prog->nr_reloc = 0;
 338         prog->insns_cnt = 0;
 339         prog->idx = -1;
 340 }
 341
 342 static char *__bpf_program__pin_name(struct bpf_program *prog)
 343 {
 344         char *name, *p;
 345
 346         name = p = strdup(prog->section_name);
 347         while ((p = strchr(p, '/')))
 348                 *p = '_';
 349
 350         return name;
 351 }
 352
 353 static int
 354 bpf_program__init(void *data, size_t size, char *section_name, int idx,
 355                   struct bpf_program *prog)
 356 {
 357         const size_t bpf_insn_sz = sizeof(struct bpf_insn);
 358
 359         if (size == 0 || size % bpf_insn_sz) {
 360                 pr_warn("corrupted section '%s', size: %zu\n",
 361                         section_name, size);
 362                 return -EINVAL;
 363         }
 364
 365         memset(prog, 0, sizeof(*prog));
 366
 367         prog->section_name = strdup(section_name);
 368         if (!prog->section_name) {
 369                 pr_warn("failed to alloc name for prog under section(%d) %s\n",
 370                         idx, section_name);
 371                 goto errout;
 372         }
 373
 374         prog->pin_name = __bpf_program__pin_name(prog);
 375         if (!prog->pin_name) {
 376                 pr_warn("failed to alloc pin name for prog under section(%d) %s\n",
 377                         idx, section_name);
 378                 goto errout;
 379         }
 380
 381         prog->insns = malloc(size);
 382         if (!prog->insns) {
 383                 pr_warn("failed to alloc insns for prog under section %s\n",
 384                         section_name);
 385                 goto errout;
 386         }
 387         prog->insns_cnt = size / bpf_insn_sz;
 388         memcpy(prog->insns, data, size);
 389         prog->idx = idx;
 390         prog->instances.fds = NULL;
 391         prog->instances.nr = -1;
 392         prog->type = BPF_PROG_TYPE_UNSPEC;
 393
 394         return 0;
 395 errout:
 396         bpf_program__exit(prog);
 397         return -ENOMEM;
 398 }
 399
 400 static int
 401 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
 402                         char *section_name, int idx)
 403 {
 404         struct bpf_program prog, *progs;
 405         int nr_progs, err;
 406
 407         err = bpf_program__init(data, size, section_name, idx, &prog);
 408         if (err)
 409                 return err;
 410
 411         prog.caps = &obj->caps;
 412         progs = obj->programs;
 413         nr_progs = obj->nr_programs;
 414
 415         progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
 416         if (!progs) {
 417                 /*
 418                  * In this case the original obj->programs
 419                  * is still valid, so don't need special treat for
 420                  * bpf_close_object().
 421                  */
 422                 pr_warn("failed to alloc a new program under section '%s'\n",
 423                         section_name);
 424                 bpf_program__exit(&prog);
 425                 return -ENOMEM;
 426         }
 427
 428         pr_debug("found program %s\n", prog.section_name);
 429         obj->programs = progs;
 430         obj->nr_programs = nr_progs + 1;
 431         prog.obj = obj;
 432         progs[nr_progs] = prog;
 433         return 0;
 434 }
 435
 436 static int
 437 bpf_object__init_prog_names(struct bpf_object *obj)
 438 {
 439         Elf_Data *symbols = obj->efile.symbols;
 440         struct bpf_program *prog;
 441         size_t pi, si;
 442
 443         for (pi = 0; pi < obj->nr_programs; pi++) {
 444                 const char *name = NULL;
 445
 446                 prog = &obj->programs[pi];
 447
 448                 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
 449                      si++) {
 450                         GElf_Sym sym;
 451
 452                         if (!gelf_getsym(symbols, si, &sym))
 453                                 continue;
 454                         if (sym.st_shndx != prog->idx)
 455                                 continue;
 456                         if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
 457                                 continue;
 458
 459                         name = elf_strptr(obj->efile.elf,
 460                                           obj->efile.strtabidx,
 461                                           sym.st_name);
 462                         if (!name) {
 463                                 pr_warn("failed to get sym name string for prog %s\n",
 464                                         prog->section_name);
 465                                 return -LIBBPF_ERRNO__LIBELF;
 466                         }
 467                 }
 468
 469                 if (!name && prog->idx == obj->efile.text_shndx)
 470                         name = ".text";
 471
 472                 if (!name) {
 473                         pr_warn("failed to find sym for prog %s\n",
 474                                 prog->section_name);
 475                         return -EINVAL;
 476                 }
 477
 478                 prog->name = strdup(name);
 479                 if (!prog->name) {
 480                         pr_warn("failed to allocate memory for prog sym %s\n",
 481                                 name);
 482                         return -ENOMEM;
 483                 }
 484         }
 485
 486         return 0;
 487 }
 488
 489 static __u32 get_kernel_version(void)
 490 {
 491         __u32 major, minor, patch;
 492         struct utsname info;
 493
 494         uname(&info);
 495         if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
 496                 return 0;
 497         return KERNEL_VERSION(major, minor, patch);
 498 }
 499
 500 static struct bpf_object *bpf_object__new(const char *path,
 501                                           const void *obj_buf,
 502                                           size_t obj_buf_sz,
 503                                           const char *obj_name)
 504 {
 505         struct bpf_object *obj;
 506         char *end;
 507
 508         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
 509         if (!obj) {
 510                 pr_warn("alloc memory failed for %s\n", path);
 511                 return ERR_PTR(-ENOMEM);
 512         }
 513
 514         strcpy(obj->path, path);
 515         if (obj_name) {
 516                 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
 517                 obj->name[sizeof(obj->name) - 1] = 0;
 518         } else {
 519                 /* Using basename() GNU version which doesn't modify arg. */
 520                 strncpy(obj->name, basename((void *)path),
 521                         sizeof(obj->name) - 1);
 522                 end = strchr(obj->name, '.');
 523                 if (end)
 524                         *end = 0;
 525         }
 526
 527         obj->efile.fd = -1;
 528         /*
 529          * Caller of this function should also call
 530          * bpf_object__elf_finish() after data collection to return
 531          * obj_buf to user. If not, we should duplicate the buffer to
 532          * avoid user freeing them before elf finish.
 533          */
 534         obj->efile.obj_buf = obj_buf;
 535         obj->efile.obj_buf_sz = obj_buf_sz;
 536         obj->efile.maps_shndx = -1;
 537         obj->efile.btf_maps_shndx = -1;
 538         obj->efile.data_shndx = -1;
 539         obj->efile.rodata_shndx = -1;
 540         obj->efile.bss_shndx = -1;
 541
 542         obj->kern_version = get_kernel_version();
 543         obj->loaded = false;
 544
 545         INIT_LIST_HEAD(&obj->list);
 546         list_add(&obj->list, &bpf_objects_list);
 547         return obj;
 548 }
 549
 550 static void bpf_object__elf_finish(struct bpf_object *obj)
 551 {
 552         if (!obj_elf_valid(obj))
 553                 return;
 554
 555         if (obj->efile.elf) {
 556                 elf_end(obj->efile.elf);
 557                 obj->efile.elf = NULL;
 558         }
 559         obj->efile.symbols = NULL;
 560         obj->efile.data = NULL;
 561         obj->efile.rodata = NULL;
 562         obj->efile.bss = NULL;
 563
 564         zfree(&obj->efile.reloc_sects);
 565         obj->efile.nr_reloc_sects = 0;
 566         zclose(obj->efile.fd);
 567         obj->efile.obj_buf = NULL;
 568         obj->efile.obj_buf_sz = 0;
 569 }
 570
 571 static int bpf_object__elf_init(struct bpf_object *obj)
 572 {
 573         int err = 0;
 574         GElf_Ehdr *ep;
 575
 576         if (obj_elf_valid(obj)) {
 577                 pr_warn("elf init: internal error\n");
 578                 return -LIBBPF_ERRNO__LIBELF;
 579         }
 580
 581         if (obj->efile.obj_buf_sz > 0) {
 582                 /*
 583                  * obj_buf should have been validated by
 584                  * bpf_object__open_buffer().
 585                  */
 586                 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
 587                                             obj->efile.obj_buf_sz);
 588         } else {
 589                 obj->efile.fd = open(obj->path, O_RDONLY);
 590                 if (obj->efile.fd < 0) {
 591                         char errmsg[STRERR_BUFSIZE], *cp;
 592
 593                         err = -errno;
 594                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
 595                         pr_warn("failed to open %s: %s\n", obj->path, cp);
 596                         return err;
 597                 }
 598
 599                 obj->efile.elf = elf_begin(obj->efile.fd,
 600                                            LIBBPF_ELF_C_READ_MMAP, NULL);
 601         }
 602
 603         if (!obj->efile.elf) {
 604                 pr_warn("failed to open %s as ELF file\n", obj->path);
 605                 err = -LIBBPF_ERRNO__LIBELF;
 606                 goto errout;
 607         }
 608
 609         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
 610                 pr_warn("failed to get EHDR from %s\n", obj->path);
 611                 err = -LIBBPF_ERRNO__FORMAT;
 612                 goto errout;
 613         }
 614         ep = &obj->efile.ehdr;
 615
 616         /* Old LLVM set e_machine to EM_NONE */
 617         if (ep->e_type != ET_REL ||
 618             (ep->e_machine && ep->e_machine != EM_BPF)) {
 619                 pr_warn("%s is not an eBPF object file\n", obj->path);
 620                 err = -LIBBPF_ERRNO__FORMAT;
 621                 goto errout;
 622         }
 623
 624         return 0;
 625 errout:
 626         bpf_object__elf_finish(obj);
 627         return err;
 628 }
 629
 630 static int bpf_object__check_endianness(struct bpf_object *obj)
 631 {
 632 #if __BYTE_ORDER == __LITTLE_ENDIAN
 633         if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
 634                 return 0;
 635 #elif __BYTE_ORDER == __BIG_ENDIAN
 636         if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
 637                 return 0;
 638 #else
 639 # error "Unrecognized __BYTE_ORDER__"
 640 #endif
 641         pr_warn("endianness mismatch.\n");
 642         return -LIBBPF_ERRNO__ENDIAN;
 643 }
 644
 645 static int
 646 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
 647 {
 648         memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
 649         pr_debug("license of %s is %s\n", obj->path, obj->license);
 650         return 0;
 651 }
 652
 653 static int
 654 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
 655 {
 656         __u32 kver;
 657
 658         if (size != sizeof(kver)) {
 659                 pr_warn("invalid kver section in %s\n", obj->path);
 660                 return -LIBBPF_ERRNO__FORMAT;
 661         }
 662         memcpy(&kver, data, sizeof(kver));
 663         obj->kern_version = kver;
 664         pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
 665         return 0;
 666 }
 667
 668 static int compare_bpf_map(const void *_a, const void *_b)
 669 {
 670         const struct bpf_map *a = _a;
 671         const struct bpf_map *b = _b;
 672
 673         if (a->sec_idx != b->sec_idx)
 674                 return a->sec_idx - b->sec_idx;
 675         return a->sec_offset - b->sec_offset;
 676 }
 677
 678 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
 679 {
 680         if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
 681             type == BPF_MAP_TYPE_HASH_OF_MAPS)
 682                 return true;
 683         return false;
 684 }
 685
 686 static int bpf_object_search_section_size(const struct bpf_object *obj,
 687                                           const char *name, size_t *d_size)
 688 {
 689         const GElf_Ehdr *ep = &obj->efile.ehdr;
 690         Elf *elf = obj->efile.elf;
 691         Elf_Scn *scn = NULL;
 692         int idx = 0;
 693
 694         while ((scn = elf_nextscn(elf, scn)) != NULL) {
 695                 const char *sec_name;
 696                 Elf_Data *data;
 697                 GElf_Shdr sh;
 698
 699                 idx++;
 700                 if (gelf_getshdr(scn, &sh) != &sh) {
 701                         pr_warn("failed to get section(%d) header from %s\n",
 702                                 idx, obj->path);
 703                         return -EIO;
 704                 }
 705
 706                 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
 707                 if (!sec_name) {
 708                         pr_warn("failed to get section(%d) name from %s\n",
 709                                 idx, obj->path);
 710                         return -EIO;
 711                 }
 712
 713                 if (strcmp(name, sec_name))
 714                         continue;
 715
 716                 data = elf_getdata(scn, 0);
 717                 if (!data) {
 718                         pr_warn("failed to get section(%d) data from %s(%s)\n",
 719                                 idx, name, obj->path);
 720                         return -EIO;
 721                 }
 722
 723                 *d_size = data->d_size;
 724                 return 0;
 725         }
 726
 727         return -ENOENT;
 728 }
 729
 730 int bpf_object__section_size(const struct bpf_object *obj, const char *name,
 731                              __u32 *size)
 732 {
 733         int ret = -ENOENT;
 734         size_t d_size;
 735
 736         *size = 0;
 737         if (!name) {
 738                 return -EINVAL;
 739         } else if (!strcmp(name, ".data")) {
 740                 if (obj->efile.data)
 741                         *size = obj->efile.data->d_size;
 742         } else if (!strcmp(name, ".bss")) {
 743                 if (obj->efile.bss)
 744                         *size = obj->efile.bss->d_size;
 745         } else if (!strcmp(name, ".rodata")) {
 746                 if (obj->efile.rodata)
 747                         *size = obj->efile.rodata->d_size;
 748         } else {
 749                 ret = bpf_object_search_section_size(obj, name, &d_size);
 750                 if (!ret)
 751                         *size = d_size;
 752         }
 753
 754         return *size ? 0 : ret;
 755 }
 756
 757 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
 758                                 __u32 *off)
 759 {
 760         Elf_Data *symbols = obj->efile.symbols;
 761         const char *sname;
 762         size_t si;
 763
 764         if (!name || !off)
 765                 return -EINVAL;
 766
 767         for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
 768                 GElf_Sym sym;
 769
 770                 if (!gelf_getsym(symbols, si, &sym))
 771                         continue;
 772                 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
 773                     GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
 774                         continue;
 775
 776                 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
 777                                    sym.st_name);
 778                 if (!sname) {
 779                         pr_warn("failed to get sym name string for var %s\n",
 780                                 name);
 781                         return -EIO;
 782                 }
 783                 if (strcmp(name, sname) == 0) {
 784                         *off = sym.st_value;
 785                         return 0;
 786                 }
 787         }
 788
 789         return -ENOENT;
 790 }
 791
 792 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
 793 {
 794         struct bpf_map *new_maps;
 795         size_t new_cap;
 796         int i;
 797
 798         if (obj->nr_maps < obj->maps_cap)
 799                 return &obj->maps[obj->nr_maps++];
 800
 801         new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
 802         new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps));
 803         if (!new_maps) {
 804                 pr_warn("alloc maps for object failed\n");
 805                 return ERR_PTR(-ENOMEM);
 806         }
 807
 808         obj->maps_cap = new_cap;
 809         obj->maps = new_maps;
 810
 811         /* zero out new maps */
 812         memset(obj->maps + obj->nr_maps, 0,
 813                (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
 814         /*
 815          * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
 816          * when failure (zclose won't close negative fd)).
 817          */
 818         for (i = obj->nr_maps; i < obj->maps_cap; i++) {
 819                 obj->maps[i].fd = -1;
 820                 obj->maps[i].inner_map_fd = -1;
 821         }
 822
 823         return &obj->maps[obj->nr_maps++];
 824 }
 825
 826 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
 827 {
 828         long page_sz = sysconf(_SC_PAGE_SIZE);
 829         size_t map_sz;
 830
 831         map_sz = roundup(map->def.value_size, 8) * map->def.max_entries;
 832         map_sz = roundup(map_sz, page_sz);
 833         return map_sz;
 834 }
 835
 836 static int
 837 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
 838                               int sec_idx, void *data, size_t data_sz)
 839 {
 840         char map_name[BPF_OBJ_NAME_LEN];
 841         struct bpf_map_def *def;
 842         struct bpf_map *map;
 843         int err;
 844
 845         map = bpf_object__add_map(obj);
 846         if (IS_ERR(map))
 847                 return PTR_ERR(map);
 848
 849         map->libbpf_type = type;
 850         map->sec_idx = sec_idx;
 851         map->sec_offset = 0;
 852         snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name,
 853                  libbpf_type_to_btf_name[type]);
 854         map->name = strdup(map_name);
 855         if (!map->name) {
 856                 pr_warn("failed to alloc map name\n");
 857                 return -ENOMEM;
 858         }
 859
 860         def = &map->def;
 861         def->type = BPF_MAP_TYPE_ARRAY;
 862         def->key_size = sizeof(int);
 863         def->value_size = data_sz;
 864         def->max_entries = 1;
 865         def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0;
 866         def->map_flags |= BPF_F_MMAPABLE;
 867
 868         pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
 869                  map_name, map->sec_idx, map->sec_offset, def->map_flags);
 870
 871         map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
 872                            MAP_SHARED | MAP_ANONYMOUS, -1, 0);
 873         if (map->mmaped == MAP_FAILED) {
 874                 err = -errno;
 875                 map->mmaped = NULL;
 876                 pr_warn("failed to alloc map '%s' content buffer: %d\n",
 877                         map->name, err);
 878                 zfree(&map->name);
 879                 return err;
 880         }
 881
 882         if (type != LIBBPF_MAP_BSS)
 883                 memcpy(map->mmaped, data, data_sz);
 884
 885         pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
 886         return 0;
 887 }
 888
 889 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
 890 {
 891         int err;
 892
 893         /*
 894          * Populate obj->maps with libbpf internal maps.
 895          */
 896         if (obj->efile.data_shndx >= 0) {
 897                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
 898                                                     obj->efile.data_shndx,
 899                                                     obj->efile.data->d_buf,
 900                                                     obj->efile.data->d_size);
 901                 if (err)
 902                         return err;
 903         }
 904         if (obj->efile.rodata_shndx >= 0) {
 905                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
 906                                                     obj->efile.rodata_shndx,
 907                                                     obj->efile.rodata->d_buf,
 908                                                     obj->efile.rodata->d_size);
 909                 if (err)
 910                         return err;
 911         }
 912         if (obj->efile.bss_shndx >= 0) {
 913                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
 914                                                     obj->efile.bss_shndx,
 915                                                     NULL,
 916                                                     obj->efile.bss->d_size);
 917                 if (err)
 918                         return err;
 919         }
 920         return 0;
 921 }
 922
 923 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
 924 {
 925         Elf_Data *symbols = obj->efile.symbols;
 926         int i, map_def_sz = 0, nr_maps = 0, nr_syms;
 927         Elf_Data *data = NULL;
 928         Elf_Scn *scn;
 929
 930         if (obj->efile.maps_shndx < 0)
 931                 return 0;
 932
 933         if (!symbols)
 934                 return -EINVAL;
 935
 936         scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
 937         if (scn)
 938                 data = elf_getdata(scn, NULL);
 939         if (!scn || !data) {
 940                 pr_warn("failed to get Elf_Data from map section %d\n",
 941                         obj->efile.maps_shndx);
 942                 return -EINVAL;
 943         }
 944
 945         /*
 946          * Count number of maps. Each map has a name.
 947          * Array of maps is not supported: only the first element is
 948          * considered.
 949          *
 950          * TODO: Detect array of map and report error.
 951          */
 952         nr_syms = symbols->d_size / sizeof(GElf_Sym);
 953         for (i = 0; i < nr_syms; i++) {
 954                 GElf_Sym sym;
 955
 956                 if (!gelf_getsym(symbols, i, &sym))
 957                         continue;
 958                 if (sym.st_shndx != obj->efile.maps_shndx)
 959                         continue;
 960                 nr_maps++;
 961         }
 962         /* Assume equally sized map definitions */
 963         pr_debug("maps in %s: %d maps in %zd bytes\n",
 964                  obj->path, nr_maps, data->d_size);
 965
 966         if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
 967                 pr_warn("unable to determine map definition size section %s, %d maps in %zd bytes\n",
 968                         obj->path, nr_maps, data->d_size);
 969                 return -EINVAL;
 970         }
 971         map_def_sz = data->d_size / nr_maps;
 972
 973         /* Fill obj->maps using data in "maps" section.  */
 974         for (i = 0; i < nr_syms; i++) {
 975                 GElf_Sym sym;
 976                 const char *map_name;
 977                 struct bpf_map_def *def;
 978                 struct bpf_map *map;
 979
 980                 if (!gelf_getsym(symbols, i, &sym))
 981                         continue;
 982                 if (sym.st_shndx != obj->efile.maps_shndx)
 983                         continue;
 984
 985                 map = bpf_object__add_map(obj);
 986                 if (IS_ERR(map))
 987                         return PTR_ERR(map);
 988
 989                 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
 990                                       sym.st_name);
 991                 if (!map_name) {
 992                         pr_warn("failed to get map #%d name sym string for obj %s\n",
 993                                 i, obj->path);
 994                         return -LIBBPF_ERRNO__FORMAT;
 995                 }
 996
 997                 map->libbpf_type = LIBBPF_MAP_UNSPEC;
 998                 map->sec_idx = sym.st_shndx;
 999                 map->sec_offset = sym.st_value;
1000                 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1001                          map_name, map->sec_idx, map->sec_offset);
1002                 if (sym.st_value + map_def_sz > data->d_size) {
1003                         pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1004                                 obj->path, map_name);
1005                         return -EINVAL;
1006                 }
1007
1008                 map->name = strdup(map_name);
1009                 if (!map->name) {
1010                         pr_warn("failed to alloc map name\n");
1011                         return -ENOMEM;
1012                 }
1013                 pr_debug("map %d is \"%s\"\n", i, map->name);
1014                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1015                 /*
1016                  * If the definition of the map in the object file fits in
1017                  * bpf_map_def, copy it.  Any extra fields in our version
1018                  * of bpf_map_def will default to zero as a result of the
1019                  * calloc above.
1020                  */
1021                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1022                         memcpy(&map->def, def, map_def_sz);
1023                 } else {
1024                         /*
1025                          * Here the map structure being read is bigger than what
1026                          * we expect, truncate if the excess bits are all zero.
1027                          * If they are not zero, reject this map as
1028                          * incompatible.
1029                          */
1030                         char *b;
1031
1032                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
1033                              b < ((char *)def) + map_def_sz; b++) {
1034                                 if (*b != 0) {
1035                                         pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1036                                                 obj->path, map_name);
1037                                         if (strict)
1038                                                 return -EINVAL;
1039                                 }
1040                         }
1041                         memcpy(&map->def, def, sizeof(struct bpf_map_def));
1042                 }
1043         }
1044         return 0;
1045 }
1046
1047 static const struct btf_type *
1048 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1049 {
1050         const struct btf_type *t = btf__type_by_id(btf, id);
1051
1052         if (res_id)
1053                 *res_id = id;
1054
1055         while (btf_is_mod(t) || btf_is_typedef(t)) {
1056                 if (res_id)
1057                         *res_id = t->type;
1058                 t = btf__type_by_id(btf, t->type);
1059         }
1060
1061         return t;
1062 }
1063
1064 /*
1065  * Fetch integer attribute of BTF map definition. Such attributes are
1066  * represented using a pointer to an array, in which dimensionality of array
1067  * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
1068  * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
1069  * type definition, while using only sizeof(void *) space in ELF data section.
1070  */
1071 static bool get_map_field_int(const char *map_name, const struct btf *btf,
1072                               const struct btf_type *def,
1073                               const struct btf_member *m, __u32 *res)
1074 {
1075         const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1076         const char *name = btf__name_by_offset(btf, m->name_off);
1077         const struct btf_array *arr_info;
1078         const struct btf_type *arr_t;
1079
1080         if (!btf_is_ptr(t)) {
1081                 pr_warn("map '%s': attr '%s': expected PTR, got %u.\n",
1082                         map_name, name, btf_kind(t));
1083                 return false;
1084         }
1085
1086         arr_t = btf__type_by_id(btf, t->type);
1087         if (!arr_t) {
1088                 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1089                         map_name, name, t->type);
1090                 return false;
1091         }
1092         if (!btf_is_array(arr_t)) {
1093                 pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n",
1094                         map_name, name, btf_kind(arr_t));
1095                 return false;
1096         }
1097         arr_info = btf_array(arr_t);
1098         *res = arr_info->nelems;
1099         return true;
1100 }
1101
1102 static int build_map_pin_path(struct bpf_map *map, const char *path)
1103 {
1104         char buf[PATH_MAX];
1105         int err, len;
1106
1107         if (!path)
1108                 path = "/sys/fs/bpf";
1109
1110         len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
1111         if (len < 0)
1112                 return -EINVAL;
1113         else if (len >= PATH_MAX)
1114                 return -ENAMETOOLONG;
1115
1116         err = bpf_map__set_pin_path(map, buf);
1117         if (err)
1118                 return err;
1119
1120         return 0;
1121 }
1122
1123 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
1124                                          const struct btf_type *sec,
1125                                          int var_idx, int sec_idx,
1126                                          const Elf_Data *data, bool strict,
1127                                          const char *pin_root_path)
1128 {
1129         const struct btf_type *var, *def, *t;
1130         const struct btf_var_secinfo *vi;
1131         const struct btf_var *var_extra;
1132         const struct btf_member *m;
1133         const char *map_name;
1134         struct bpf_map *map;
1135         int vlen, i;
1136
1137         vi = btf_var_secinfos(sec) + var_idx;
1138         var = btf__type_by_id(obj->btf, vi->type);
1139         var_extra = btf_var(var);
1140         map_name = btf__name_by_offset(obj->btf, var->name_off);
1141         vlen = btf_vlen(var);
1142
1143         if (map_name == NULL || map_name[0] == '\0') {
1144                 pr_warn("map #%d: empty name.\n", var_idx);
1145                 return -EINVAL;
1146         }
1147         if ((__u64)vi->offset + vi->size > data->d_size) {
1148                 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
1149                 return -EINVAL;
1150         }
1151         if (!btf_is_var(var)) {
1152                 pr_warn("map '%s': unexpected var kind %u.\n",
1153                         map_name, btf_kind(var));
1154                 return -EINVAL;
1155         }
1156         if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
1157             var_extra->linkage != BTF_VAR_STATIC) {
1158                 pr_warn("map '%s': unsupported var linkage %u.\n",
1159                         map_name, var_extra->linkage);
1160                 return -EOPNOTSUPP;
1161         }
1162
1163         def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
1164         if (!btf_is_struct(def)) {
1165                 pr_warn("map '%s': unexpected def kind %u.\n",
1166                         map_name, btf_kind(var));
1167                 return -EINVAL;
1168         }
1169         if (def->size > vi->size) {
1170                 pr_warn("map '%s': invalid def size.\n", map_name);
1171                 return -EINVAL;
1172         }
1173
1174         map = bpf_object__add_map(obj);
1175         if (IS_ERR(map))
1176                 return PTR_ERR(map);
1177         map->name = strdup(map_name);
1178         if (!map->name) {
1179                 pr_warn("map '%s': failed to alloc map name.\n", map_name);
1180                 return -ENOMEM;
1181         }
1182         map->libbpf_type = LIBBPF_MAP_UNSPEC;
1183         map->def.type = BPF_MAP_TYPE_UNSPEC;
1184         map->sec_idx = sec_idx;
1185         map->sec_offset = vi->offset;
1186         pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
1187                  map_name, map->sec_idx, map->sec_offset);
1188
1189         vlen = btf_vlen(def);
1190         m = btf_members(def);
1191         for (i = 0; i < vlen; i++, m++) {
1192                 const char *name = btf__name_by_offset(obj->btf, m->name_off);
1193
1194                 if (!name) {
1195                         pr_warn("map '%s': invalid field #%d.\n", map_name, i);
1196                         return -EINVAL;
1197                 }
1198                 if (strcmp(name, "type") == 0) {
1199                         if (!get_map_field_int(map_name, obj->btf, def, m,
1200                                                &map->def.type))
1201                                 return -EINVAL;
1202                         pr_debug("map '%s': found type = %u.\n",
1203                                  map_name, map->def.type);
1204                 } else if (strcmp(name, "max_entries") == 0) {
1205                         if (!get_map_field_int(map_name, obj->btf, def, m,
1206                                                &map->def.max_entries))
1207                                 return -EINVAL;
1208                         pr_debug("map '%s': found max_entries = %u.\n",
1209                                  map_name, map->def.max_entries);
1210                 } else if (strcmp(name, "map_flags") == 0) {
1211                         if (!get_map_field_int(map_name, obj->btf, def, m,
1212                                                &map->def.map_flags))
1213                                 return -EINVAL;
1214                         pr_debug("map '%s': found map_flags = %u.\n",
1215                                  map_name, map->def.map_flags);
1216                 } else if (strcmp(name, "key_size") == 0) {
1217                         __u32 sz;
1218
1219                         if (!get_map_field_int(map_name, obj->btf, def, m,
1220                                                &sz))
1221                                 return -EINVAL;
1222                         pr_debug("map '%s': found key_size = %u.\n",
1223                                  map_name, sz);
1224                         if (map->def.key_size && map->def.key_size != sz) {
1225                                 pr_warn("map '%s': conflicting key size %u != %u.\n",
1226                                         map_name, map->def.key_size, sz);
1227                                 return -EINVAL;
1228                         }
1229                         map->def.key_size = sz;
1230                 } else if (strcmp(name, "key") == 0) {
1231                         __s64 sz;
1232
1233                         t = btf__type_by_id(obj->btf, m->type);
1234                         if (!t) {
1235                                 pr_warn("map '%s': key type [%d] not found.\n",
1236                                         map_name, m->type);
1237                                 return -EINVAL;
1238                         }
1239                         if (!btf_is_ptr(t)) {
1240                                 pr_warn("map '%s': key spec is not PTR: %u.\n",
1241                                         map_name, btf_kind(t));
1242                                 return -EINVAL;
1243                         }
1244                         sz = btf__resolve_size(obj->btf, t->type);
1245                         if (sz < 0) {
1246                                 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
1247                                         map_name, t->type, (ssize_t)sz);
1248                                 return sz;
1249                         }
1250                         pr_debug("map '%s': found key [%u], sz = %zd.\n",
1251                                  map_name, t->type, (ssize_t)sz);
1252                         if (map->def.key_size && map->def.key_size != sz) {
1253                                 pr_warn("map '%s': conflicting key size %u != %zd.\n",
1254                                         map_name, map->def.key_size, (ssize_t)sz);
1255                                 return -EINVAL;
1256                         }
1257                         map->def.key_size = sz;
1258                         map->btf_key_type_id = t->type;
1259                 } else if (strcmp(name, "value_size") == 0) {
1260                         __u32 sz;
1261
1262                         if (!get_map_field_int(map_name, obj->btf, def, m,
1263                                                &sz))
1264                                 return -EINVAL;
1265                         pr_debug("map '%s': found value_size = %u.\n",
1266                                  map_name, sz);
1267                         if (map->def.value_size && map->def.value_size != sz) {
1268                                 pr_warn("map '%s': conflicting value size %u != %u.\n",
1269                                         map_name, map->def.value_size, sz);
1270                                 return -EINVAL;
1271                         }
1272                         map->def.value_size = sz;
1273                 } else if (strcmp(name, "value") == 0) {
1274                         __s64 sz;
1275
1276                         t = btf__type_by_id(obj->btf, m->type);
1277                         if (!t) {
1278                                 pr_warn("map '%s': value type [%d] not found.\n",
1279                                         map_name, m->type);
1280                                 return -EINVAL;
1281                         }
1282                         if (!btf_is_ptr(t)) {
1283                                 pr_warn("map '%s': value spec is not PTR: %u.\n",
1284                                         map_name, btf_kind(t));
1285                                 return -EINVAL;
1286                         }
1287                         sz = btf__resolve_size(obj->btf, t->type);
1288                         if (sz < 0) {
1289                                 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
1290                                         map_name, t->type, (ssize_t)sz);
1291                                 return sz;
1292                         }
1293                         pr_debug("map '%s': found value [%u], sz = %zd.\n",
1294                                  map_name, t->type, (ssize_t)sz);
1295                         if (map->def.value_size && map->def.value_size != sz) {
1296                                 pr_warn("map '%s': conflicting value size %u != %zd.\n",
1297                                         map_name, map->def.value_size, (ssize_t)sz);
1298                                 return -EINVAL;
1299                         }
1300                         map->def.value_size = sz;
1301                         map->btf_value_type_id = t->type;
1302                 } else if (strcmp(name, "pinning") == 0) {
1303                         __u32 val;
1304                         int err;
1305
1306                         if (!get_map_field_int(map_name, obj->btf, def, m,
1307                                                &val))
1308                                 return -EINVAL;
1309                         pr_debug("map '%s': found pinning = %u.\n",
1310                                  map_name, val);
1311
1312                         if (val != LIBBPF_PIN_NONE &&
1313                             val != LIBBPF_PIN_BY_NAME) {
1314                                 pr_warn("map '%s': invalid pinning value %u.\n",
1315                                         map_name, val);
1316                                 return -EINVAL;
1317                         }
1318                         if (val == LIBBPF_PIN_BY_NAME) {
1319                                 err = build_map_pin_path(map, pin_root_path);
1320                                 if (err) {
1321                                         pr_warn("map '%s': couldn't build pin path.\n",
1322                                                 map_name);
1323                                         return err;
1324                                 }
1325                         }
1326                 } else {
1327                         if (strict) {
1328                                 pr_warn("map '%s': unknown field '%s'.\n",
1329                                         map_name, name);
1330                                 return -ENOTSUP;
1331                         }
1332                         pr_debug("map '%s': ignoring unknown field '%s'.\n",
1333                                  map_name, name);
1334                 }
1335         }
1336
1337         if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
1338                 pr_warn("map '%s': map type isn't specified.\n", map_name);
1339                 return -EINVAL;
1340         }
1341
1342         return 0;
1343 }
1344
1345 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
1346                                           const char *pin_root_path)
1347 {
1348         const struct btf_type *sec = NULL;
1349         int nr_types, i, vlen, err;
1350         const struct btf_type *t;
1351         const char *name;
1352         Elf_Data *data;
1353         Elf_Scn *scn;
1354
1355         if (obj->efile.btf_maps_shndx < 0)
1356                 return 0;
1357
1358         scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx);
1359         if (scn)
1360                 data = elf_getdata(scn, NULL);
1361         if (!scn || !data) {
1362                 pr_warn("failed to get Elf_Data from map section %d (%s)\n",
1363                         obj->efile.maps_shndx, MAPS_ELF_SEC);
1364                 return -EINVAL;
1365         }
1366
1367         nr_types = btf__get_nr_types(obj->btf);
1368         for (i = 1; i <= nr_types; i++) {
1369                 t = btf__type_by_id(obj->btf, i);
1370                 if (!btf_is_datasec(t))
1371                         continue;
1372                 name = btf__name_by_offset(obj->btf, t->name_off);
1373                 if (strcmp(name, MAPS_ELF_SEC) == 0) {
1374                         sec = t;
1375                         break;
1376                 }
1377         }
1378
1379         if (!sec) {
1380                 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
1381                 return -ENOENT;
1382         }
1383
1384         vlen = btf_vlen(sec);
1385         for (i = 0; i < vlen; i++) {
1386                 err = bpf_object__init_user_btf_map(obj, sec, i,
1387                                                     obj->efile.btf_maps_shndx,
1388                                                     data, strict,
1389                                                     pin_root_path);
1390                 if (err)
1391                         return err;
1392         }
1393
1394         return 0;
1395 }
1396
1397 static int bpf_object__init_maps(struct bpf_object *obj,
1398                                  const struct bpf_object_open_opts *opts)
1399 {
1400         const char *pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
1401         bool strict = !OPTS_GET(opts, relaxed_maps, false);
1402         int err;
1403
1404         err = bpf_object__init_user_maps(obj, strict);
1405         if (err)
1406                 return err;
1407
1408         err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
1409         if (err)
1410                 return err;
1411
1412         err = bpf_object__init_global_data_maps(obj);
1413         if (err)
1414                 return err;
1415
1416         if (obj->nr_maps) {
1417                 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]),
1418                       compare_bpf_map);
1419         }
1420         return 0;
1421 }
1422
1423 static bool section_have_execinstr(struct bpf_object *obj, int idx)
1424 {
1425         Elf_Scn *scn;
1426         GElf_Shdr sh;
1427
1428         scn = elf_getscn(obj->efile.elf, idx);
1429         if (!scn)
1430                 return false;
1431
1432         if (gelf_getshdr(scn, &sh) != &sh)
1433                 return false;
1434
1435         if (sh.sh_flags & SHF_EXECINSTR)
1436                 return true;
1437
1438         return false;
1439 }
1440
1441 static void bpf_object__sanitize_btf(struct bpf_object *obj)
1442 {
1443         bool has_datasec = obj->caps.btf_datasec;
1444         bool has_func = obj->caps.btf_func;
1445         struct btf *btf = obj->btf;
1446         struct btf_type *t;
1447         int i, j, vlen;
1448
1449         if (!obj->btf || (has_func && has_datasec))
1450                 return;
1451
1452         for (i = 1; i <= btf__get_nr_types(btf); i++) {
1453                 t = (struct btf_type *)btf__type_by_id(btf, i);
1454
1455                 if (!has_datasec && btf_is_var(t)) {
1456                         /* replace VAR with INT */
1457                         t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
1458                         /*
1459                          * using size = 1 is the safest choice, 4 will be too
1460                          * big and cause kernel BTF validation failure if
1461                          * original variable took less than 4 bytes
1462                          */
1463                         t->size = 1;
1464                         *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
1465                 } else if (!has_datasec && btf_is_datasec(t)) {
1466                         /* replace DATASEC with STRUCT */
1467                         const struct btf_var_secinfo *v = btf_var_secinfos(t);
1468                         struct btf_member *m = btf_members(t);
1469                         struct btf_type *vt;
1470                         char *name;
1471
1472                         name = (char *)btf__name_by_offset(btf, t->name_off);
1473                         while (*name) {
1474                                 if (*name == '.')
1475                                         *name = '_';
1476                                 name++;
1477                         }
1478
1479                         vlen = btf_vlen(t);
1480                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
1481                         for (j = 0; j < vlen; j++, v++, m++) {
1482                                 /* order of field assignments is important */
1483                                 m->offset = v->offset * 8;
1484                                 m->type = v->type;
1485                                 /* preserve variable name as member name */
1486                                 vt = (void *)btf__type_by_id(btf, v->type);
1487                                 m->name_off = vt->name_off;
1488                         }
1489                 } else if (!has_func && btf_is_func_proto(t)) {
1490                         /* replace FUNC_PROTO with ENUM */
1491                         vlen = btf_vlen(t);
1492                         t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
1493                         t->size = sizeof(__u32); /* kernel enforced */
1494                 } else if (!has_func && btf_is_func(t)) {
1495                         /* replace FUNC with TYPEDEF */
1496                         t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
1497                 }
1498         }
1499 }
1500
1501 static void bpf_object__sanitize_btf_ext(struct bpf_object *obj)
1502 {
1503         if (!obj->btf_ext)
1504                 return;
1505
1506         if (!obj->caps.btf_func) {
1507                 btf_ext__free(obj->btf_ext);
1508                 obj->btf_ext = NULL;
1509         }
1510 }
1511
1512 static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj)
1513 {
1514         return obj->efile.btf_maps_shndx >= 0;
1515 }
1516
1517 static int bpf_object__init_btf(struct bpf_object *obj,
1518                                 Elf_Data *btf_data,
1519                                 Elf_Data *btf_ext_data)
1520 {
1521         bool btf_required = bpf_object__is_btf_mandatory(obj);
1522         int err = 0;
1523
1524         if (btf_data) {
1525                 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
1526                 if (IS_ERR(obj->btf)) {
1527                         pr_warn("Error loading ELF section %s: %d.\n",
1528                                 BTF_ELF_SEC, err);
1529                         goto out;
1530                 }
1531                 err = btf__finalize_data(obj, obj->btf);
1532                 if (err) {
1533                         pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
1534                         goto out;
1535                 }
1536         }
1537         if (btf_ext_data) {
1538                 if (!obj->btf) {
1539                         pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
1540                                  BTF_EXT_ELF_SEC, BTF_ELF_SEC);
1541                         goto out;
1542                 }
1543                 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
1544                                             btf_ext_data->d_size);
1545                 if (IS_ERR(obj->btf_ext)) {
1546                         pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
1547                                 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
1548                         obj->btf_ext = NULL;
1549                         goto out;
1550                 }
1551         }
1552 out:
1553         if (err || IS_ERR(obj->btf)) {
1554                 if (btf_required)
1555                         err = err ? : PTR_ERR(obj->btf);
1556                 else
1557                         err = 0;
1558                 if (!IS_ERR_OR_NULL(obj->btf))
1559                         btf__free(obj->btf);
1560                 obj->btf = NULL;
1561         }
1562         if (btf_required && !obj->btf) {
1563                 pr_warn("BTF is required, but is missing or corrupted.\n");
1564                 return err == 0 ? -ENOENT : err;
1565         }
1566         return 0;
1567 }
1568
1569 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
1570 {
1571         int err = 0;
1572
1573         if (!obj->btf)
1574                 return 0;
1575
1576         bpf_object__sanitize_btf(obj);
1577         bpf_object__sanitize_btf_ext(obj);
1578
1579         err = btf__load(obj->btf);
1580         if (err) {
1581                 pr_warn("Error loading %s into kernel: %d.\n",
1582                         BTF_ELF_SEC, err);
1583                 btf__free(obj->btf);
1584                 obj->btf = NULL;
1585                 /* btf_ext can't exist without btf, so free it as well */
1586                 if (obj->btf_ext) {
1587                         btf_ext__free(obj->btf_ext);
1588                         obj->btf_ext = NULL;
1589                 }
1590
1591                 if (bpf_object__is_btf_mandatory(obj))
1592                         return err;
1593         }
1594         return 0;
1595 }
1596
1597 static int bpf_object__elf_collect(struct bpf_object *obj)
1598 {
1599         Elf *elf = obj->efile.elf;
1600         GElf_Ehdr *ep = &obj->efile.ehdr;
1601         Elf_Data *btf_ext_data = NULL;
1602         Elf_Data *btf_data = NULL;
1603         Elf_Scn *scn = NULL;
1604         int idx = 0, err = 0;
1605
1606         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1607         if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
1608                 pr_warn("failed to get e_shstrndx from %s\n", obj->path);
1609                 return -LIBBPF_ERRNO__FORMAT;
1610         }
1611
1612         while ((scn = elf_nextscn(elf, scn)) != NULL) {
1613                 char *name;
1614                 GElf_Shdr sh;
1615                 Elf_Data *data;
1616
1617                 idx++;
1618                 if (gelf_getshdr(scn, &sh) != &sh) {
1619                         pr_warn("failed to get section(%d) header from %s\n",
1620                                 idx, obj->path);
1621                         return -LIBBPF_ERRNO__FORMAT;
1622                 }
1623
1624                 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
1625                 if (!name) {
1626                         pr_warn("failed to get section(%d) name from %s\n",
1627                                 idx, obj->path);
1628                         return -LIBBPF_ERRNO__FORMAT;
1629                 }
1630
1631                 data = elf_getdata(scn, 0);
1632                 if (!data) {
1633                         pr_warn("failed to get section(%d) data from %s(%s)\n",
1634                                 idx, name, obj->path);
1635                         return -LIBBPF_ERRNO__FORMAT;
1636                 }
1637                 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
1638                          idx, name, (unsigned long)data->d_size,
1639                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
1640                          (int)sh.sh_type);
1641
1642                 if (strcmp(name, "license") == 0) {
1643                         err = bpf_object__init_license(obj,
1644                                                        data->d_buf,
1645                                                        data->d_size);
1646                         if (err)
1647                                 return err;
1648                 } else if (strcmp(name, "version") == 0) {
1649                         err = bpf_object__init_kversion(obj,
1650                                                         data->d_buf,
1651                                                         data->d_size);
1652                         if (err)
1653                                 return err;
1654                 } else if (strcmp(name, "maps") == 0) {
1655                         obj->efile.maps_shndx = idx;
1656                 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
1657                         obj->efile.btf_maps_shndx = idx;
1658                 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
1659                         btf_data = data;
1660                 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
1661                         btf_ext_data = data;
1662                 } else if (sh.sh_type == SHT_SYMTAB) {
1663                         if (obj->efile.symbols) {
1664                                 pr_warn("bpf: multiple SYMTAB in %s\n",
1665                                         obj->path);
1666                                 return -LIBBPF_ERRNO__FORMAT;
1667                         }
1668                         obj->efile.symbols = data;
1669                         obj->efile.strtabidx = sh.sh_link;
1670                 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
1671                         if (sh.sh_flags & SHF_EXECINSTR) {
1672                                 if (strcmp(name, ".text") == 0)
1673                                         obj->efile.text_shndx = idx;
1674                                 err = bpf_object__add_program(obj, data->d_buf,
1675                                                               data->d_size,
1676                                                               name, idx);
1677                                 if (err) {
1678                                         char errmsg[STRERR_BUFSIZE];
1679                                         char *cp;
1680
1681                                         cp = libbpf_strerror_r(-err, errmsg,
1682                                                                sizeof(errmsg));
1683                                         pr_warn("failed to alloc program %s (%s): %s",
1684                                                 name, obj->path, cp);
1685                                         return err;
1686                                 }
1687                         } else if (strcmp(name, ".data") == 0) {
1688                                 obj->efile.data = data;
1689                                 obj->efile.data_shndx = idx;
1690                         } else if (strcmp(name, ".rodata") == 0) {
1691                                 obj->efile.rodata = data;
1692                                 obj->efile.rodata_shndx = idx;
1693                         } else {
1694                                 pr_debug("skip section(%d) %s\n", idx, name);
1695                         }
1696                 } else if (sh.sh_type == SHT_REL) {
1697                         int nr_sects = obj->efile.nr_reloc_sects;
1698                         void *sects = obj->efile.reloc_sects;
1699                         int sec = sh.sh_info; /* points to other section */
1700
1701                         /* Only do relo for section with exec instructions */
1702                         if (!section_have_execinstr(obj, sec)) {
1703                                 pr_debug("skip relo %s(%d) for section(%d)\n",
1704                                          name, idx, sec);
1705                                 continue;
1706                         }
1707
1708                         sects = reallocarray(sects, nr_sects + 1,
1709                                              sizeof(*obj->efile.reloc_sects));
1710                         if (!sects) {
1711                                 pr_warn("reloc_sects realloc failed\n");
1712                                 return -ENOMEM;
1713                         }
1714
1715                         obj->efile.reloc_sects = sects;
1716                         obj->efile.nr_reloc_sects++;
1717
1718                         obj->efile.reloc_sects[nr_sects].shdr = sh;
1719                         obj->efile.reloc_sects[nr_sects].data = data;
1720                 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) {
1721                         obj->efile.bss = data;
1722                         obj->efile.bss_shndx = idx;
1723                 } else {
1724                         pr_debug("skip section(%d) %s\n", idx, name);
1725                 }
1726         }
1727
1728         if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
1729                 pr_warn("Corrupted ELF file: index of strtab invalid\n");
1730                 return -LIBBPF_ERRNO__FORMAT;
1731         }
1732         return bpf_object__init_btf(obj, btf_data, btf_ext_data);
1733 }
1734
1735 static struct bpf_program *
1736 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
1737 {
1738         struct bpf_program *prog;
1739         size_t i;
1740
1741         for (i = 0; i < obj->nr_programs; i++) {
1742                 prog = &obj->programs[i];
1743                 if (prog->idx == idx)
1744                         return prog;
1745         }
1746         return NULL;
1747 }
1748
1749 struct bpf_program *
1750 bpf_object__find_program_by_title(const struct bpf_object *obj,
1751                                   const char *title)
1752 {
1753         struct bpf_program *pos;
1754
1755         bpf_object__for_each_program(pos, obj) {
1756                 if (pos->section_name && !strcmp(pos->section_name, title))
1757                         return pos;
1758         }
1759         return NULL;
1760 }
1761
1762 struct bpf_program *
1763 bpf_object__find_program_by_name(const struct bpf_object *obj,
1764                                  const char *name)
1765 {
1766         struct bpf_program *prog;
1767
1768         bpf_object__for_each_program(prog, obj) {
1769                 if (!strcmp(prog->name, name))
1770                         return prog;
1771         }
1772         return NULL;
1773 }
1774
1775 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
1776                                       int shndx)
1777 {
1778         return shndx == obj->efile.data_shndx ||
1779                shndx == obj->efile.bss_shndx ||
1780                shndx == obj->efile.rodata_shndx;
1781 }
1782
1783 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
1784                                       int shndx)
1785 {
1786         return shndx == obj->efile.maps_shndx ||
1787                shndx == obj->efile.btf_maps_shndx;
1788 }
1789
1790 static enum libbpf_map_type
1791 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
1792 {
1793         if (shndx == obj->efile.data_shndx)
1794                 return LIBBPF_MAP_DATA;
1795         else if (shndx == obj->efile.bss_shndx)
1796                 return LIBBPF_MAP_BSS;
1797         else if (shndx == obj->efile.rodata_shndx)
1798                 return LIBBPF_MAP_RODATA;
1799         else
1800                 return LIBBPF_MAP_UNSPEC;
1801 }
1802
1803 static int bpf_program__record_reloc(struct bpf_program *prog,
1804                                      struct reloc_desc *reloc_desc,
1805                                      __u32 insn_idx, const char *name,
1806                                      const GElf_Sym *sym, const GElf_Rel *rel)
1807 {
1808         struct bpf_insn *insn = &prog->insns[insn_idx];
1809         size_t map_idx, nr_maps = prog->obj->nr_maps;
1810         struct bpf_object *obj = prog->obj;
1811         __u32 shdr_idx = sym->st_shndx;
1812         enum libbpf_map_type type;
1813         struct bpf_map *map;
1814
1815         /* sub-program call relocation */
1816         if (insn->code == (BPF_JMP | BPF_CALL)) {
1817                 if (insn->src_reg != BPF_PSEUDO_CALL) {
1818                         pr_warn("incorrect bpf_call opcode\n");
1819                         return -LIBBPF_ERRNO__RELOC;
1820                 }
1821                 /* text_shndx can be 0, if no default "main" program exists */
1822                 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
1823                         pr_warn("bad call relo against section %u\n", shdr_idx);
1824                         return -LIBBPF_ERRNO__RELOC;
1825                 }
1826                 if (sym->st_value % 8) {
1827                         pr_warn("bad call relo offset: %zu\n",
1828                                 (size_t)sym->st_value);
1829                         return -LIBBPF_ERRNO__RELOC;
1830                 }
1831                 reloc_desc->type = RELO_CALL;
1832                 reloc_desc->insn_idx = insn_idx;
1833                 reloc_desc->sym_off = sym->st_value;
1834                 obj->has_pseudo_calls = true;
1835                 return 0;
1836         }
1837
1838         if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
1839                 pr_warn("invalid relo for insns[%d].code 0x%x\n",
1840                         insn_idx, insn->code);
1841                 return -LIBBPF_ERRNO__RELOC;
1842         }
1843         if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
1844                 pr_warn("invalid relo for \'%s\' in special section 0x%x; forgot to initialize global var?..\n",
1845                         name, shdr_idx);
1846                 return -LIBBPF_ERRNO__RELOC;
1847         }
1848
1849         type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
1850
1851         /* generic map reference relocation */
1852         if (type == LIBBPF_MAP_UNSPEC) {
1853                 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
1854                         pr_warn("bad map relo against section %u\n",
1855                                 shdr_idx);
1856                         return -LIBBPF_ERRNO__RELOC;
1857                 }
1858                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
1859                         map = &obj->maps[map_idx];
1860                         if (map->libbpf_type != type ||
1861                             map->sec_idx != sym->st_shndx ||
1862                             map->sec_offset != sym->st_value)
1863                                 continue;
1864                         pr_debug("found map %zd (%s, sec %d, off %zu) for insn %u\n",
1865                                  map_idx, map->name, map->sec_idx,
1866                                  map->sec_offset, insn_idx);
1867                         break;
1868                 }
1869                 if (map_idx >= nr_maps) {
1870                         pr_warn("map relo failed to find map for sec %u, off %zu\n",
1871                                 shdr_idx, (size_t)sym->st_value);
1872                         return -LIBBPF_ERRNO__RELOC;
1873                 }
1874                 reloc_desc->type = RELO_LD64;
1875                 reloc_desc->insn_idx = insn_idx;
1876                 reloc_desc->map_idx = map_idx;
1877                 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
1878                 return 0;
1879         }
1880
1881         /* global data map relocation */
1882         if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
1883                 pr_warn("bad data relo against section %u\n", shdr_idx);
1884                 return -LIBBPF_ERRNO__RELOC;
1885         }
1886         for (map_idx = 0; map_idx < nr_maps; map_idx++) {
1887                 map = &obj->maps[map_idx];
1888                 if (map->libbpf_type != type)
1889                         continue;
1890                 pr_debug("found data map %zd (%s, sec %d, off %zu) for insn %u\n",
1891                          map_idx, map->name, map->sec_idx, map->sec_offset,
1892                          insn_idx);
1893                 break;
1894         }
1895         if (map_idx >= nr_maps) {
1896                 pr_warn("data relo failed to find map for sec %u\n",
1897                         shdr_idx);
1898                 return -LIBBPF_ERRNO__RELOC;
1899         }
1900
1901         reloc_desc->type = RELO_DATA;
1902         reloc_desc->insn_idx = insn_idx;
1903         reloc_desc->map_idx = map_idx;
1904         reloc_desc->sym_off = sym->st_value;
1905         return 0;
1906 }
1907
1908 static int
1909 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
1910                            Elf_Data *data, struct bpf_object *obj)
1911 {
1912         Elf_Data *symbols = obj->efile.symbols;
1913         int err, i, nrels;
1914
1915         pr_debug("collecting relocating info for: '%s'\n", prog->section_name);
1916         nrels = shdr->sh_size / shdr->sh_entsize;
1917
1918         prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
1919         if (!prog->reloc_desc) {
1920                 pr_warn("failed to alloc memory in relocation\n");
1921                 return -ENOMEM;
1922         }
1923         prog->nr_reloc = nrels;
1924
1925         for (i = 0; i < nrels; i++) {
1926                 const char *name;
1927                 __u32 insn_idx;
1928                 GElf_Sym sym;
1929                 GElf_Rel rel;
1930
1931                 if (!gelf_getrel(data, i, &rel)) {
1932                         pr_warn("relocation: failed to get %d reloc\n", i);
1933                         return -LIBBPF_ERRNO__FORMAT;
1934                 }
1935                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
1936                         pr_warn("relocation: symbol %"PRIx64" not found\n",
1937                                 GELF_R_SYM(rel.r_info));
1938                         return -LIBBPF_ERRNO__FORMAT;
1939                 }
1940                 if (rel.r_offset % sizeof(struct bpf_insn))
1941                         return -LIBBPF_ERRNO__FORMAT;
1942
1943                 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
1944                 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1945                                   sym.st_name) ? : "<?>";
1946
1947                 pr_debug("relo for shdr %u, symb %zu, value %zu, type %d, bind %d, name %d (\'%s\'), insn %u\n",
1948                          (__u32)sym.st_shndx, (size_t)GELF_R_SYM(rel.r_info),
1949                          (size_t)sym.st_value, GELF_ST_TYPE(sym.st_info),
1950                          GELF_ST_BIND(sym.st_info), sym.st_name, name,
1951                          insn_idx);
1952
1953                 err = bpf_program__record_reloc(prog, &prog->reloc_desc[i],
1954                                                 insn_idx, name, &sym, &rel);
1955                 if (err)
1956                         return err;
1957         }
1958         return 0;
1959 }
1960
1961 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
1962 {
1963         struct bpf_map_def *def = &map->def;
1964         __u32 key_type_id = 0, value_type_id = 0;
1965         int ret;
1966
1967         /* if it's BTF-defined map, we don't need to search for type IDs */
1968         if (map->sec_idx == obj->efile.btf_maps_shndx)
1969                 return 0;
1970
1971         if (!bpf_map__is_internal(map)) {
1972                 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
1973                                            def->value_size, &key_type_id,
1974                                            &value_type_id);
1975         } else {
1976                 /*
1977                  * LLVM annotates global data differently in BTF, that is,
1978                  * only as '.data', '.bss' or '.rodata'.
1979                  */
1980                 ret = btf__find_by_name(obj->btf,
1981                                 libbpf_type_to_btf_name[map->libbpf_type]);
1982         }
1983         if (ret < 0)
1984                 return ret;
1985
1986         map->btf_key_type_id = key_type_id;
1987         map->btf_value_type_id = bpf_map__is_internal(map) ?
1988                                  ret : value_type_id;
1989         return 0;
1990 }
1991
1992 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
1993 {
1994         struct bpf_map_info info = {};
1995         __u32 len = sizeof(info);
1996         int new_fd, err;
1997         char *new_name;
1998
1999         err = bpf_obj_get_info_by_fd(fd, &info, &len);
2000         if (err)
2001                 return err;
2002
2003         new_name = strdup(info.name);
2004         if (!new_name)
2005                 return -errno;
2006
2007         new_fd = open("/", O_RDONLY | O_CLOEXEC);
2008         if (new_fd < 0) {
2009                 err = -errno;
2010                 goto err_free_new_name;
2011         }
2012
2013         new_fd = dup3(fd, new_fd, O_CLOEXEC);
2014         if (new_fd < 0) {
2015                 err = -errno;
2016                 goto err_close_new_fd;
2017         }
2018
2019         err = zclose(map->fd);
2020         if (err) {
2021                 err = -errno;
2022                 goto err_close_new_fd;
2023         }
2024         free(map->name);
2025
2026         map->fd = new_fd;
2027         map->name = new_name;
2028         map->def.type = info.type;
2029         map->def.key_size = info.key_size;
2030         map->def.value_size = info.value_size;
2031         map->def.max_entries = info.max_entries;
2032         map->def.map_flags = info.map_flags;
2033         map->btf_key_type_id = info.btf_key_type_id;
2034         map->btf_value_type_id = info.btf_value_type_id;
2035         map->reused = true;
2036
2037         return 0;
2038
2039 err_close_new_fd:
2040         close(new_fd);
2041 err_free_new_name:
2042         free(new_name);
2043         return err;
2044 }
2045
2046 int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
2047 {
2048         if (!map || !max_entries)
2049                 return -EINVAL;
2050
2051         /* If map already created, its attributes can't be changed. */
2052         if (map->fd >= 0)
2053                 return -EBUSY;
2054
2055         map->def.max_entries = max_entries;
2056
2057         return 0;
2058 }
2059
2060 static int
2061 bpf_object__probe_name(struct bpf_object *obj)
2062 {
2063         struct bpf_load_program_attr attr;
2064         char *cp, errmsg[STRERR_BUFSIZE];
2065         struct bpf_insn insns[] = {
2066                 BPF_MOV64_IMM(BPF_REG_0, 0),
2067                 BPF_EXIT_INSN(),
2068         };
2069         int ret;
2070
2071         /* make sure basic loading works */
2072
2073         memset(&attr, 0, sizeof(attr));
2074         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
2075         attr.insns = insns;
2076         attr.insns_cnt = ARRAY_SIZE(insns);
2077         attr.license = "GPL";
2078
2079         ret = bpf_load_program_xattr(&attr, NULL, 0);
2080         if (ret < 0) {
2081                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
2082                 pr_warn("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n",
2083                         __func__, cp, errno);
2084                 return -errno;
2085         }
2086         close(ret);
2087
2088         /* now try the same program, but with the name */
2089
2090         attr.name = "test";
2091         ret = bpf_load_program_xattr(&attr, NULL, 0);
2092         if (ret >= 0) {
2093                 obj->caps.name = 1;
2094                 close(ret);
2095         }
2096
2097         return 0;
2098 }
2099
2100 static int
2101 bpf_object__probe_global_data(struct bpf_object *obj)
2102 {
2103         struct bpf_load_program_attr prg_attr;
2104         struct bpf_create_map_attr map_attr;
2105         char *cp, errmsg[STRERR_BUFSIZE];
2106         struct bpf_insn insns[] = {
2107                 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
2108                 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
2109                 BPF_MOV64_IMM(BPF_REG_0, 0),
2110                 BPF_EXIT_INSN(),
2111         };
2112         int ret, map;
2113
2114         memset(&map_attr, 0, sizeof(map_attr));
2115         map_attr.map_type = BPF_MAP_TYPE_ARRAY;
2116         map_attr.key_size = sizeof(int);
2117         map_attr.value_size = 32;
2118         map_attr.max_entries = 1;
2119
2120         map = bpf_create_map_xattr(&map_attr);
2121         if (map < 0) {
2122                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
2123                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
2124                         __func__, cp, errno);
2125                 return -errno;
2126         }
2127
2128         insns[0].imm = map;
2129
2130         memset(&prg_attr, 0, sizeof(prg_attr));
2131         prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
2132         prg_attr.insns = insns;
2133         prg_attr.insns_cnt = ARRAY_SIZE(insns);
2134         prg_attr.license = "GPL";
2135
2136         ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
2137         if (ret >= 0) {
2138                 obj->caps.global_data = 1;
2139                 close(ret);
2140         }
2141
2142         close(map);
2143         return 0;
2144 }
2145
2146 static int bpf_object__probe_btf_func(struct bpf_object *obj)
2147 {
2148         static const char strs[] = "\0int\0x\0a";
2149         /* void x(int a) {} */
2150         __u32 types[] = {
2151                 /* int */
2152                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
2153                 /* FUNC_PROTO */                                /* [2] */
2154                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
2155                 BTF_PARAM_ENC(7, 1),
2156                 /* FUNC x */                                    /* [3] */
2157                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
2158         };
2159         int btf_fd;
2160
2161         btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
2162                                       strs, sizeof(strs));
2163         if (btf_fd >= 0) {
2164                 obj->caps.btf_func = 1;
2165                 close(btf_fd);
2166                 return 1;
2167         }
2168
2169         return 0;
2170 }
2171
2172 static int bpf_object__probe_btf_datasec(struct bpf_object *obj)
2173 {
2174         static const char strs[] = "\0x\0.data";
2175         /* static int a; */
2176         __u32 types[] = {
2177                 /* int */
2178                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
2179                 /* VAR x */                                     /* [2] */
2180                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
2181                 BTF_VAR_STATIC,
2182                 /* DATASEC val */                               /* [3] */
2183                 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
2184                 BTF_VAR_SECINFO_ENC(2, 0, 4),
2185         };
2186         int btf_fd;
2187
2188         btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
2189                                       strs, sizeof(strs));
2190         if (btf_fd >= 0) {
2191                 obj->caps.btf_datasec = 1;
2192                 close(btf_fd);
2193                 return 1;
2194         }
2195
2196         return 0;
2197 }
2198
2199 static int bpf_object__probe_array_mmap(struct bpf_object *obj)
2200 {
2201         struct bpf_create_map_attr attr = {
2202                 .map_type = BPF_MAP_TYPE_ARRAY,
2203                 .map_flags = BPF_F_MMAPABLE,
2204                 .key_size = sizeof(int),
2205                 .value_size = sizeof(int),
2206                 .max_entries = 1,
2207         };
2208         int fd;
2209
2210         fd = bpf_create_map_xattr(&attr);
2211         if (fd >= 0) {
2212                 obj->caps.array_mmap = 1;
2213                 close(fd);
2214                 return 1;
2215         }
2216
2217         return 0;
2218 }
2219
2220 static int
2221 bpf_object__probe_caps(struct bpf_object *obj)
2222 {
2223         int (*probe_fn[])(struct bpf_object *obj) = {
2224                 bpf_object__probe_name,
2225                 bpf_object__probe_global_data,
2226                 bpf_object__probe_btf_func,
2227                 bpf_object__probe_btf_datasec,
2228                 bpf_object__probe_array_mmap,
2229         };
2230         int i, ret;
2231
2232         for (i = 0; i < ARRAY_SIZE(probe_fn); i++) {
2233                 ret = probe_fn[i](obj);
2234                 if (ret < 0)
2235                         pr_debug("Probe #%d failed with %d.\n", i, ret);
2236         }
2237
2238         return 0;
2239 }
2240
2241 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
2242 {
2243         struct bpf_map_info map_info = {};
2244         char msg[STRERR_BUFSIZE];
2245         __u32 map_info_len;
2246
2247         map_info_len = sizeof(map_info);
2248
2249         if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
2250                 pr_warn("failed to get map info for map FD %d: %s\n",
2251                         map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
2252                 return false;
2253         }
2254
2255         return (map_info.type == map->def.type &&
2256                 map_info.key_size == map->def.key_size &&
2257                 map_info.value_size == map->def.value_size &&
2258                 map_info.max_entries == map->def.max_entries &&
2259                 map_info.map_flags == map->def.map_flags);
2260 }
2261
2262 static int
2263 bpf_object__reuse_map(struct bpf_map *map)
2264 {
2265         char *cp, errmsg[STRERR_BUFSIZE];
2266         int err, pin_fd;
2267
2268         pin_fd = bpf_obj_get(map->pin_path);
2269         if (pin_fd < 0) {
2270                 err = -errno;
2271                 if (err == -ENOENT) {
2272                         pr_debug("found no pinned map to reuse at '%s'\n",
2273                                  map->pin_path);
2274                         return 0;
2275                 }
2276
2277                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
2278                 pr_warn("couldn't retrieve pinned map '%s': %s\n",
2279                         map->pin_path, cp);
2280                 return err;
2281         }
2282
2283         if (!map_is_reuse_compat(map, pin_fd)) {
2284                 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
2285                         map->pin_path);
2286                 close(pin_fd);
2287                 return -EINVAL;
2288         }
2289
2290         err = bpf_map__reuse_fd(map, pin_fd);
2291         if (err) {
2292                 close(pin_fd);
2293                 return err;
2294         }
2295         map->pinned = true;
2296         pr_debug("reused pinned map at '%s'\n", map->pin_path);
2297
2298         return 0;
2299 }
2300
2301 static int
2302 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
2303 {
2304         char *cp, errmsg[STRERR_BUFSIZE];
2305         int err, zero = 0;
2306
2307         /* Nothing to do here since kernel already zero-initializes .bss map. */
2308         if (map->libbpf_type == LIBBPF_MAP_BSS)
2309                 return 0;
2310
2311         err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
2312         if (err) {
2313                 err = -errno;
2314                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
2315                 pr_warn("Error setting initial map(%s) contents: %s\n",
2316                         map->name, cp);
2317                 return err;
2318         }
2319
2320         /* Freeze .rodata map as read-only from syscall side. */
2321         if (map->libbpf_type == LIBBPF_MAP_RODATA) {
2322                 err = bpf_map_freeze(map->fd);
2323                 if (err) {
2324                         err = -errno;
2325                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
2326                         pr_warn("Error freezing map(%s) as read-only: %s\n",
2327                                 map->name, cp);
2328                         return err;
2329                 }
2330         }
2331         return 0;
2332 }
2333
2334 static int
2335 bpf_object__create_maps(struct bpf_object *obj)
2336 {
2337         struct bpf_create_map_attr create_attr = {};
2338         int nr_cpus = 0;
2339         unsigned int i;
2340         int err;
2341
2342         for (i = 0; i < obj->nr_maps; i++) {
2343                 struct bpf_map *map = &obj->maps[i];
2344                 struct bpf_map_def *def = &map->def;
2345                 char *cp, errmsg[STRERR_BUFSIZE];
2346                 int *pfd = &map->fd;
2347
2348                 if (map->pin_path) {
2349                         err = bpf_object__reuse_map(map);
2350                         if (err) {
2351                                 pr_warn("error reusing pinned map %s\n",
2352                                         map->name);
2353                                 return err;
2354                         }
2355                 }
2356
2357                 if (map->fd >= 0) {
2358                         pr_debug("skip map create (preset) %s: fd=%d\n",
2359                                  map->name, map->fd);
2360                         continue;
2361                 }
2362
2363                 if (obj->caps.name)
2364                         create_attr.name = map->name;
2365                 create_attr.map_ifindex = map->map_ifindex;
2366                 create_attr.map_type = def->type;
2367                 create_attr.map_flags = def->map_flags;
2368                 create_attr.key_size = def->key_size;
2369                 create_attr.value_size = def->value_size;
2370                 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
2371                     !def->max_entries) {
2372                         if (!nr_cpus)
2373                                 nr_cpus = libbpf_num_possible_cpus();
2374                         if (nr_cpus < 0) {
2375                                 pr_warn("failed to determine number of system CPUs: %d\n",
2376                                         nr_cpus);
2377                                 err = nr_cpus;
2378                                 goto err_out;
2379                         }
2380                         pr_debug("map '%s': setting size to %d\n",
2381                                  map->name, nr_cpus);
2382                         create_attr.max_entries = nr_cpus;
2383                 } else {
2384                         create_attr.max_entries = def->max_entries;
2385                 }
2386                 create_attr.btf_fd = 0;
2387                 create_attr.btf_key_type_id = 0;
2388                 create_attr.btf_value_type_id = 0;
2389                 if (bpf_map_type__is_map_in_map(def->type) &&
2390                     map->inner_map_fd >= 0)
2391                         create_attr.inner_map_fd = map->inner_map_fd;
2392
2393                 if (obj->btf && !bpf_map_find_btf_info(obj, map)) {
2394                         create_attr.btf_fd = btf__fd(obj->btf);
2395                         create_attr.btf_key_type_id = map->btf_key_type_id;
2396                         create_attr.btf_value_type_id = map->btf_value_type_id;
2397                 }
2398
2399                 *pfd = bpf_create_map_xattr(&create_attr);
2400                 if (*pfd < 0 && (create_attr.btf_key_type_id ||
2401                                  create_attr.btf_value_type_id)) {
2402                         err = -errno;
2403                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
2404                         pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
2405                                 map->name, cp, err);
2406                         create_attr.btf_fd = 0;
2407                         create_attr.btf_key_type_id = 0;
2408                         create_attr.btf_value_type_id = 0;
2409                         map->btf_key_type_id = 0;
2410                         map->btf_value_type_id = 0;
2411                         *pfd = bpf_create_map_xattr(&create_attr);
2412                 }
2413
2414                 if (*pfd < 0) {
2415                         size_t j;
2416
2417                         err = -errno;
2418 err_out:
2419                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
2420                         pr_warn("failed to create map (name: '%s'): %s(%d)\n",
2421                                 map->name, cp, err);
2422                         for (j = 0; j < i; j++)
2423                                 zclose(obj->maps[j].fd);
2424                         return err;
2425                 }
2426
2427                 if (bpf_map__is_internal(map)) {
2428                         err = bpf_object__populate_internal_map(obj, map);
2429                         if (err < 0) {
2430                                 zclose(*pfd);
2431                                 goto err_out;
2432                         }
2433                 }
2434
2435                 if (map->pin_path && !map->pinned) {
2436                         err = bpf_map__pin(map, NULL);
2437                         if (err) {
2438                                 pr_warn("failed to auto-pin map name '%s' at '%s'\n",
2439                                         map->name, map->pin_path);
2440                                 return err;
2441                         }
2442                 }
2443
2444                 pr_debug("created map %s: fd=%d\n", map->name, *pfd);
2445         }
2446
2447         return 0;
2448 }
2449
2450 static int
2451 check_btf_ext_reloc_err(struct bpf_program *prog, int err,
2452                         void *btf_prog_info, const char *info_name)
2453 {
2454         if (err != -ENOENT) {
2455                 pr_warn("Error in loading %s for sec %s.\n",
2456                         info_name, prog->section_name);
2457                 return err;
2458         }
2459
2460         /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */
2461
2462         if (btf_prog_info) {
2463                 /*
2464                  * Some info has already been found but has problem
2465                  * in the last btf_ext reloc. Must have to error out.
2466                  */
2467                 pr_warn("Error in relocating %s for sec %s.\n",
2468                         info_name, prog->section_name);
2469                 return err;
2470         }
2471
2472         /* Have problem loading the very first info. Ignore the rest. */
2473         pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n",
2474                 info_name, prog->section_name, info_name);
2475         return 0;
2476 }
2477
2478 static int
2479 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
2480                           const char *section_name,  __u32 insn_offset)
2481 {
2482         int err;
2483
2484         if (!insn_offset || prog->func_info) {
2485                 /*
2486                  * !insn_offset => main program
2487                  *
2488                  * For sub prog, the main program's func_info has to
2489                  * be loaded first (i.e. prog->func_info != NULL)
2490                  */
2491                 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext,
2492                                                section_name, insn_offset,
2493                                                &prog->func_info,
2494                                                &prog->func_info_cnt);
2495                 if (err)
2496                         return check_btf_ext_reloc_err(prog, err,
2497                                                        prog->func_info,
2498                                                        "bpf_func_info");
2499
2500                 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
2501         }
2502
2503         if (!insn_offset || prog->line_info) {
2504                 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
2505                                                section_name, insn_offset,
2506                                                &prog->line_info,
2507                                                &prog->line_info_cnt);
2508                 if (err)
2509                         return check_btf_ext_reloc_err(prog, err,
2510                                                        prog->line_info,
2511                                                        "bpf_line_info");
2512
2513                 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
2514         }
2515
2516         return 0;
2517 }
2518
2519 #define BPF_CORE_SPEC_MAX_LEN 64
2520
2521 /* represents BPF CO-RE field or array element accessor */
2522 struct bpf_core_accessor {
2523         __u32 type_id;          /* struct/union type or array element type */
2524         __u32 idx;              /* field index or array index */
2525         const char *name;       /* field name or NULL for array accessor */
2526 };
2527
2528 struct bpf_core_spec {
2529         const struct btf *btf;
2530         /* high-level spec: named fields and array indices only */
2531         struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
2532         /* high-level spec length */
2533         int len;
2534         /* raw, low-level spec: 1-to-1 with accessor spec string */
2535         int raw_spec[BPF_CORE_SPEC_MAX_LEN];
2536         /* raw spec length */
2537         int raw_len;
2538         /* field bit offset represented by spec */
2539         __u32 bit_offset;
2540 };
2541
2542 static bool str_is_empty(const char *s)
2543 {
2544         return !s || !s[0];
2545 }
2546
2547 /*
2548  * Turn bpf_field_reloc into a low- and high-level spec representation,
2549  * validating correctness along the way, as well as calculating resulting
2550  * field bit offset, specified by accessor string. Low-level spec captures
2551  * every single level of nestedness, including traversing anonymous
2552  * struct/union members. High-level one only captures semantically meaningful
2553  * "turning points": named fields and array indicies.
2554  * E.g., for this case:
2555  *
2556  *   struct sample {
2557  *       int __unimportant;
2558  *       struct {
2559  *           int __1;
2560  *           int __2;
2561  *           int a[7];
2562  *       };
2563  *   };
2564  *
2565  *   struct sample *s = ...;
2566  *
2567  *   int x = &s->a[3]; // access string = '0:1:2:3'
2568  *
2569  * Low-level spec has 1:1 mapping with each element of access string (it's
2570  * just a parsed access string representation): [0, 1, 2, 3].
2571  *
2572  * High-level spec will capture only 3 points:
2573  *   - intial zero-index access by pointer (&s->... is the same as &s[0]...);
2574  *   - field 'a' access (corresponds to '2' in low-level spec);
2575  *   - array element #3 access (corresponds to '3' in low-level spec).
2576  *
2577  */
2578 static int bpf_core_spec_parse(const struct btf *btf,
2579                                __u32 type_id,
2580                                const char *spec_str,
2581                                struct bpf_core_spec *spec)
2582 {
2583         int access_idx, parsed_len, i;
2584         const struct btf_type *t;
2585         const char *name;
2586         __u32 id;
2587         __s64 sz;
2588
2589         if (str_is_empty(spec_str) || *spec_str == ':')
2590                 return -EINVAL;
2591
2592         memset(spec, 0, sizeof(*spec));
2593         spec->btf = btf;
2594
2595         /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
2596         while (*spec_str) {
2597                 if (*spec_str == ':')
2598                         ++spec_str;
2599                 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
2600                         return -EINVAL;
2601                 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
2602                         return -E2BIG;
2603                 spec_str += parsed_len;
2604                 spec->raw_spec[spec->raw_len++] = access_idx;
2605         }
2606
2607         if (spec->raw_len == 0)
2608                 return -EINVAL;
2609
2610         /* first spec value is always reloc type array index */
2611         t = skip_mods_and_typedefs(btf, type_id, &id);
2612         if (!t)
2613                 return -EINVAL;
2614
2615         access_idx = spec->raw_spec[0];
2616         spec->spec[0].type_id = id;
2617         spec->spec[0].idx = access_idx;
2618         spec->len++;
2619
2620         sz = btf__resolve_size(btf, id);
2621         if (sz < 0)
2622                 return sz;
2623         spec->bit_offset = access_idx * sz * 8;
2624
2625         for (i = 1; i < spec->raw_len; i++) {
2626                 t = skip_mods_and_typedefs(btf, id, &id);
2627                 if (!t)
2628                         return -EINVAL;
2629
2630                 access_idx = spec->raw_spec[i];
2631
2632                 if (btf_is_composite(t)) {
2633                         const struct btf_member *m;
2634                         __u32 bit_offset;
2635
2636                         if (access_idx >= btf_vlen(t))
2637                                 return -EINVAL;
2638
2639                         bit_offset = btf_member_bit_offset(t, access_idx);
2640                         spec->bit_offset += bit_offset;
2641
2642                         m = btf_members(t) + access_idx;
2643                         if (m->name_off) {
2644                                 name = btf__name_by_offset(btf, m->name_off);
2645                                 if (str_is_empty(name))
2646                                         return -EINVAL;
2647
2648                                 spec->spec[spec->len].type_id = id;
2649                                 spec->spec[spec->len].idx = access_idx;
2650                                 spec->spec[spec->len].name = name;
2651                                 spec->len++;
2652                         }
2653
2654                         id = m->type;
2655                 } else if (btf_is_array(t)) {
2656                         const struct btf_array *a = btf_array(t);
2657
2658                         t = skip_mods_and_typedefs(btf, a->type, &id);
2659                         if (!t || access_idx >= a->nelems)
2660                                 return -EINVAL;
2661
2662                         spec->spec[spec->len].type_id = id;
2663                         spec->spec[spec->len].idx = access_idx;
2664                         spec->len++;
2665
2666                         sz = btf__resolve_size(btf, id);
2667                         if (sz < 0)
2668                                 return sz;
2669                         spec->bit_offset += access_idx * sz * 8;
2670                 } else {
2671                         pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n",
2672                                 type_id, spec_str, i, id, btf_kind(t));
2673                         return -EINVAL;
2674                 }
2675         }
2676
2677         return 0;
2678 }
2679
2680 static bool bpf_core_is_flavor_sep(const char *s)
2681 {
2682         /* check X___Y name pattern, where X and Y are not underscores */
2683         return s[0] != '_' &&                                 /* X */
2684                s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */
2685                s[4] != '_';                                   /* Y */
2686 }
2687
2688 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
2689  * before last triple underscore. Struct name part after last triple
2690  * underscore is ignored by BPF CO-RE relocation during relocation matching.
2691  */
2692 static size_t bpf_core_essential_name_len(const char *name)
2693 {
2694         size_t n = strlen(name);
2695         int i;
2696
2697         for (i = n - 5; i >= 0; i--) {
2698                 if (bpf_core_is_flavor_sep(name + i))
2699                         return i + 1;
2700         }
2701         return n;
2702 }
2703
2704 /* dynamically sized list of type IDs */
2705 struct ids_vec {
2706         __u32 *data;
2707         int len;
2708 };
2709
2710 static void bpf_core_free_cands(struct ids_vec *cand_ids)
2711 {
2712         free(cand_ids->data);
2713         free(cand_ids);
2714 }
2715
2716 static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf,
2717                                            __u32 local_type_id,
2718                                            const struct btf *targ_btf)
2719 {
2720         size_t local_essent_len, targ_essent_len;
2721         const char *local_name, *targ_name;
2722         const struct btf_type *t;
2723         struct ids_vec *cand_ids;
2724         __u32 *new_ids;
2725         int i, err, n;
2726
2727         t = btf__type_by_id(local_btf, local_type_id);
2728         if (!t)
2729                 return ERR_PTR(-EINVAL);
2730
2731         local_name = btf__name_by_offset(local_btf, t->name_off);
2732         if (str_is_empty(local_name))
2733                 return ERR_PTR(-EINVAL);
2734         local_essent_len = bpf_core_essential_name_len(local_name);
2735
2736         cand_ids = calloc(1, sizeof(*cand_ids));
2737         if (!cand_ids)
2738                 return ERR_PTR(-ENOMEM);
2739
2740         n = btf__get_nr_types(targ_btf);
2741         for (i = 1; i <= n; i++) {
2742                 t = btf__type_by_id(targ_btf, i);
2743                 targ_name = btf__name_by_offset(targ_btf, t->name_off);
2744                 if (str_is_empty(targ_name))
2745                         continue;
2746
2747                 targ_essent_len = bpf_core_essential_name_len(targ_name);
2748                 if (targ_essent_len != local_essent_len)
2749                         continue;
2750
2751                 if (strncmp(local_name, targ_name, local_essent_len) == 0) {
2752                         pr_debug("[%d] %s: found candidate [%d] %s\n",
2753                                  local_type_id, local_name, i, targ_name);
2754                         new_ids = realloc(cand_ids->data, cand_ids->len + 1);
2755                         if (!new_ids) {
2756                                 err = -ENOMEM;
2757                                 goto err_out;
2758                         }
2759                         cand_ids->data = new_ids;
2760                         cand_ids->data[cand_ids->len++] = i;
2761                 }
2762         }
2763         return cand_ids;
2764 err_out:
2765         bpf_core_free_cands(cand_ids);
2766         return ERR_PTR(err);
2767 }
2768
2769 /* Check two types for compatibility, skipping const/volatile/restrict and
2770  * typedefs, to ensure we are relocating compatible entities:
2771  *   - any two STRUCTs/UNIONs are compatible and can be mixed;
2772  *   - any two FWDs are compatible, if their names match (modulo flavor suffix);
2773  *   - any two PTRs are always compatible;
2774  *   - for ENUMs, names should be the same (ignoring flavor suffix) or at
2775  *     least one of enums should be anonymous;
2776  *   - for ENUMs, check sizes, names are ignored;
2777  *   - for INT, size and signedness are ignored;
2778  *   - for ARRAY, dimensionality is ignored, element types are checked for
2779  *     compatibility recursively;
2780  *   - everything else shouldn't be ever a target of relocation.
2781  * These rules are not set in stone and probably will be adjusted as we get
2782  * more experience with using BPF CO-RE relocations.
2783  */
2784 static int bpf_core_fields_are_compat(const struct btf *local_btf,
2785                                       __u32 local_id,
2786                                       const struct btf *targ_btf,
2787                                       __u32 targ_id)
2788 {
2789         const struct btf_type *local_type, *targ_type;
2790
2791 recur:
2792         local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
2793         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
2794         if (!local_type || !targ_type)
2795                 return -EINVAL;
2796
2797         if (btf_is_composite(local_type) && btf_is_composite(targ_type))
2798                 return 1;
2799         if (btf_kind(local_type) != btf_kind(targ_type))
2800                 return 0;
2801
2802         switch (btf_kind(local_type)) {
2803         case BTF_KIND_PTR:
2804                 return 1;
2805         case BTF_KIND_FWD:
2806         case BTF_KIND_ENUM: {
2807                 const char *local_name, *targ_name;
2808                 size_t local_len, targ_len;
2809
2810                 local_name = btf__name_by_offset(local_btf,
2811                                                  local_type->name_off);
2812                 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
2813                 local_len = bpf_core_essential_name_len(local_name);
2814                 targ_len = bpf_core_essential_name_len(targ_name);
2815                 /* one of them is anonymous or both w/ same flavor-less names */
2816                 return local_len == 0 || targ_len == 0 ||
2817                        (local_len == targ_len &&
2818                         strncmp(local_name, targ_name, local_len) == 0);
2819         }
2820         case BTF_KIND_INT:
2821                 /* just reject deprecated bitfield-like integers; all other
2822                  * integers are by default compatible between each other
2823                  */
2824                 return btf_int_offset(local_type) == 0 &&
2825                        btf_int_offset(targ_type) == 0;
2826         case BTF_KIND_ARRAY:
2827                 local_id = btf_array(local_type)->type;
2828                 targ_id = btf_array(targ_type)->type;
2829                 goto recur;
2830         default:
2831                 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
2832                         btf_kind(local_type), local_id, targ_id);
2833                 return 0;
2834         }
2835 }
2836
2837 /*
2838  * Given single high-level named field accessor in local type, find
2839  * corresponding high-level accessor for a target type. Along the way,
2840  * maintain low-level spec for target as well. Also keep updating target
2841  * bit offset.
2842  *
2843  * Searching is performed through recursive exhaustive enumeration of all
2844  * fields of a struct/union. If there are any anonymous (embedded)
2845  * structs/unions, they are recursively searched as well. If field with
2846  * desired name is found, check compatibility between local and target types,
2847  * before returning result.
2848  *
2849  * 1 is returned, if field is found.
2850  * 0 is returned if no compatible field is found.
2851  * <0 is returned on error.
2852  */
2853 static int bpf_core_match_member(const struct btf *local_btf,
2854                                  const struct bpf_core_accessor *local_acc,
2855                                  const struct btf *targ_btf,
2856                                  __u32 targ_id,
2857                                  struct bpf_core_spec *spec,
2858                                  __u32 *next_targ_id)
2859 {
2860         const struct btf_type *local_type, *targ_type;
2861         const struct btf_member *local_member, *m;
2862         const char *local_name, *targ_name;
2863         __u32 local_id;
2864         int i, n, found;
2865
2866         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
2867         if (!targ_type)
2868                 return -EINVAL;
2869         if (!btf_is_composite(targ_type))
2870                 return 0;
2871
2872         local_id = local_acc->type_id;
2873         local_type = btf__type_by_id(local_btf, local_id);
2874         local_member = btf_members(local_type) + local_acc->idx;
2875         local_name = btf__name_by_offset(local_btf, local_member->name_off);
2876
2877         n = btf_vlen(targ_type);
2878         m = btf_members(targ_type);
2879         for (i = 0; i < n; i++, m++) {
2880                 __u32 bit_offset;
2881
2882                 bit_offset = btf_member_bit_offset(targ_type, i);
2883
2884                 /* too deep struct/union/array nesting */
2885                 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
2886                         return -E2BIG;
2887
2888                 /* speculate this member will be the good one */
2889                 spec->bit_offset += bit_offset;
2890                 spec->raw_spec[spec->raw_len++] = i;
2891
2892                 targ_name = btf__name_by_offset(targ_btf, m->name_off);
2893                 if (str_is_empty(targ_name)) {
2894                         /* embedded struct/union, we need to go deeper */
2895                         found = bpf_core_match_member(local_btf, local_acc,
2896                                                       targ_btf, m->type,
2897                                                       spec, next_targ_id);
2898                         if (found) /* either found or error */
2899                                 return found;
2900                 } else if (strcmp(local_name, targ_name) == 0) {
2901                         /* matching named field */
2902                         struct bpf_core_accessor *targ_acc;
2903
2904                         targ_acc = &spec->spec[spec->len++];
2905                         targ_acc->type_id = targ_id;
2906                         targ_acc->idx = i;
2907                         targ_acc->name = targ_name;
2908
2909                         *next_targ_id = m->type;
2910                         found = bpf_core_fields_are_compat(local_btf,
2911                                                            local_member->type,
2912                                                            targ_btf, m->type);
2913                         if (!found)
2914                                 spec->len--; /* pop accessor */
2915                         return found;
2916                 }
2917                 /* member turned out not to be what we looked for */
2918                 spec->bit_offset -= bit_offset;
2919                 spec->raw_len--;
2920         }
2921
2922         return 0;
2923 }
2924
2925 /*
2926  * Try to match local spec to a target type and, if successful, produce full
2927  * target spec (high-level, low-level + bit offset).
2928  */
2929 static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
2930                                const struct btf *targ_btf, __u32 targ_id,
2931                                struct bpf_core_spec *targ_spec)
2932 {
2933         const struct btf_type *targ_type;
2934         const struct bpf_core_accessor *local_acc;
2935         struct bpf_core_accessor *targ_acc;
2936         int i, sz, matched;
2937
2938         memset(targ_spec, 0, sizeof(*targ_spec));
2939         targ_spec->btf = targ_btf;
2940
2941         local_acc = &local_spec->spec[0];
2942         targ_acc = &targ_spec->spec[0];
2943
2944         for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
2945                 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
2946                                                    &targ_id);
2947                 if (!targ_type)
2948                         return -EINVAL;
2949
2950                 if (local_acc->name) {
2951                         matched = bpf_core_match_member(local_spec->btf,
2952                                                         local_acc,
2953                                                         targ_btf, targ_id,
2954                                                         targ_spec, &targ_id);
2955                         if (matched <= 0)
2956                                 return matched;
2957                 } else {
2958                         /* for i=0, targ_id is already treated as array element
2959                          * type (because it's the original struct), for others
2960                          * we should find array element type first
2961                          */
2962                         if (i > 0) {
2963                                 const struct btf_array *a;
2964
2965                                 if (!btf_is_array(targ_type))
2966                                         return 0;
2967
2968                                 a = btf_array(targ_type);
2969                                 if (local_acc->idx >= a->nelems)
2970                                         return 0;
2971                                 if (!skip_mods_and_typedefs(targ_btf, a->type,
2972                                                             &targ_id))
2973                                         return -EINVAL;
2974                         }
2975
2976                         /* too deep struct/union/array nesting */
2977                         if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
2978                                 return -E2BIG;
2979
2980                         targ_acc->type_id = targ_id;
2981                         targ_acc->idx = local_acc->idx;
2982                         targ_acc->name = NULL;
2983                         targ_spec->len++;
2984                         targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
2985                         targ_spec->raw_len++;
2986
2987                         sz = btf__resolve_size(targ_btf, targ_id);
2988                         if (sz < 0)
2989                                 return sz;
2990                         targ_spec->bit_offset += local_acc->idx * sz * 8;
2991                 }
2992         }
2993
2994         return 1;
2995 }
2996
2997 static int bpf_core_calc_field_relo(const struct bpf_program *prog,
2998                                     const struct bpf_field_reloc *relo,
2999                                     const struct bpf_core_spec *spec,
3000                                     __u32 *val, bool *validate)
3001 {
3002         const struct bpf_core_accessor *acc = &spec->spec[spec->len - 1];
3003         const struct btf_type *t = btf__type_by_id(spec->btf, acc->type_id);
3004         __u32 byte_off, byte_sz, bit_off, bit_sz;
3005         const struct btf_member *m;
3006         const struct btf_type *mt;
3007         bool bitfield;
3008         __s64 sz;
3009
3010         /* a[n] accessor needs special handling */
3011         if (!acc->name) {
3012                 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
3013                         *val = spec->bit_offset / 8;
3014                 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
3015                         sz = btf__resolve_size(spec->btf, acc->type_id);
3016                         if (sz < 0)
3017                                 return -EINVAL;
3018                         *val = sz;
3019                 } else {
3020                         pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
3021                                 bpf_program__title(prog, false),
3022                                 relo->kind, relo->insn_off / 8);
3023                         return -EINVAL;
3024                 }
3025                 if (validate)
3026                         *validate = true;
3027                 return 0;
3028         }
3029
3030         m = btf_members(t) + acc->idx;
3031         mt = skip_mods_and_typedefs(spec->btf, m->type, NULL);
3032         bit_off = spec->bit_offset;
3033         bit_sz = btf_member_bitfield_size(t, acc->idx);
3034
3035         bitfield = bit_sz > 0;
3036         if (bitfield) {
3037                 byte_sz = mt->size;
3038                 byte_off = bit_off / 8 / byte_sz * byte_sz;
3039                 /* figure out smallest int size necessary for bitfield load */
3040                 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
3041                         if (byte_sz >= 8) {
3042                                 /* bitfield can't be read with 64-bit read */
3043                                 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
3044                                         bpf_program__title(prog, false),
3045                                         relo->kind, relo->insn_off / 8);
3046                                 return -E2BIG;
3047                         }
3048                         byte_sz *= 2;
3049                         byte_off = bit_off / 8 / byte_sz * byte_sz;
3050                 }
3051         } else {
3052                 sz = btf__resolve_size(spec->btf, m->type);
3053                 if (sz < 0)
3054                         return -EINVAL;
3055                 byte_sz = sz;
3056                 byte_off = spec->bit_offset / 8;
3057                 bit_sz = byte_sz * 8;
3058         }
3059
3060         /* for bitfields, all the relocatable aspects are ambiguous and we
3061          * might disagree with compiler, so turn off validation of expected
3062          * value, except for signedness
3063          */
3064         if (validate)
3065                 *validate = !bitfield;
3066
3067         switch (relo->kind) {
3068         case BPF_FIELD_BYTE_OFFSET:
3069                 *val = byte_off;
3070                 break;
3071         case BPF_FIELD_BYTE_SIZE:
3072                 *val = byte_sz;
3073                 break;
3074         case BPF_FIELD_SIGNED:
3075                 /* enums will be assumed unsigned */
3076                 *val = btf_is_enum(mt) ||
3077                        (btf_int_encoding(mt) & BTF_INT_SIGNED);
3078                 if (validate)
3079                         *validate = true; /* signedness is never ambiguous */
3080                 break;
3081         case BPF_FIELD_LSHIFT_U64:
3082 #if __BYTE_ORDER == __LITTLE_ENDIAN
3083                 *val = 64 - (bit_off + bit_sz - byte_off  * 8);
3084 #else
3085                 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
3086 #endif
3087                 break;
3088         case BPF_FIELD_RSHIFT_U64:
3089                 *val = 64 - bit_sz;
3090                 if (validate)
3091                         *validate = true; /* right shift is never ambiguous */
3092                 break;
3093         case BPF_FIELD_EXISTS:
3094         default:
3095                 pr_warn("prog '%s': unknown relo %d at insn #%d\n",
3096                         bpf_program__title(prog, false),
3097                         relo->kind, relo->insn_off / 8);
3098                 return -EINVAL;
3099         }
3100
3101         return 0;
3102 }
3103
3104 /*
3105  * Patch relocatable BPF instruction.
3106  *
3107  * Patched value is determined by relocation kind and target specification.
3108  * For field existence relocation target spec will be NULL if field is not
3109  * found.
3110  * Expected insn->imm value is determined using relocation kind and local
3111  * spec, and is checked before patching instruction. If actual insn->imm value
3112  * is wrong, bail out with error.
3113  *
3114  * Currently three kinds of BPF instructions are supported:
3115  * 1. rX = <imm> (assignment with immediate operand);
3116  * 2. rX += <imm> (arithmetic operations with immediate operand);
3117  */
3118 static int bpf_core_reloc_insn(struct bpf_program *prog,
3119                                const struct bpf_field_reloc *relo,
3120                                const struct bpf_core_spec *local_spec,
3121                                const struct bpf_core_spec *targ_spec)
3122 {
3123         bool failed = false, validate = true;
3124         __u32 orig_val, new_val;
3125         struct bpf_insn *insn;
3126         int insn_idx, err;
3127         __u8 class;
3128
3129         if (relo->insn_off % sizeof(struct bpf_insn))
3130                 return -EINVAL;
3131         insn_idx = relo->insn_off / sizeof(struct bpf_insn);
3132
3133         if (relo->kind == BPF_FIELD_EXISTS) {
3134                 orig_val = 1; /* can't generate EXISTS relo w/o local field */
3135                 new_val = targ_spec ? 1 : 0;
3136         } else if (!targ_spec) {
3137                 failed = true;
3138                 new_val = (__u32)-1;
3139         } else {
3140                 err = bpf_core_calc_field_relo(prog, relo, local_spec,
3141                                                &orig_val, &validate);
3142                 if (err)
3143                         return err;
3144                 err = bpf_core_calc_field_relo(prog, relo, targ_spec,
3145                                                &new_val, NULL);
3146                 if (err)
3147                         return err;
3148         }
3149
3150         insn = &prog->insns[insn_idx];
3151         class = BPF_CLASS(insn->code);
3152
3153         if (class == BPF_ALU || class == BPF_ALU64) {
3154                 if (BPF_SRC(insn->code) != BPF_K)
3155                         return -EINVAL;
3156                 if (!failed && validate && insn->imm != orig_val) {
3157                         pr_warn("prog '%s': unexpected insn #%d value: got %u, exp %u -> %u\n",
3158                                 bpf_program__title(prog, false), insn_idx,
3159                                 insn->imm, orig_val, new_val);
3160                         return -EINVAL;
3161                 }
3162                 orig_val = insn->imm;
3163                 insn->imm = new_val;
3164                 pr_debug("prog '%s': patched insn #%d (ALU/ALU64)%s imm %u -> %u\n",
3165                          bpf_program__title(prog, false), insn_idx,
3166                          failed ? " w/ failed reloc" : "", orig_val, new_val);
3167         } else {
3168                 pr_warn("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",
3169                         bpf_program__title(prog, false),
3170                         insn_idx, insn->code, insn->src_reg, insn->dst_reg,
3171                         insn->off, insn->imm);
3172                 return -EINVAL;
3173         }
3174
3175         return 0;
3176 }
3177
3178 static struct btf *btf_load_raw(const char *path)
3179 {
3180         struct btf *btf;
3181         size_t read_cnt;
3182         struct stat st;
3183         void *data;
3184         FILE *f;
3185
3186         if (stat(path, &st))
3187                 return ERR_PTR(-errno);
3188
3189         data = malloc(st.st_size);
3190         if (!data)
3191                 return ERR_PTR(-ENOMEM);
3192
3193         f = fopen(path, "rb");
3194         if (!f) {
3195                 btf = ERR_PTR(-errno);
3196                 goto cleanup;
3197         }
3198
3199         read_cnt = fread(data, 1, st.st_size, f);
3200         fclose(f);
3201         if (read_cnt < st.st_size) {
3202                 btf = ERR_PTR(-EBADF);
3203                 goto cleanup;
3204         }
3205
3206         btf = btf__new(data, read_cnt);
3207
3208 cleanup:
3209         free(data);
3210         return btf;
3211 }
3212
3213 /*
3214  * Probe few well-known locations for vmlinux kernel image and try to load BTF
3215  * data out of it to use for target BTF.
3216  */
3217 static struct btf *bpf_core_find_kernel_btf(void)
3218 {
3219         struct {
3220                 const char *path_fmt;
3221                 bool raw_btf;
3222         } locations[] = {
3223                 /* try canonical vmlinux BTF through sysfs first */
3224                 { "/sys/kernel/btf/vmlinux", true /* raw BTF */ },
3225                 /* fall back to trying to find vmlinux ELF on disk otherwise */
3226                 { "/boot/vmlinux-%1$s" },
3227                 { "/lib/modules/%1$s/vmlinux-%1$s" },
3228                 { "/lib/modules/%1$s/build/vmlinux" },
3229                 { "/usr/lib/modules/%1$s/kernel/vmlinux" },
3230                 { "/usr/lib/debug/boot/vmlinux-%1$s" },
3231                 { "/usr/lib/debug/boot/vmlinux-%1$s.debug" },
3232                 { "/usr/lib/debug/lib/modules/%1$s/vmlinux" },
3233         };
3234         char path[PATH_MAX + 1];
3235         struct utsname buf;
3236         struct btf *btf;
3237         int i;
3238
3239         uname(&buf);
3240
3241         for (i = 0; i < ARRAY_SIZE(locations); i++) {
3242                 snprintf(path, PATH_MAX, locations[i].path_fmt, buf.release);
3243
3244                 if (access(path, R_OK))
3245                         continue;
3246
3247                 if (locations[i].raw_btf)
3248                         btf = btf_load_raw(path);
3249                 else
3250                         btf = btf__parse_elf(path, NULL);
3251
3252                 pr_debug("loading kernel BTF '%s': %ld\n",
3253                          path, IS_ERR(btf) ? PTR_ERR(btf) : 0);
3254                 if (IS_ERR(btf))
3255                         continue;
3256
3257                 return btf;
3258         }
3259
3260         pr_warn("failed to find valid kernel BTF\n");
3261         return ERR_PTR(-ESRCH);
3262 }
3263
3264 /* Output spec definition in the format:
3265  * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
3266  * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
3267  */
3268 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
3269 {
3270         const struct btf_type *t;
3271         const char *s;
3272         __u32 type_id;
3273         int i;
3274
3275         type_id = spec->spec[0].type_id;
3276         t = btf__type_by_id(spec->btf, type_id);
3277         s = btf__name_by_offset(spec->btf, t->name_off);
3278         libbpf_print(level, "[%u] %s + ", type_id, s);
3279
3280         for (i = 0; i < spec->raw_len; i++)
3281                 libbpf_print(level, "%d%s", spec->raw_spec[i],
3282                              i == spec->raw_len - 1 ? " => " : ":");
3283
3284         libbpf_print(level, "%u.%u @ &x",
3285                      spec->bit_offset / 8, spec->bit_offset % 8);
3286
3287         for (i = 0; i < spec->len; i++) {
3288                 if (spec->spec[i].name)
3289                         libbpf_print(level, ".%s", spec->spec[i].name);
3290                 else
3291                         libbpf_print(level, "[%u]", spec->spec[i].idx);
3292         }
3293
3294 }
3295
3296 static size_t bpf_core_hash_fn(const void *key, void *ctx)
3297 {
3298         return (size_t)key;
3299 }
3300
3301 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
3302 {
3303         return k1 == k2;
3304 }
3305
3306 static void *u32_as_hash_key(__u32 x)
3307 {
3308         return (void *)(uintptr_t)x;
3309 }
3310
3311 /*
3312  * CO-RE relocate single instruction.
3313  *
3314  * The outline and important points of the algorithm:
3315  * 1. For given local type, find corresponding candidate target types.
3316  *    Candidate type is a type with the same "essential" name, ignoring
3317  *    everything after last triple underscore (___). E.g., `sample`,
3318  *    `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
3319  *    for each other. Names with triple underscore are referred to as
3320  *    "flavors" and are useful, among other things, to allow to
3321  *    specify/support incompatible variations of the same kernel struct, which
3322  *    might differ between different kernel versions and/or build
3323  *    configurations.
3324  *
3325  *    N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
3326  *    converter, when deduplicated BTF of a kernel still contains more than
3327  *    one different types with the same name. In that case, ___2, ___3, etc
3328  *    are appended starting from second name conflict. But start flavors are
3329  *    also useful to be defined "locally", in BPF program, to extract same
3330  *    data from incompatible changes between different kernel
3331  *    versions/configurations. For instance, to handle field renames between
3332  *    kernel versions, one can use two flavors of the struct name with the
3333  *    same common name and use conditional relocations to extract that field,
3334  *    depending on target kernel version.
3335  * 2. For each candidate type, try to match local specification to this
3336  *    candidate target type. Matching involves finding corresponding
3337  *    high-level spec accessors, meaning that all named fields should match,
3338  *    as well as all array accesses should be within the actual bounds. Also,
3339  *    types should be compatible (see bpf_core_fields_are_compat for details).
3340  * 3. It is supported and expected that there might be multiple flavors
3341  *    matching the spec. As long as all the specs resolve to the same set of
3342  *    offsets across all candidates, there is no error. If there is any
3343  *    ambiguity, CO-RE relocation will fail. This is necessary to accomodate
3344  *    imprefection of BTF deduplication, which can cause slight duplication of
3345  *    the same BTF type, if some directly or indirectly referenced (by
3346  *    pointer) type gets resolved to different actual types in different
3347  *    object files. If such situation occurs, deduplicated BTF will end up
3348  *    with two (or more) structurally identical types, which differ only in
3349  *    types they refer to through pointer. This should be OK in most cases and
3350  *    is not an error.
3351  * 4. Candidate types search is performed by linearly scanning through all
3352  *    types in target BTF. It is anticipated that this is overall more
3353  *    efficient memory-wise and not significantly worse (if not better)
3354  *    CPU-wise compared to prebuilding a map from all local type names to
3355  *    a list of candidate type names. It's also sped up by caching resolved
3356  *    list of matching candidates per each local "root" type ID, that has at
3357  *    least one bpf_field_reloc associated with it. This list is shared
3358  *    between multiple relocations for the same type ID and is updated as some
3359  *    of the candidates are pruned due to structural incompatibility.
3360  */
3361 static int bpf_core_reloc_field(struct bpf_program *prog,
3362                                  const struct bpf_field_reloc *relo,
3363                                  int relo_idx,
3364                                  const struct btf *local_btf,
3365                                  const struct btf *targ_btf,
3366                                  struct hashmap *cand_cache)
3367 {
3368         const char *prog_name = bpf_program__title(prog, false);
3369         struct bpf_core_spec local_spec, cand_spec, targ_spec;
3370         const void *type_key = u32_as_hash_key(relo->type_id);
3371         const struct btf_type *local_type, *cand_type;
3372         const char *local_name, *cand_name;
3373         struct ids_vec *cand_ids;
3374         __u32 local_id, cand_id;
3375         const char *spec_str;
3376         int i, j, err;
3377
3378         local_id = relo->type_id;
3379         local_type = btf__type_by_id(local_btf, local_id);
3380         if (!local_type)
3381                 return -EINVAL;
3382
3383         local_name = btf__name_by_offset(local_btf, local_type->name_off);
3384         if (str_is_empty(local_name))
3385                 return -EINVAL;
3386
3387         spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
3388         if (str_is_empty(spec_str))
3389                 return -EINVAL;
3390
3391         err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec);
3392         if (err) {
3393                 pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n",
3394                         prog_name, relo_idx, local_id, local_name, spec_str,
3395                         err);
3396                 return -EINVAL;
3397         }
3398
3399         pr_debug("prog '%s': relo #%d: kind %d, spec is ", prog_name, relo_idx,
3400                  relo->kind);
3401         bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
3402         libbpf_print(LIBBPF_DEBUG, "\n");
3403
3404         if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) {
3405                 cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf);
3406                 if (IS_ERR(cand_ids)) {
3407                         pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld",
3408                                 prog_name, relo_idx, local_id, local_name,
3409                                 PTR_ERR(cand_ids));
3410                         return PTR_ERR(cand_ids);
3411                 }
3412                 err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL);
3413                 if (err) {
3414                         bpf_core_free_cands(cand_ids);
3415                         return err;
3416                 }
3417         }
3418
3419         for (i = 0, j = 0; i < cand_ids->len; i++) {
3420                 cand_id = cand_ids->data[i];
3421                 cand_type = btf__type_by_id(targ_btf, cand_id);
3422                 cand_name = btf__name_by_offset(targ_btf, cand_type->name_off);
3423
3424                 err = bpf_core_spec_match(&local_spec, targ_btf,
3425                                           cand_id, &cand_spec);
3426                 pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ",
3427                          prog_name, relo_idx, i, cand_name);
3428                 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
3429                 libbpf_print(LIBBPF_DEBUG, ": %d\n", err);
3430                 if (err < 0) {
3431                         pr_warn("prog '%s': relo #%d: matching error: %d\n",
3432                                 prog_name, relo_idx, err);
3433                         return err;
3434                 }
3435                 if (err == 0)
3436                         continue;
3437
3438                 if (j == 0) {
3439                         targ_spec = cand_spec;
3440                 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
3441                         /* if there are many candidates, they should all
3442                          * resolve to the same bit offset
3443                          */
3444                         pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n",
3445                                 prog_name, relo_idx, cand_spec.bit_offset,
3446                                 targ_spec.bit_offset);
3447                         return -EINVAL;
3448                 }
3449
3450                 cand_ids->data[j++] = cand_spec.spec[0].type_id;
3451         }
3452
3453         /*
3454          * For BPF_FIELD_EXISTS relo or when relaxed CO-RE reloc mode is
3455          * requested, it's expected that we might not find any candidates.
3456          * In this case, if field wasn't found in any candidate, the list of
3457          * candidates shouldn't change at all, we'll just handle relocating
3458          * appropriately, depending on relo's kind.
3459          */
3460         if (j > 0)
3461                 cand_ids->len = j;
3462
3463         if (j == 0 && !prog->obj->relaxed_core_relocs &&
3464             relo->kind != BPF_FIELD_EXISTS) {
3465                 pr_warn("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n",
3466                         prog_name, relo_idx, local_id, local_name, spec_str);
3467                 return -ESRCH;
3468         }
3469
3470         /* bpf_core_reloc_insn should know how to handle missing targ_spec */
3471         err = bpf_core_reloc_insn(prog, relo, &local_spec,
3472                                   j ? &targ_spec : NULL);
3473         if (err) {
3474                 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
3475                         prog_name, relo_idx, relo->insn_off, err);
3476                 return -EINVAL;
3477         }
3478
3479         return 0;
3480 }
3481
3482 static int
3483 bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path)
3484 {
3485         const struct btf_ext_info_sec *sec;
3486         const struct bpf_field_reloc *rec;
3487         const struct btf_ext_info *seg;
3488         struct hashmap_entry *entry;
3489         struct hashmap *cand_cache = NULL;
3490         struct bpf_program *prog;
3491         struct btf *targ_btf;
3492         const char *sec_name;
3493         int i, err = 0;
3494
3495         if (targ_btf_path)
3496                 targ_btf = btf__parse_elf(targ_btf_path, NULL);
3497         else
3498                 targ_btf = bpf_core_find_kernel_btf();
3499         if (IS_ERR(targ_btf)) {
3500                 pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
3501                 return PTR_ERR(targ_btf);
3502         }
3503
3504         cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
3505         if (IS_ERR(cand_cache)) {
3506                 err = PTR_ERR(cand_cache);
3507                 goto out;
3508         }
3509
3510         seg = &obj->btf_ext->field_reloc_info;
3511         for_each_btf_ext_sec(seg, sec) {
3512                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3513                 if (str_is_empty(sec_name)) {
3514                         err = -EINVAL;
3515                         goto out;
3516                 }
3517                 prog = bpf_object__find_program_by_title(obj, sec_name);
3518                 if (!prog) {
3519                         pr_warn("failed to find program '%s' for CO-RE offset relocation\n",
3520                                 sec_name);
3521                         err = -EINVAL;
3522                         goto out;
3523                 }
3524
3525                 pr_debug("prog '%s': performing %d CO-RE offset relocs\n",
3526                          sec_name, sec->num_info);
3527
3528                 for_each_btf_ext_rec(seg, sec, i, rec) {
3529                         err = bpf_core_reloc_field(prog, rec, i, obj->btf,
3530                                                    targ_btf, cand_cache);
3531                         if (err) {
3532                                 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
3533                                         sec_name, i, err);
3534                                 goto out;
3535                         }
3536                 }
3537         }
3538
3539 out:
3540         btf__free(targ_btf);
3541         if (!IS_ERR_OR_NULL(cand_cache)) {
3542                 hashmap__for_each_entry(cand_cache, entry, i) {
3543                         bpf_core_free_cands(entry->value);
3544                 }
3545                 hashmap__free(cand_cache);
3546         }
3547         return err;
3548 }
3549
3550 static int
3551 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
3552 {
3553         int err = 0;
3554
3555         if (obj->btf_ext->field_reloc_info.len)
3556                 err = bpf_core_reloc_fields(obj, targ_btf_path);
3557
3558         return err;
3559 }
3560
3561 static int
3562 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
3563                         struct reloc_desc *relo)
3564 {
3565         struct bpf_insn *insn, *new_insn;
3566         struct bpf_program *text;
3567         size_t new_cnt;
3568         int err;
3569
3570         if (relo->type != RELO_CALL)
3571                 return -LIBBPF_ERRNO__RELOC;
3572
3573         if (prog->idx == obj->efile.text_shndx) {
3574                 pr_warn("relo in .text insn %d into off %d (insn #%d)\n",
3575                         relo->insn_idx, relo->sym_off, relo->sym_off / 8);
3576                 return -LIBBPF_ERRNO__RELOC;
3577         }
3578
3579         if (prog->main_prog_cnt == 0) {
3580                 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
3581                 if (!text) {
3582                         pr_warn("no .text section found yet relo into text exist\n");
3583                         return -LIBBPF_ERRNO__RELOC;
3584                 }
3585                 new_cnt = prog->insns_cnt + text->insns_cnt;
3586                 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
3587                 if (!new_insn) {
3588                         pr_warn("oom in prog realloc\n");
3589                         return -ENOMEM;
3590                 }
3591                 prog->insns = new_insn;
3592
3593                 if (obj->btf_ext) {
3594                         err = bpf_program_reloc_btf_ext(prog, obj,
3595                                                         text->section_name,
3596                                                         prog->insns_cnt);
3597                         if (err)
3598                                 return err;
3599                 }
3600
3601                 memcpy(new_insn + prog->insns_cnt, text->insns,
3602                        text->insns_cnt * sizeof(*insn));
3603                 prog->main_prog_cnt = prog->insns_cnt;
3604                 prog->insns_cnt = new_cnt;
3605                 pr_debug("added %zd insn from %s to prog %s\n",
3606                          text->insns_cnt, text->section_name,
3607                          prog->section_name);
3608         }
3609         insn = &prog->insns[relo->insn_idx];
3610         insn->imm += relo->sym_off / 8 + prog->main_prog_cnt - relo->insn_idx;
3611         return 0;
3612 }
3613
3614 static int
3615 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
3616 {
3617         int i, err;
3618
3619         if (!prog)
3620                 return 0;
3621
3622         if (obj->btf_ext) {
3623                 err = bpf_program_reloc_btf_ext(prog, obj,
3624                                                 prog->section_name, 0);
3625                 if (err)
3626                         return err;
3627         }
3628
3629         if (!prog->reloc_desc)
3630                 return 0;
3631
3632         for (i = 0; i < prog->nr_reloc; i++) {
3633                 struct reloc_desc *relo = &prog->reloc_desc[i];
3634
3635                 if (relo->type == RELO_LD64 || relo->type == RELO_DATA) {
3636                         struct bpf_insn *insn = &prog->insns[relo->insn_idx];
3637
3638                         if (relo->insn_idx + 1 >= (int)prog->insns_cnt) {
3639                                 pr_warn("relocation out of range: '%s'\n",
3640                                         prog->section_name);
3641                                 return -LIBBPF_ERRNO__RELOC;
3642                         }
3643
3644                         if (relo->type != RELO_DATA) {
3645                                 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
3646                         } else {
3647                                 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
3648                                 insn[1].imm = insn[0].imm + relo->sym_off;
3649                         }
3650                         insn[0].imm = obj->maps[relo->map_idx].fd;
3651                 } else if (relo->type == RELO_CALL) {
3652                         err = bpf_program__reloc_text(prog, obj, relo);
3653                         if (err)
3654                                 return err;
3655                 }
3656         }
3657
3658         zfree(&prog->reloc_desc);
3659         prog->nr_reloc = 0;
3660         return 0;
3661 }
3662
3663 static int
3664 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
3665 {
3666         struct bpf_program *prog;
3667         size_t i;
3668         int err;
3669
3670         if (obj->btf_ext) {
3671                 err = bpf_object__relocate_core(obj, targ_btf_path);
3672                 if (err) {
3673                         pr_warn("failed to perform CO-RE relocations: %d\n",
3674                                 err);
3675                         return err;
3676                 }
3677         }
3678         for (i = 0; i < obj->nr_programs; i++) {
3679                 prog = &obj->programs[i];
3680
3681                 err = bpf_program__relocate(prog, obj);
3682                 if (err) {
3683                         pr_warn("failed to relocate '%s'\n", prog->section_name);
3684                         return err;
3685                 }
3686         }
3687         return 0;
3688 }
3689
3690 static int bpf_object__collect_reloc(struct bpf_object *obj)
3691 {
3692         int i, err;
3693
3694         if (!obj_elf_valid(obj)) {
3695                 pr_warn("Internal error: elf object is closed\n");
3696                 return -LIBBPF_ERRNO__INTERNAL;
3697         }
3698
3699         for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
3700                 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
3701                 Elf_Data *data = obj->efile.reloc_sects[i].data;
3702                 int idx = shdr->sh_info;
3703                 struct bpf_program *prog;
3704
3705                 if (shdr->sh_type != SHT_REL) {
3706                         pr_warn("internal error at %d\n", __LINE__);
3707                         return -LIBBPF_ERRNO__INTERNAL;
3708                 }
3709
3710                 prog = bpf_object__find_prog_by_idx(obj, idx);
3711                 if (!prog) {
3712                         pr_warn("relocation failed: no section(%d)\n", idx);
3713                         return -LIBBPF_ERRNO__RELOC;
3714                 }
3715
3716                 err = bpf_program__collect_reloc(prog, shdr, data, obj);
3717                 if (err)
3718                         return err;
3719         }
3720         return 0;
3721 }
3722
3723 static int
3724 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
3725              char *license, __u32 kern_version, int *pfd)
3726 {
3727         struct bpf_load_program_attr load_attr;
3728         char *cp, errmsg[STRERR_BUFSIZE];
3729         int log_buf_size = BPF_LOG_BUF_SIZE;
3730         char *log_buf;
3731         int btf_fd, ret;
3732
3733         if (!insns || !insns_cnt)
3734                 return -EINVAL;
3735
3736         memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
3737         load_attr.prog_type = prog->type;
3738         load_attr.expected_attach_type = prog->expected_attach_type;
3739         if (prog->caps->name)
3740                 load_attr.name = prog->name;
3741         load_attr.insns = insns;
3742         load_attr.insns_cnt = insns_cnt;
3743         load_attr.license = license;
3744         if (prog->type == BPF_PROG_TYPE_TRACING) {
3745                 load_attr.attach_prog_fd = prog->attach_prog_fd;
3746                 load_attr.attach_btf_id = prog->attach_btf_id;
3747         } else {
3748                 load_attr.kern_version = kern_version;
3749                 load_attr.prog_ifindex = prog->prog_ifindex;
3750         }
3751         /* if .BTF.ext was loaded, kernel supports associated BTF for prog */
3752         if (prog->obj->btf_ext)
3753                 btf_fd = bpf_object__btf_fd(prog->obj);
3754         else
3755                 btf_fd = -1;
3756         load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0;
3757         load_attr.func_info = prog->func_info;
3758         load_attr.func_info_rec_size = prog->func_info_rec_size;
3759         load_attr.func_info_cnt = prog->func_info_cnt;
3760         load_attr.line_info = prog->line_info;
3761         load_attr.line_info_rec_size = prog->line_info_rec_size;
3762         load_attr.line_info_cnt = prog->line_info_cnt;
3763         load_attr.log_level = prog->log_level;
3764         load_attr.prog_flags = prog->prog_flags;
3765
3766 retry_load:
3767         log_buf = malloc(log_buf_size);
3768         if (!log_buf)
3769                 pr_warn("Alloc log buffer for bpf loader error, continue without log\n");
3770
3771         ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);
3772
3773         if (ret >= 0) {
3774                 if (load_attr.log_level)
3775                         pr_debug("verifier log:\n%s", log_buf);
3776                 *pfd = ret;
3777                 ret = 0;
3778                 goto out;
3779         }
3780
3781         if (errno == ENOSPC) {
3782                 log_buf_size <<= 1;
3783                 free(log_buf);
3784                 goto retry_load;
3785         }
3786         ret = -errno;
3787         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3788         pr_warn("load bpf program failed: %s\n", cp);
3789
3790         if (log_buf && log_buf[0] != '\0') {
3791                 ret = -LIBBPF_ERRNO__VERIFY;
3792                 pr_warn("-- BEGIN DUMP LOG ---\n");
3793                 pr_warn("\n%s\n", log_buf);
3794                 pr_warn("-- END LOG --\n");
3795         } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
3796                 pr_warn("Program too large (%zu insns), at most %d insns\n",
3797                         load_attr.insns_cnt, BPF_MAXINSNS);
3798                 ret = -LIBBPF_ERRNO__PROG2BIG;
3799         } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
3800                 /* Wrong program type? */
3801                 int fd;
3802
3803                 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
3804                 load_attr.expected_attach_type = 0;
3805                 fd = bpf_load_program_xattr(&load_attr, NULL, 0);
3806                 if (fd >= 0) {
3807                         close(fd);
3808                         ret = -LIBBPF_ERRNO__PROGTYPE;
3809                         goto out;
3810                 }
3811         }
3812
3813 out:
3814         free(log_buf);
3815         return ret;
3816 }
3817
3818 static int libbpf_find_attach_btf_id(const char *name,
3819                                      enum bpf_attach_type attach_type,
3820                                      __u32 attach_prog_fd);
3821
3822 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
3823 {
3824         int err = 0, fd, i, btf_id;
3825
3826         if (prog->type == BPF_PROG_TYPE_TRACING) {
3827                 btf_id = libbpf_find_attach_btf_id(prog->section_name,
3828                                                    prog->expected_attach_type,
3829                                                    prog->attach_prog_fd);
3830                 if (btf_id <= 0)
3831                         return btf_id;
3832                 prog->attach_btf_id = btf_id;
3833         }
3834
3835         if (prog->instances.nr < 0 || !prog->instances.fds) {
3836                 if (prog->preprocessor) {
3837                         pr_warn("Internal error: can't load program '%s'\n",
3838                                 prog->section_name);
3839                         return -LIBBPF_ERRNO__INTERNAL;
3840                 }
3841
3842                 prog->instances.fds = malloc(sizeof(int));
3843                 if (!prog->instances.fds) {
3844                         pr_warn("Not enough memory for BPF fds\n");
3845                         return -ENOMEM;
3846                 }
3847                 prog->instances.nr = 1;
3848                 prog->instances.fds[0] = -1;
3849         }
3850
3851         if (!prog->preprocessor) {
3852                 if (prog->instances.nr != 1) {
3853                         pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n",
3854                                 prog->section_name, prog->instances.nr);
3855                 }
3856                 err = load_program(prog, prog->insns, prog->insns_cnt,
3857                                    license, kern_ver, &fd);
3858                 if (!err)
3859                         prog->instances.fds[0] = fd;
3860                 goto out;
3861         }
3862
3863         for (i = 0; i < prog->instances.nr; i++) {
3864                 struct bpf_prog_prep_result result;
3865                 bpf_program_prep_t preprocessor = prog->preprocessor;
3866
3867                 memset(&result, 0, sizeof(result));
3868                 err = preprocessor(prog, i, prog->insns,
3869                                    prog->insns_cnt, &result);
3870                 if (err) {
3871                         pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
3872                                 i, prog->section_name);
3873                         goto out;
3874                 }
3875
3876                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
3877                         pr_debug("Skip loading the %dth instance of program '%s'\n",
3878                                  i, prog->section_name);
3879                         prog->instances.fds[i] = -1;
3880                         if (result.pfd)
3881                                 *result.pfd = -1;
3882                         continue;
3883                 }
3884
3885                 err = load_program(prog, result.new_insn_ptr,
3886                                    result.new_insn_cnt, license, kern_ver, &fd);
3887                 if (err) {
3888                         pr_warn("Loading the %dth instance of program '%s' failed\n",
3889                                 i, prog->section_name);
3890                         goto out;
3891                 }
3892
3893                 if (result.pfd)
3894                         *result.pfd = fd;
3895                 prog->instances.fds[i] = fd;
3896         }
3897 out:
3898         if (err)
3899                 pr_warn("failed to load program '%s'\n", prog->section_name);
3900         zfree(&prog->insns);
3901         prog->insns_cnt = 0;
3902         return err;
3903 }
3904
3905 static bool bpf_program__is_function_storage(const struct bpf_program *prog,
3906                                              const struct bpf_object *obj)
3907 {
3908         return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
3909 }
3910
3911 static int
3912 bpf_object__load_progs(struct bpf_object *obj, int log_level)
3913 {
3914         size_t i;
3915         int err;
3916
3917         for (i = 0; i < obj->nr_programs; i++) {
3918                 if (bpf_program__is_function_storage(&obj->programs[i], obj))
3919                         continue;
3920                 obj->programs[i].log_level |= log_level;
3921                 err = bpf_program__load(&obj->programs[i],
3922                                         obj->license,
3923                                         obj->kern_version);
3924                 if (err)
3925                         return err;
3926         }
3927         return 0;
3928 }
3929
3930 static struct bpf_object *
3931 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
3932                    const struct bpf_object_open_opts *opts)
3933 {
3934         struct bpf_program *prog;
3935         struct bpf_object *obj;
3936         const char *obj_name;
3937         char tmp_name[64];
3938         __u32 attach_prog_fd;
3939         int err;
3940
3941         if (elf_version(EV_CURRENT) == EV_NONE) {
3942                 pr_warn("failed to init libelf for %s\n",
3943                         path ? : "(mem buf)");
3944                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
3945         }
3946
3947         if (!OPTS_VALID(opts, bpf_object_open_opts))
3948                 return ERR_PTR(-EINVAL);
3949
3950         obj_name = OPTS_GET(opts, object_name, NULL);
3951         if (obj_buf) {
3952                 if (!obj_name) {
3953                         snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
3954                                  (unsigned long)obj_buf,
3955                                  (unsigned long)obj_buf_sz);
3956                         obj_name = tmp_name;
3957                 }
3958                 path = obj_name;
3959                 pr_debug("loading object '%s' from buffer\n", obj_name);
3960         }
3961
3962         obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
3963         if (IS_ERR(obj))
3964                 return obj;
3965
3966         obj->relaxed_core_relocs = OPTS_GET(opts, relaxed_core_relocs, false);
3967         attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
3968
3969         err = bpf_object__elf_init(obj);
3970         err = err ? : bpf_object__check_endianness(obj);
3971         err = err ? : bpf_object__elf_collect(obj);
3972         err = err ? : bpf_object__init_maps(obj, opts);
3973         err = err ? : bpf_object__init_prog_names(obj);
3974         err = err ? : bpf_object__collect_reloc(obj);
3975         if (err)
3976                 goto out;
3977         bpf_object__elf_finish(obj);
3978
3979         bpf_object__for_each_program(prog, obj) {
3980                 enum bpf_prog_type prog_type;
3981                 enum bpf_attach_type attach_type;
3982
3983                 err = libbpf_prog_type_by_name(prog->section_name, &prog_type,
3984                                                &attach_type);
3985                 if (err == -ESRCH)
3986                         /* couldn't guess, but user might manually specify */
3987                         continue;
3988                 if (err)
3989                         goto out;
3990
3991                 bpf_program__set_type(prog, prog_type);
3992                 bpf_program__set_expected_attach_type(prog, attach_type);
3993                 if (prog_type == BPF_PROG_TYPE_TRACING)
3994                         prog->attach_prog_fd = attach_prog_fd;
3995         }
3996
3997         return obj;
3998 out:
3999         bpf_object__close(obj);
4000         return ERR_PTR(err);
4001 }
4002
4003 static struct bpf_object *
4004 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
4005 {
4006         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
4007                 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
4008         );
4009
4010         /* param validation */
4011         if (!attr->file)
4012                 return NULL;
4013
4014         pr_debug("loading %s\n", attr->file);
4015         return __bpf_object__open(attr->file, NULL, 0, &opts);
4016 }
4017
4018 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
4019 {
4020         return __bpf_object__open_xattr(attr, 0);
4021 }
4022
4023 struct bpf_object *bpf_object__open(const char *path)
4024 {
4025         struct bpf_object_open_attr attr = {
4026                 .file           = path,
4027                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
4028         };
4029
4030         return bpf_object__open_xattr(&attr);
4031 }
4032
4033 struct bpf_object *
4034 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
4035 {
4036         if (!path)
4037                 return ERR_PTR(-EINVAL);
4038
4039         pr_debug("loading %s\n", path);
4040
4041         return __bpf_object__open(path, NULL, 0, opts);
4042 }
4043
4044 struct bpf_object *
4045 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
4046                      const struct bpf_object_open_opts *opts)
4047 {
4048         if (!obj_buf || obj_buf_sz == 0)
4049                 return ERR_PTR(-EINVAL);
4050
4051         return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
4052 }
4053
4054 struct bpf_object *
4055 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
4056                         const char *name)
4057 {
4058         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
4059                 .object_name = name,
4060                 /* wrong default, but backwards-compatible */
4061                 .relaxed_maps = true,
4062         );
4063
4064         /* returning NULL is wrong, but backwards-compatible */
4065         if (!obj_buf || obj_buf_sz == 0)
4066                 return NULL;
4067
4068         return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
4069 }
4070
4071 int bpf_object__unload(struct bpf_object *obj)
4072 {
4073         size_t i;
4074
4075         if (!obj)
4076                 return -EINVAL;
4077
4078         for (i = 0; i < obj->nr_maps; i++)
4079                 zclose(obj->maps[i].fd);
4080
4081         for (i = 0; i < obj->nr_programs; i++)
4082                 bpf_program__unload(&obj->programs[i]);
4083
4084         return 0;
4085 }
4086
4087 static int bpf_object__sanitize_maps(struct bpf_object *obj)
4088 {
4089         struct bpf_map *m;
4090
4091         bpf_object__for_each_map(m, obj) {
4092                 if (!bpf_map__is_internal(m))
4093                         continue;
4094                 if (!obj->caps.global_data) {
4095                         pr_warn("kernel doesn't support global data\n");
4096                         return -ENOTSUP;
4097                 }
4098                 if (!obj->caps.array_mmap)
4099                         m->def.map_flags ^= BPF_F_MMAPABLE;
4100         }
4101
4102         return 0;
4103 }
4104
4105 int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
4106 {
4107         struct bpf_object *obj;
4108         int err, i;
4109
4110         if (!attr)
4111                 return -EINVAL;
4112         obj = attr->obj;
4113         if (!obj)
4114                 return -EINVAL;
4115
4116         if (obj->loaded) {
4117                 pr_warn("object should not be loaded twice\n");
4118                 return -EINVAL;
4119         }
4120
4121         obj->loaded = true;
4122
4123         err = bpf_object__probe_caps(obj);
4124         err = err ? : bpf_object__sanitize_and_load_btf(obj);
4125         err = err ? : bpf_object__sanitize_maps(obj);
4126         err = err ? : bpf_object__create_maps(obj);
4127         err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
4128         err = err ? : bpf_object__load_progs(obj, attr->log_level);
4129         if (err)
4130                 goto out;
4131
4132         return 0;
4133 out:
4134         /* unpin any maps that were auto-pinned during load */
4135         for (i = 0; i < obj->nr_maps; i++)
4136                 if (obj->maps[i].pinned && !obj->maps[i].reused)
4137                         bpf_map__unpin(&obj->maps[i], NULL);
4138
4139         bpf_object__unload(obj);
4140         pr_warn("failed to load object '%s'\n", obj->path);
4141         return err;
4142 }
4143
4144 int bpf_object__load(struct bpf_object *obj)
4145 {
4146         struct bpf_object_load_attr attr = {
4147                 .obj = obj,
4148         };
4149
4150         return bpf_object__load_xattr(&attr);
4151 }
4152
4153 static int make_parent_dir(const char *path)
4154 {
4155         char *cp, errmsg[STRERR_BUFSIZE];
4156         char *dname, *dir;
4157         int err = 0;
4158
4159         dname = strdup(path);
4160         if (dname == NULL)
4161                 return -ENOMEM;
4162
4163         dir = dirname(dname);
4164         if (mkdir(dir, 0700) && errno != EEXIST)
4165                 err = -errno;
4166
4167         free(dname);
4168         if (err) {
4169                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4170                 pr_warn("failed to mkdir %s: %s\n", path, cp);
4171         }
4172         return err;
4173 }
4174
4175 static int check_path(const char *path)
4176 {
4177         char *cp, errmsg[STRERR_BUFSIZE];
4178         struct statfs st_fs;
4179         char *dname, *dir;
4180         int err = 0;
4181
4182         if (path == NULL)
4183                 return -EINVAL;
4184
4185         dname = strdup(path);
4186         if (dname == NULL)
4187                 return -ENOMEM;
4188
4189         dir = dirname(dname);
4190         if (statfs(dir, &st_fs)) {
4191                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
4192                 pr_warn("failed to statfs %s: %s\n", dir, cp);
4193                 err = -errno;
4194         }
4195         free(dname);
4196
4197         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
4198                 pr_warn("specified path %s is not on BPF FS\n", path);
4199                 err = -EINVAL;
4200         }
4201
4202         return err;
4203 }
4204
4205 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
4206                               int instance)
4207 {
4208         char *cp, errmsg[STRERR_BUFSIZE];
4209         int err;
4210
4211         err = make_parent_dir(path);
4212         if (err)
4213                 return err;
4214
4215         err = check_path(path);
4216         if (err)
4217                 return err;
4218
4219         if (prog == NULL) {
4220                 pr_warn("invalid program pointer\n");
4221                 return -EINVAL;
4222         }
4223
4224         if (instance < 0 || instance >= prog->instances.nr) {
4225                 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
4226                         instance, prog->section_name, prog->instances.nr);
4227                 return -EINVAL;
4228         }
4229
4230         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
4231                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
4232                 pr_warn("failed to pin program: %s\n", cp);
4233                 return -errno;
4234         }
4235         pr_debug("pinned program '%s'\n", path);
4236
4237         return 0;
4238 }
4239
4240 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
4241                                 int instance)
4242 {
4243         int err;
4244
4245         err = check_path(path);
4246         if (err)
4247                 return err;
4248
4249         if (prog == NULL) {
4250                 pr_warn("invalid program pointer\n");
4251                 return -EINVAL;
4252         }
4253
4254         if (instance < 0 || instance >= prog->instances.nr) {
4255                 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
4256                         instance, prog->section_name, prog->instances.nr);
4257                 return -EINVAL;
4258         }
4259
4260         err = unlink(path);
4261         if (err != 0)
4262                 return -errno;
4263         pr_debug("unpinned program '%s'\n", path);
4264
4265         return 0;
4266 }
4267
4268 int bpf_program__pin(struct bpf_program *prog, const char *path)
4269 {
4270         int i, err;
4271
4272         err = make_parent_dir(path);
4273         if (err)
4274                 return err;
4275
4276         err = check_path(path);
4277         if (err)
4278                 return err;
4279
4280         if (prog == NULL) {
4281                 pr_warn("invalid program pointer\n");
4282                 return -EINVAL;
4283         }
4284
4285         if (prog->instances.nr <= 0) {
4286                 pr_warn("no instances of prog %s to pin\n",
4287                            prog->section_name);
4288                 return -EINVAL;
4289         }
4290
4291         if (prog->instances.nr == 1) {
4292                 /* don't create subdirs when pinning single instance */
4293                 return bpf_program__pin_instance(prog, path, 0);
4294         }
4295
4296         for (i = 0; i < prog->instances.nr; i++) {
4297                 char buf[PATH_MAX];
4298                 int len;
4299
4300                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
4301                 if (len < 0) {
4302                         err = -EINVAL;
4303                         goto err_unpin;
4304                 } else if (len >= PATH_MAX) {
4305                         err = -ENAMETOOLONG;
4306                         goto err_unpin;
4307                 }
4308
4309                 err = bpf_program__pin_instance(prog, buf, i);
4310                 if (err)
4311                         goto err_unpin;
4312         }
4313
4314         return 0;
4315
4316 err_unpin:
4317         for (i = i - 1; i >= 0; i--) {
4318                 char buf[PATH_MAX];
4319                 int len;
4320
4321                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
4322                 if (len < 0)
4323                         continue;
4324                 else if (len >= PATH_MAX)
4325                         continue;
4326
4327                 bpf_program__unpin_instance(prog, buf, i);
4328         }
4329
4330         rmdir(path);
4331
4332         return err;
4333 }
4334
4335 int bpf_program__unpin(struct bpf_program *prog, const char *path)
4336 {
4337         int i, err;
4338
4339         err = check_path(path);
4340         if (err)
4341                 return err;
4342
4343         if (prog == NULL) {
4344                 pr_warn("invalid program pointer\n");
4345                 return -EINVAL;
4346         }
4347
4348         if (prog->instances.nr <= 0) {
4349                 pr_warn("no instances of prog %s to pin\n",
4350                            prog->section_name);
4351                 return -EINVAL;
4352         }
4353
4354         if (prog->instances.nr == 1) {
4355                 /* don't create subdirs when pinning single instance */
4356                 return bpf_program__unpin_instance(prog, path, 0);
4357         }
4358
4359         for (i = 0; i < prog->instances.nr; i++) {
4360                 char buf[PATH_MAX];
4361                 int len;
4362
4363                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
4364                 if (len < 0)
4365                         return -EINVAL;
4366                 else if (len >= PATH_MAX)
4367                         return -ENAMETOOLONG;
4368
4369                 err = bpf_program__unpin_instance(prog, buf, i);
4370                 if (err)
4371                         return err;
4372         }
4373
4374         err = rmdir(path);
4375         if (err)
4376                 return -errno;
4377
4378         return 0;
4379 }
4380
4381 int bpf_map__pin(struct bpf_map *map, const char *path)
4382 {
4383         char *cp, errmsg[STRERR_BUFSIZE];
4384         int err;
4385
4386         if (map == NULL) {
4387                 pr_warn("invalid map pointer\n");
4388                 return -EINVAL;
4389         }
4390
4391         if (map->pin_path) {
4392                 if (path && strcmp(path, map->pin_path)) {
4393                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
4394                                 bpf_map__name(map), map->pin_path, path);
4395                         return -EINVAL;
4396                 } else if (map->pinned) {
4397                         pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
4398                                  bpf_map__name(map), map->pin_path);
4399                         return 0;
4400                 }
4401         } else {
4402                 if (!path) {
4403                         pr_warn("missing a path to pin map '%s' at\n",
4404                                 bpf_map__name(map));
4405                         return -EINVAL;
4406                 } else if (map->pinned) {
4407                         pr_warn("map '%s' already pinned\n", bpf_map__name(map));
4408                         return -EEXIST;
4409                 }
4410
4411                 map->pin_path = strdup(path);
4412                 if (!map->pin_path) {
4413                         err = -errno;
4414                         goto out_err;
4415                 }
4416         }
4417
4418         err = make_parent_dir(map->pin_path);
4419         if (err)
4420                 return err;
4421
4422         err = check_path(map->pin_path);
4423         if (err)
4424                 return err;
4425
4426         if (bpf_obj_pin(map->fd, map->pin_path)) {
4427                 err = -errno;
4428                 goto out_err;
4429         }
4430
4431         map->pinned = true;
4432         pr_debug("pinned map '%s'\n", map->pin_path);
4433
4434         return 0;
4435
4436 out_err:
4437         cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4438         pr_warn("failed to pin map: %s\n", cp);
4439         return err;
4440 }
4441
4442 int bpf_map__unpin(struct bpf_map *map, const char *path)
4443 {
4444         int err;
4445
4446         if (map == NULL) {
4447                 pr_warn("invalid map pointer\n");
4448                 return -EINVAL;
4449         }
4450
4451         if (map->pin_path) {
4452                 if (path && strcmp(path, map->pin_path)) {
4453                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
4454                                 bpf_map__name(map), map->pin_path, path);
4455                         return -EINVAL;
4456                 }
4457                 path = map->pin_path;
4458         } else if (!path) {
4459                 pr_warn("no path to unpin map '%s' from\n",
4460                         bpf_map__name(map));
4461                 return -EINVAL;
4462         }
4463
4464         err = check_path(path);
4465         if (err)
4466                 return err;
4467
4468         err = unlink(path);
4469         if (err != 0)
4470                 return -errno;
4471
4472         map->pinned = false;
4473         pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
4474
4475         return 0;
4476 }
4477
4478 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
4479 {
4480         char *new = NULL;
4481
4482         if (path) {
4483                 new = strdup(path);
4484                 if (!new)
4485                         return -errno;
4486         }
4487
4488         free(map->pin_path);
4489         map->pin_path = new;
4490         return 0;
4491 }
4492
4493 const char *bpf_map__get_pin_path(const struct bpf_map *map)
4494 {
4495         return map->pin_path;
4496 }
4497
4498 bool bpf_map__is_pinned(const struct bpf_map *map)
4499 {
4500         return map->pinned;
4501 }
4502
4503 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
4504 {
4505         struct bpf_map *map;
4506         int err;
4507
4508         if (!obj)
4509                 return -ENOENT;
4510
4511         if (!obj->loaded) {
4512                 pr_warn("object not yet loaded; load it first\n");
4513                 return -ENOENT;
4514         }
4515
4516         bpf_object__for_each_map(map, obj) {
4517                 char *pin_path = NULL;
4518                 char buf[PATH_MAX];
4519
4520                 if (path) {
4521                         int len;
4522
4523                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
4524                                        bpf_map__name(map));
4525                         if (len < 0) {
4526                                 err = -EINVAL;
4527                                 goto err_unpin_maps;
4528                         } else if (len >= PATH_MAX) {
4529                                 err = -ENAMETOOLONG;
4530                                 goto err_unpin_maps;
4531                         }
4532                         pin_path = buf;
4533                 } else if (!map->pin_path) {
4534                         continue;
4535                 }
4536
4537                 err = bpf_map__pin(map, pin_path);
4538                 if (err)
4539                         goto err_unpin_maps;
4540         }
4541
4542         return 0;
4543
4544 err_unpin_maps:
4545         while ((map = bpf_map__prev(map, obj))) {
4546                 if (!map->pin_path)
4547                         continue;
4548
4549                 bpf_map__unpin(map, NULL);
4550         }
4551
4552         return err;
4553 }
4554
4555 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
4556 {
4557         struct bpf_map *map;
4558         int err;
4559
4560         if (!obj)
4561                 return -ENOENT;
4562
4563         bpf_object__for_each_map(map, obj) {
4564                 char *pin_path = NULL;
4565                 char buf[PATH_MAX];
4566
4567                 if (path) {
4568                         int len;
4569
4570                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
4571                                        bpf_map__name(map));
4572                         if (len < 0)
4573                                 return -EINVAL;
4574                         else if (len >= PATH_MAX)
4575                                 return -ENAMETOOLONG;
4576                         pin_path = buf;
4577                 } else if (!map->pin_path) {
4578                         continue;
4579                 }
4580
4581                 err = bpf_map__unpin(map, pin_path);
4582                 if (err)
4583                         return err;
4584         }
4585
4586         return 0;
4587 }
4588
4589 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
4590 {
4591         struct bpf_program *prog;
4592         int err;
4593
4594         if (!obj)
4595                 return -ENOENT;
4596
4597         if (!obj->loaded) {
4598                 pr_warn("object not yet loaded; load it first\n");
4599                 return -ENOENT;
4600         }
4601
4602         bpf_object__for_each_program(prog, obj) {
4603                 char buf[PATH_MAX];
4604                 int len;
4605
4606                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
4607                                prog->pin_name);
4608                 if (len < 0) {
4609                         err = -EINVAL;
4610                         goto err_unpin_programs;
4611                 } else if (len >= PATH_MAX) {
4612                         err = -ENAMETOOLONG;
4613                         goto err_unpin_programs;
4614                 }
4615
4616                 err = bpf_program__pin(prog, buf);
4617                 if (err)
4618                         goto err_unpin_programs;
4619         }
4620
4621         return 0;
4622
4623 err_unpin_programs:
4624         while ((prog = bpf_program__prev(prog, obj))) {
4625                 char buf[PATH_MAX];
4626                 int len;
4627
4628                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
4629                                prog->pin_name);
4630                 if (len < 0)
4631                         continue;
4632                 else if (len >= PATH_MAX)
4633                         continue;
4634
4635                 bpf_program__unpin(prog, buf);
4636         }
4637
4638         return err;
4639 }
4640
4641 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
4642 {
4643         struct bpf_program *prog;
4644         int err;
4645
4646         if (!obj)
4647                 return -ENOENT;
4648
4649         bpf_object__for_each_program(prog, obj) {
4650                 char buf[PATH_MAX];
4651                 int len;
4652
4653                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
4654                                prog->pin_name);
4655                 if (len < 0)
4656                         return -EINVAL;
4657                 else if (len >= PATH_MAX)
4658                         return -ENAMETOOLONG;
4659
4660                 err = bpf_program__unpin(prog, buf);
4661                 if (err)
4662                         return err;
4663         }
4664
4665         return 0;
4666 }
4667
4668 int bpf_object__pin(struct bpf_object *obj, const char *path)
4669 {
4670         int err;
4671
4672         err = bpf_object__pin_maps(obj, path);
4673         if (err)
4674                 return err;
4675
4676         err = bpf_object__pin_programs(obj, path);
4677         if (err) {
4678                 bpf_object__unpin_maps(obj, path);
4679                 return err;
4680         }
4681
4682         return 0;
4683 }
4684
4685 void bpf_object__close(struct bpf_object *obj)
4686 {
4687         size_t i;
4688
4689         if (!obj)
4690                 return;
4691
4692         if (obj->clear_priv)
4693                 obj->clear_priv(obj, obj->priv);
4694
4695         bpf_object__elf_finish(obj);
4696         bpf_object__unload(obj);
4697         btf__free(obj->btf);
4698         btf_ext__free(obj->btf_ext);
4699
4700         for (i = 0; i < obj->nr_maps; i++) {
4701                 struct bpf_map *map = &obj->maps[i];
4702
4703                 if (map->clear_priv)
4704                         map->clear_priv(map, map->priv);
4705                 map->priv = NULL;
4706                 map->clear_priv = NULL;
4707
4708                 if (map->mmaped) {
4709                         munmap(map->mmaped, bpf_map_mmap_sz(map));
4710                         map->mmaped = NULL;
4711                 }
4712
4713                 zfree(&map->name);
4714                 zfree(&map->pin_path);
4715         }
4716
4717         zfree(&obj->maps);
4718         obj->nr_maps = 0;
4719
4720         if (obj->programs && obj->nr_programs) {
4721                 for (i = 0; i < obj->nr_programs; i++)
4722                         bpf_program__exit(&obj->programs[i]);
4723         }
4724         zfree(&obj->programs);
4725
4726         list_del(&obj->list);
4727         free(obj);
4728 }
4729
4730 struct bpf_object *
4731 bpf_object__next(struct bpf_object *prev)
4732 {
4733         struct bpf_object *next;
4734
4735         if (!prev)
4736                 next = list_first_entry(&bpf_objects_list,
4737                                         struct bpf_object,
4738                                         list);
4739         else
4740                 next = list_next_entry(prev, list);
4741
4742         /* Empty list is noticed here so don't need checking on entry. */
4743         if (&next->list == &bpf_objects_list)
4744                 return NULL;
4745
4746         return next;
4747 }
4748
4749 const char *bpf_object__name(const struct bpf_object *obj)
4750 {
4751         return obj ? obj->name : ERR_PTR(-EINVAL);
4752 }
4753
4754 unsigned int bpf_object__kversion(const struct bpf_object *obj)
4755 {
4756         return obj ? obj->kern_version : 0;
4757 }
4758
4759 struct btf *bpf_object__btf(const struct bpf_object *obj)
4760 {
4761         return obj ? obj->btf : NULL;
4762 }
4763
4764 int bpf_object__btf_fd(const struct bpf_object *obj)
4765 {
4766         return obj->btf ? btf__fd(obj->btf) : -1;
4767 }
4768
4769 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
4770                          bpf_object_clear_priv_t clear_priv)
4771 {
4772         if (obj->priv && obj->clear_priv)
4773                 obj->clear_priv(obj, obj->priv);
4774
4775         obj->priv = priv;
4776         obj->clear_priv = clear_priv;
4777         return 0;
4778 }
4779
4780 void *bpf_object__priv(const struct bpf_object *obj)
4781 {
4782         return obj ? obj->priv : ERR_PTR(-EINVAL);
4783 }
4784
4785 static struct bpf_program *
4786 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
4787                     bool forward)
4788 {
4789         size_t nr_programs = obj->nr_programs;
4790         ssize_t idx;
4791
4792         if (!nr_programs)
4793                 return NULL;
4794
4795         if (!p)
4796                 /* Iter from the beginning */
4797                 return forward ? &obj->programs[0] :
4798                         &obj->programs[nr_programs - 1];
4799
4800         if (p->obj != obj) {
4801                 pr_warn("error: program handler doesn't match object\n");
4802                 return NULL;
4803         }
4804
4805         idx = (p - obj->programs) + (forward ? 1 : -1);
4806         if (idx >= obj->nr_programs || idx < 0)
4807                 return NULL;
4808         return &obj->programs[idx];
4809 }
4810
4811 struct bpf_program *
4812 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
4813 {
4814         struct bpf_program *prog = prev;
4815
4816         do {
4817                 prog = __bpf_program__iter(prog, obj, true);
4818         } while (prog && bpf_program__is_function_storage(prog, obj));
4819
4820         return prog;
4821 }
4822
4823 struct bpf_program *
4824 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
4825 {
4826         struct bpf_program *prog = next;
4827
4828         do {
4829                 prog = __bpf_program__iter(prog, obj, false);
4830         } while (prog && bpf_program__is_function_storage(prog, obj));
4831
4832         return prog;
4833 }
4834
4835 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
4836                           bpf_program_clear_priv_t clear_priv)
4837 {
4838         if (prog->priv && prog->clear_priv)
4839                 prog->clear_priv(prog, prog->priv);
4840
4841         prog->priv = priv;
4842         prog->clear_priv = clear_priv;
4843         return 0;
4844 }
4845
4846 void *bpf_program__priv(const struct bpf_program *prog)
4847 {
4848         return prog ? prog->priv : ERR_PTR(-EINVAL);
4849 }
4850
4851 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
4852 {
4853         prog->prog_ifindex = ifindex;
4854 }
4855
4856 const char *bpf_program__name(const struct bpf_program *prog)
4857 {
4858         return prog->name;
4859 }
4860
4861 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
4862 {
4863         const char *title;
4864
4865         title = prog->section_name;
4866         if (needs_copy) {
4867                 title = strdup(title);
4868                 if (!title) {
4869                         pr_warn("failed to strdup program title\n");
4870                         return ERR_PTR(-ENOMEM);
4871                 }
4872         }
4873
4874         return title;
4875 }
4876
4877 int bpf_program__fd(const struct bpf_program *prog)
4878 {
4879         return bpf_program__nth_fd(prog, 0);
4880 }
4881
4882 size_t bpf_program__size(const struct bpf_program *prog)
4883 {
4884         return prog->insns_cnt * sizeof(struct bpf_insn);
4885 }
4886
4887 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
4888                           bpf_program_prep_t prep)
4889 {
4890         int *instances_fds;
4891
4892         if (nr_instances <= 0 || !prep)
4893                 return -EINVAL;
4894
4895         if (prog->instances.nr > 0 || prog->instances.fds) {
4896                 pr_warn("Can't set pre-processor after loading\n");
4897                 return -EINVAL;
4898         }
4899
4900         instances_fds = malloc(sizeof(int) * nr_instances);
4901         if (!instances_fds) {
4902                 pr_warn("alloc memory failed for fds\n");
4903                 return -ENOMEM;
4904         }
4905
4906         /* fill all fd with -1 */
4907         memset(instances_fds, -1, sizeof(int) * nr_instances);
4908
4909         prog->instances.nr = nr_instances;
4910         prog->instances.fds = instances_fds;
4911         prog->preprocessor = prep;
4912         return 0;
4913 }
4914
4915 int bpf_program__nth_fd(const struct bpf_program *prog, int n)
4916 {
4917         int fd;
4918
4919         if (!prog)
4920                 return -EINVAL;
4921
4922         if (n >= prog->instances.nr || n < 0) {
4923                 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
4924                         n, prog->section_name, prog->instances.nr);
4925                 return -EINVAL;
4926         }
4927
4928         fd = prog->instances.fds[n];
4929         if (fd < 0) {
4930                 pr_warn("%dth instance of program '%s' is invalid\n",
4931                         n, prog->section_name);
4932                 return -ENOENT;
4933         }
4934
4935         return fd;
4936 }
4937
4938 enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
4939 {
4940         return prog->type;
4941 }
4942
4943 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
4944 {
4945         prog->type = type;
4946 }
4947
4948 static bool bpf_program__is_type(const struct bpf_program *prog,
4949                                  enum bpf_prog_type type)
4950 {
4951         return prog ? (prog->type == type) : false;
4952 }
4953
4954 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                           \
4955 int bpf_program__set_##NAME(struct bpf_program *prog)           \
4956 {                                                               \
4957         if (!prog)                                              \
4958                 return -EINVAL;                                 \
4959         bpf_program__set_type(prog, TYPE);                      \
4960         return 0;                                               \
4961 }                                                               \
4962                                                                 \
4963 bool bpf_program__is_##NAME(const struct bpf_program *prog)     \
4964 {                                                               \
4965         return bpf_program__is_type(prog, TYPE);                \
4966 }                                                               \
4967
4968 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
4969 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
4970 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
4971 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
4972 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
4973 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
4974 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
4975 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
4976 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
4977
4978 enum bpf_attach_type
4979 bpf_program__get_expected_attach_type(struct bpf_program *prog)
4980 {
4981         return prog->expected_attach_type;
4982 }
4983
4984 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
4985                                            enum bpf_attach_type type)
4986 {
4987         prog->expected_attach_type = type;
4988 }
4989
4990 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, btf, atype) \
4991         { string, sizeof(string) - 1, ptype, eatype, is_attachable, btf, atype }
4992
4993 /* Programs that can NOT be attached. */
4994 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
4995
4996 /* Programs that can be attached. */
4997 #define BPF_APROG_SEC(string, ptype, atype) \
4998         BPF_PROG_SEC_IMPL(string, ptype, 0, 1, 0, atype)
4999
5000 /* Programs that must specify expected attach type at load time. */
5001 #define BPF_EAPROG_SEC(string, ptype, eatype) \
5002         BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, 0, eatype)
5003
5004 /* Programs that use BTF to identify attach point */
5005 #define BPF_PROG_BTF(string, ptype, eatype) \
5006         BPF_PROG_SEC_IMPL(string, ptype, eatype, 0, 1, 0)
5007
5008 /* Programs that can be attached but attach type can't be identified by section
5009  * name. Kept for backward compatibility.
5010  */
5011 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
5012
5013 #define SEC_DEF(sec_pfx, ptype, ...) {                                      \
5014         .sec = sec_pfx,                                                     \
5015         .len = sizeof(sec_pfx) - 1,                                         \
5016         .prog_type = BPF_PROG_TYPE_##ptype,                                 \
5017         __VA_ARGS__                                                         \
5018 }
5019
5020 struct bpf_sec_def;
5021
5022 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
5023                                         struct bpf_program *prog);
5024
5025 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
5026                                       struct bpf_program *prog);
5027 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
5028                                   struct bpf_program *prog);
5029 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
5030                                       struct bpf_program *prog);
5031 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
5032                                      struct bpf_program *prog);
5033
5034 struct bpf_sec_def {
5035         const char *sec;
5036         size_t len;
5037         enum bpf_prog_type prog_type;
5038         enum bpf_attach_type expected_attach_type;
5039         bool is_attachable;
5040         bool is_attach_btf;
5041         enum bpf_attach_type attach_type;
5042         attach_fn_t attach_fn;
5043 };
5044
5045 static const struct bpf_sec_def section_defs[] = {
5046         BPF_PROG_SEC("socket",                  BPF_PROG_TYPE_SOCKET_FILTER),
5047         BPF_PROG_SEC("sk_reuseport",            BPF_PROG_TYPE_SK_REUSEPORT),
5048         SEC_DEF("kprobe/", KPROBE,
5049                 .attach_fn = attach_kprobe),
5050         BPF_PROG_SEC("uprobe/",                 BPF_PROG_TYPE_KPROBE),
5051         SEC_DEF("kretprobe/", KPROBE,
5052                 .attach_fn = attach_kprobe),
5053         BPF_PROG_SEC("uretprobe/",              BPF_PROG_TYPE_KPROBE),
5054         BPF_PROG_SEC("classifier",              BPF_PROG_TYPE_SCHED_CLS),
5055         BPF_PROG_SEC("action",                  BPF_PROG_TYPE_SCHED_ACT),
5056         SEC_DEF("tracepoint/", TRACEPOINT,
5057                 .attach_fn = attach_tp),
5058         SEC_DEF("tp/", TRACEPOINT,
5059                 .attach_fn = attach_tp),
5060         SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
5061                 .attach_fn = attach_raw_tp),
5062         SEC_DEF("raw_tp/", RAW_TRACEPOINT,
5063                 .attach_fn = attach_raw_tp),
5064         SEC_DEF("tp_btf/", TRACING,
5065                 .expected_attach_type = BPF_TRACE_RAW_TP,
5066                 .is_attach_btf = true,
5067                 .attach_fn = attach_trace),
5068         SEC_DEF("fentry/", TRACING,
5069                 .expected_attach_type = BPF_TRACE_FENTRY,
5070                 .is_attach_btf = true,
5071                 .attach_fn = attach_trace),
5072         SEC_DEF("fexit/", TRACING,
5073                 .expected_attach_type = BPF_TRACE_FEXIT,
5074                 .is_attach_btf = true,
5075                 .attach_fn = attach_trace),
5076         BPF_PROG_SEC("xdp",                     BPF_PROG_TYPE_XDP),
5077         BPF_PROG_SEC("perf_event",              BPF_PROG_TYPE_PERF_EVENT),
5078         BPF_PROG_SEC("lwt_in",                  BPF_PROG_TYPE_LWT_IN),
5079         BPF_PROG_SEC("lwt_out",                 BPF_PROG_TYPE_LWT_OUT),
5080         BPF_PROG_SEC("lwt_xmit",                BPF_PROG_TYPE_LWT_XMIT),
5081         BPF_PROG_SEC("lwt_seg6local",           BPF_PROG_TYPE_LWT_SEG6LOCAL),
5082         BPF_APROG_SEC("cgroup_skb/ingress",     BPF_PROG_TYPE_CGROUP_SKB,
5083                                                 BPF_CGROUP_INET_INGRESS),
5084         BPF_APROG_SEC("cgroup_skb/egress",      BPF_PROG_TYPE_CGROUP_SKB,
5085                                                 BPF_CGROUP_INET_EGRESS),
5086         BPF_APROG_COMPAT("cgroup/skb",          BPF_PROG_TYPE_CGROUP_SKB),
5087         BPF_APROG_SEC("cgroup/sock",            BPF_PROG_TYPE_CGROUP_SOCK,
5088                                                 BPF_CGROUP_INET_SOCK_CREATE),
5089         BPF_EAPROG_SEC("cgroup/post_bind4",     BPF_PROG_TYPE_CGROUP_SOCK,
5090                                                 BPF_CGROUP_INET4_POST_BIND),
5091         BPF_EAPROG_SEC("cgroup/post_bind6",     BPF_PROG_TYPE_CGROUP_SOCK,
5092                                                 BPF_CGROUP_INET6_POST_BIND),
5093         BPF_APROG_SEC("cgroup/dev",             BPF_PROG_TYPE_CGROUP_DEVICE,
5094                                                 BPF_CGROUP_DEVICE),
5095         BPF_APROG_SEC("sockops",                BPF_PROG_TYPE_SOCK_OPS,
5096                                                 BPF_CGROUP_SOCK_OPS),
5097         BPF_APROG_SEC("sk_skb/stream_parser",   BPF_PROG_TYPE_SK_SKB,
5098                                                 BPF_SK_SKB_STREAM_PARSER),
5099         BPF_APROG_SEC("sk_skb/stream_verdict",  BPF_PROG_TYPE_SK_SKB,
5100                                                 BPF_SK_SKB_STREAM_VERDICT),
5101         BPF_APROG_COMPAT("sk_skb",              BPF_PROG_TYPE_SK_SKB),
5102         BPF_APROG_SEC("sk_msg",                 BPF_PROG_TYPE_SK_MSG,
5103                                                 BPF_SK_MSG_VERDICT),
5104         BPF_APROG_SEC("lirc_mode2",             BPF_PROG_TYPE_LIRC_MODE2,
5105                                                 BPF_LIRC_MODE2),
5106         BPF_APROG_SEC("flow_dissector",         BPF_PROG_TYPE_FLOW_DISSECTOR,
5107                                                 BPF_FLOW_DISSECTOR),
5108         BPF_EAPROG_SEC("cgroup/bind4",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
5109                                                 BPF_CGROUP_INET4_BIND),
5110         BPF_EAPROG_SEC("cgroup/bind6",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
5111                                                 BPF_CGROUP_INET6_BIND),
5112         BPF_EAPROG_SEC("cgroup/connect4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
5113                                                 BPF_CGROUP_INET4_CONNECT),
5114         BPF_EAPROG_SEC("cgroup/connect6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
5115                                                 BPF_CGROUP_INET6_CONNECT),
5116         BPF_EAPROG_SEC("cgroup/sendmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
5117                                                 BPF_CGROUP_UDP4_SENDMSG),
5118         BPF_EAPROG_SEC("cgroup/sendmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
5119                                                 BPF_CGROUP_UDP6_SENDMSG),
5120         BPF_EAPROG_SEC("cgroup/recvmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
5121                                                 BPF_CGROUP_UDP4_RECVMSG),
5122         BPF_EAPROG_SEC("cgroup/recvmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
5123                                                 BPF_CGROUP_UDP6_RECVMSG),
5124         BPF_EAPROG_SEC("cgroup/sysctl",         BPF_PROG_TYPE_CGROUP_SYSCTL,
5125                                                 BPF_CGROUP_SYSCTL),
5126         BPF_EAPROG_SEC("cgroup/getsockopt",     BPF_PROG_TYPE_CGROUP_SOCKOPT,
5127                                                 BPF_CGROUP_GETSOCKOPT),
5128         BPF_EAPROG_SEC("cgroup/setsockopt",     BPF_PROG_TYPE_CGROUP_SOCKOPT,
5129                                                 BPF_CGROUP_SETSOCKOPT),
5130 };
5131
5132 #undef BPF_PROG_SEC_IMPL
5133 #undef BPF_PROG_SEC
5134 #undef BPF_APROG_SEC
5135 #undef BPF_EAPROG_SEC
5136 #undef BPF_APROG_COMPAT
5137 #undef SEC_DEF
5138
5139 #define MAX_TYPE_NAME_SIZE 32
5140
5141 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
5142 {
5143         int i, n = ARRAY_SIZE(section_defs);
5144
5145         for (i = 0; i < n; i++) {
5146                 if (strncmp(sec_name,
5147                             section_defs[i].sec, section_defs[i].len))
5148                         continue;
5149                 return &section_defs[i];
5150         }
5151         return NULL;
5152 }
5153
5154 static char *libbpf_get_type_names(bool attach_type)
5155 {
5156         int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
5157         char *buf;
5158
5159         buf = malloc(len);
5160         if (!buf)
5161                 return NULL;
5162
5163         buf[0] = '\0';
5164         /* Forge string buf with all available names */
5165         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
5166                 if (attach_type && !section_defs[i].is_attachable)
5167                         continue;
5168
5169                 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
5170                         free(buf);
5171                         return NULL;
5172                 }
5173                 strcat(buf, " ");
5174                 strcat(buf, section_defs[i].sec);
5175         }
5176
5177         return buf;
5178 }
5179
5180 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
5181                              enum bpf_attach_type *expected_attach_type)
5182 {
5183         const struct bpf_sec_def *sec_def;
5184         char *type_names;
5185
5186         if (!name)
5187                 return -EINVAL;
5188
5189         sec_def = find_sec_def(name);
5190         if (sec_def) {
5191                 *prog_type = sec_def->prog_type;
5192                 *expected_attach_type = sec_def->expected_attach_type;
5193                 return 0;
5194         }
5195
5196         pr_warn("failed to guess program type from ELF section '%s'\n", name);
5197         type_names = libbpf_get_type_names(false);
5198         if (type_names != NULL) {
5199                 pr_debug("supported section(type) names are:%s\n", type_names);
5200                 free(type_names);
5201         }
5202
5203         return -ESRCH;
5204 }
5205
5206 #define BTF_PREFIX "btf_trace_"
5207 int libbpf_find_vmlinux_btf_id(const char *name,
5208                                enum bpf_attach_type attach_type)
5209 {
5210         struct btf *btf = bpf_core_find_kernel_btf();
5211         char raw_tp_btf[128] = BTF_PREFIX;
5212         char *dst = raw_tp_btf + sizeof(BTF_PREFIX) - 1;
5213         const char *btf_name;
5214         int err = -EINVAL;
5215         __u32 kind;
5216
5217         if (IS_ERR(btf)) {
5218                 pr_warn("vmlinux BTF is not found\n");
5219                 return -EINVAL;
5220         }
5221
5222         if (attach_type == BPF_TRACE_RAW_TP) {
5223                 /* prepend "btf_trace_" prefix per kernel convention */
5224                 strncat(dst, name, sizeof(raw_tp_btf) - sizeof(BTF_PREFIX));
5225                 btf_name = raw_tp_btf;
5226                 kind = BTF_KIND_TYPEDEF;
5227         } else {
5228                 btf_name = name;
5229                 kind = BTF_KIND_FUNC;
5230         }
5231         err = btf__find_by_name_kind(btf, btf_name, kind);
5232         btf__free(btf);
5233         return err;
5234 }
5235
5236 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
5237 {
5238         struct bpf_prog_info_linear *info_linear;
5239         struct bpf_prog_info *info;
5240         struct btf *btf = NULL;
5241         int err = -EINVAL;
5242
5243         info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
5244         if (IS_ERR_OR_NULL(info_linear)) {
5245                 pr_warn("failed get_prog_info_linear for FD %d\n",
5246                         attach_prog_fd);
5247                 return -EINVAL;
5248         }
5249         info = &info_linear->info;
5250         if (!info->btf_id) {
5251                 pr_warn("The target program doesn't have BTF\n");
5252                 goto out;
5253         }
5254         if (btf__get_from_id(info->btf_id, &btf)) {
5255                 pr_warn("Failed to get BTF of the program\n");
5256                 goto out;
5257         }
5258         err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
5259         btf__free(btf);
5260         if (err <= 0) {
5261                 pr_warn("%s is not found in prog's BTF\n", name);
5262                 goto out;
5263         }
5264 out:
5265         free(info_linear);
5266         return err;
5267 }
5268
5269 static int libbpf_find_attach_btf_id(const char *name,
5270                                      enum bpf_attach_type attach_type,
5271                                      __u32 attach_prog_fd)
5272 {
5273         int i, err;
5274
5275         if (!name)
5276                 return -EINVAL;
5277
5278         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
5279                 if (!section_defs[i].is_attach_btf)
5280                         continue;
5281                 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
5282                         continue;
5283                 if (attach_prog_fd)
5284                         err = libbpf_find_prog_btf_id(name + section_defs[i].len,
5285                                                       attach_prog_fd);
5286                 else
5287                         err = libbpf_find_vmlinux_btf_id(name + section_defs[i].len,
5288                                                          attach_type);
5289                 if (err <= 0)
5290                         pr_warn("%s is not found in vmlinux BTF\n", name);
5291                 return err;
5292         }
5293         pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name);
5294         return -ESRCH;
5295 }
5296
5297 int libbpf_attach_type_by_name(const char *name,
5298                                enum bpf_attach_type *attach_type)
5299 {
5300         char *type_names;
5301         int i;
5302
5303         if (!name)
5304                 return -EINVAL;
5305
5306         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
5307                 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
5308                         continue;
5309                 if (!section_defs[i].is_attachable)
5310                         return -EINVAL;
5311                 *attach_type = section_defs[i].attach_type;
5312                 return 0;
5313         }
5314         pr_warn("failed to guess attach type based on ELF section name '%s'\n", name);
5315         type_names = libbpf_get_type_names(true);
5316         if (type_names != NULL) {
5317                 pr_info("attachable section(type) names are:%s\n", type_names);
5318                 free(type_names);
5319         }
5320
5321         return -EINVAL;
5322 }
5323
5324 int bpf_map__fd(const struct bpf_map *map)
5325 {
5326         return map ? map->fd : -EINVAL;
5327 }
5328
5329 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
5330 {
5331         return map ? &map->def : ERR_PTR(-EINVAL);
5332 }
5333
5334 const char *bpf_map__name(const struct bpf_map *map)
5335 {
5336         return map ? map->name : NULL;
5337 }
5338
5339 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
5340 {
5341         return map ? map->btf_key_type_id : 0;
5342 }
5343
5344 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
5345 {
5346         return map ? map->btf_value_type_id : 0;
5347 }
5348
5349 int bpf_map__set_priv(struct bpf_map *map, void *priv,
5350                      bpf_map_clear_priv_t clear_priv)
5351 {
5352         if (!map)
5353                 return -EINVAL;
5354
5355         if (map->priv) {
5356                 if (map->clear_priv)
5357                         map->clear_priv(map, map->priv);
5358         }
5359
5360         map->priv = priv;
5361         map->clear_priv = clear_priv;
5362         return 0;
5363 }
5364
5365 void *bpf_map__priv(const struct bpf_map *map)
5366 {
5367         return map ? map->priv : ERR_PTR(-EINVAL);
5368 }
5369
5370 bool bpf_map__is_offload_neutral(const struct bpf_map *map)
5371 {
5372         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
5373 }
5374
5375 bool bpf_map__is_internal(const struct bpf_map *map)
5376 {
5377         return map->libbpf_type != LIBBPF_MAP_UNSPEC;
5378 }
5379
5380 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
5381 {
5382         map->map_ifindex = ifindex;
5383 }
5384
5385 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
5386 {
5387         if (!bpf_map_type__is_map_in_map(map->def.type)) {
5388                 pr_warn("error: unsupported map type\n");
5389                 return -EINVAL;
5390         }
5391         if (map->inner_map_fd != -1) {
5392                 pr_warn("error: inner_map_fd already specified\n");
5393                 return -EINVAL;
5394         }
5395         map->inner_map_fd = fd;
5396         return 0;
5397 }
5398
5399 static struct bpf_map *
5400 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
5401 {
5402         ssize_t idx;
5403         struct bpf_map *s, *e;
5404
5405         if (!obj || !obj->maps)
5406                 return NULL;
5407
5408         s = obj->maps;
5409         e = obj->maps + obj->nr_maps;
5410
5411         if ((m < s) || (m >= e)) {
5412                 pr_warn("error in %s: map handler doesn't belong to object\n",
5413                          __func__);
5414                 return NULL;
5415         }
5416
5417         idx = (m - obj->maps) + i;
5418         if (idx >= obj->nr_maps || idx < 0)
5419                 return NULL;
5420         return &obj->maps[idx];
5421 }
5422
5423 struct bpf_map *
5424 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
5425 {
5426         if (prev == NULL)
5427                 return obj->maps;
5428
5429         return __bpf_map__iter(prev, obj, 1);
5430 }
5431
5432 struct bpf_map *
5433 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
5434 {
5435         if (next == NULL) {
5436                 if (!obj->nr_maps)
5437                         return NULL;
5438                 return obj->maps + obj->nr_maps - 1;
5439         }
5440
5441         return __bpf_map__iter(next, obj, -1);
5442 }
5443
5444 struct bpf_map *
5445 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
5446 {
5447         struct bpf_map *pos;
5448
5449         bpf_object__for_each_map(pos, obj) {
5450                 if (pos->name && !strcmp(pos->name, name))
5451                         return pos;
5452         }
5453         return NULL;
5454 }
5455
5456 int
5457 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
5458 {
5459         return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
5460 }
5461
5462 struct bpf_map *
5463 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
5464 {
5465         return ERR_PTR(-ENOTSUP);
5466 }
5467
5468 long libbpf_get_error(const void *ptr)
5469 {
5470         return PTR_ERR_OR_ZERO(ptr);
5471 }
5472
5473 int bpf_prog_load(const char *file, enum bpf_prog_type type,
5474                   struct bpf_object **pobj, int *prog_fd)
5475 {
5476         struct bpf_prog_load_attr attr;
5477
5478         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
5479         attr.file = file;
5480         attr.prog_type = type;
5481         attr.expected_attach_type = 0;
5482
5483         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
5484 }
5485
5486 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
5487                         struct bpf_object **pobj, int *prog_fd)
5488 {
5489         struct bpf_object_open_attr open_attr = {};
5490         struct bpf_program *prog, *first_prog = NULL;
5491         struct bpf_object *obj;
5492         struct bpf_map *map;
5493         int err;
5494
5495         if (!attr)
5496                 return -EINVAL;
5497         if (!attr->file)
5498                 return -EINVAL;
5499
5500         open_attr.file = attr->file;
5501         open_attr.prog_type = attr->prog_type;
5502
5503         obj = bpf_object__open_xattr(&open_attr);
5504         if (IS_ERR_OR_NULL(obj))
5505                 return -ENOENT;
5506
5507         bpf_object__for_each_program(prog, obj) {
5508                 enum bpf_attach_type attach_type = attr->expected_attach_type;
5509                 /*
5510                  * to preserve backwards compatibility, bpf_prog_load treats
5511                  * attr->prog_type, if specified, as an override to whatever
5512                  * bpf_object__open guessed
5513                  */
5514                 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
5515                         bpf_program__set_type(prog, attr->prog_type);
5516                         bpf_program__set_expected_attach_type(prog,
5517                                                               attach_type);
5518                 }
5519                 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
5520                         /*
5521                          * we haven't guessed from section name and user
5522                          * didn't provide a fallback type, too bad...
5523                          */
5524                         bpf_object__close(obj);
5525                         return -EINVAL;
5526                 }
5527
5528                 prog->prog_ifindex = attr->ifindex;
5529                 prog->log_level = attr->log_level;
5530                 prog->prog_flags = attr->prog_flags;
5531                 if (!first_prog)
5532                         first_prog = prog;
5533         }
5534
5535         bpf_object__for_each_map(map, obj) {
5536                 if (!bpf_map__is_offload_neutral(map))
5537                         map->map_ifindex = attr->ifindex;
5538         }
5539
5540         if (!first_prog) {
5541                 pr_warn("object file doesn't contain bpf program\n");
5542                 bpf_object__close(obj);
5543                 return -ENOENT;
5544         }
5545
5546         err = bpf_object__load(obj);
5547         if (err) {
5548                 bpf_object__close(obj);
5549                 return -EINVAL;
5550         }
5551
5552         *pobj = obj;
5553         *prog_fd = bpf_program__fd(first_prog);
5554         return 0;
5555 }
5556
5557 struct bpf_link {
5558         int (*destroy)(struct bpf_link *link);
5559 };
5560
5561 int bpf_link__destroy(struct bpf_link *link)
5562 {
5563         int err;
5564
5565         if (!link)
5566                 return 0;
5567
5568         err = link->destroy(link);
5569         free(link);
5570
5571         return err;
5572 }
5573
5574 struct bpf_link_fd {
5575         struct bpf_link link; /* has to be at the top of struct */
5576         int fd; /* hook FD */
5577 };
5578
5579 static int bpf_link__destroy_perf_event(struct bpf_link *link)
5580 {
5581         struct bpf_link_fd *l = (void *)link;
5582         int err;
5583
5584         err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0);
5585         if (err)
5586                 err = -errno;
5587
5588         close(l->fd);
5589         return err;
5590 }
5591
5592 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
5593                                                 int pfd)
5594 {
5595         char errmsg[STRERR_BUFSIZE];
5596         struct bpf_link_fd *link;
5597         int prog_fd, err;
5598
5599         if (pfd < 0) {
5600                 pr_warn("program '%s': invalid perf event FD %d\n",
5601                         bpf_program__title(prog, false), pfd);
5602                 return ERR_PTR(-EINVAL);
5603         }
5604         prog_fd = bpf_program__fd(prog);
5605         if (prog_fd < 0) {
5606                 pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n",
5607                         bpf_program__title(prog, false));
5608                 return ERR_PTR(-EINVAL);
5609         }
5610
5611         link = malloc(sizeof(*link));
5612         if (!link)
5613                 return ERR_PTR(-ENOMEM);
5614         link->link.destroy = &bpf_link__destroy_perf_event;
5615         link->fd = pfd;
5616
5617         if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
5618                 err = -errno;
5619                 free(link);
5620                 pr_warn("program '%s': failed to attach to pfd %d: %s\n",
5621                         bpf_program__title(prog, false), pfd,
5622                            libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5623                 return ERR_PTR(err);
5624         }
5625         if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
5626                 err = -errno;
5627                 free(link);
5628                 pr_warn("program '%s': failed to enable pfd %d: %s\n",
5629                         bpf_program__title(prog, false), pfd,
5630                            libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5631                 return ERR_PTR(err);
5632         }
5633         return (struct bpf_link *)link;
5634 }
5635
5636 /*
5637  * this function is expected to parse integer in the range of [0, 2^31-1] from
5638  * given file using scanf format string fmt. If actual parsed value is
5639  * negative, the result might be indistinguishable from error
5640  */
5641 static int parse_uint_from_file(const char *file, const char *fmt)
5642 {
5643         char buf[STRERR_BUFSIZE];
5644         int err, ret;
5645         FILE *f;
5646
5647         f = fopen(file, "r");
5648         if (!f) {
5649                 err = -errno;
5650                 pr_debug("failed to open '%s': %s\n", file,
5651                          libbpf_strerror_r(err, buf, sizeof(buf)));
5652                 return err;
5653         }
5654         err = fscanf(f, fmt, &ret);
5655         if (err != 1) {
5656                 err = err == EOF ? -EIO : -errno;
5657                 pr_debug("failed to parse '%s': %s\n", file,
5658                         libbpf_strerror_r(err, buf, sizeof(buf)));
5659                 fclose(f);
5660                 return err;
5661         }
5662         fclose(f);
5663         return ret;
5664 }
5665
5666 static int determine_kprobe_perf_type(void)
5667 {
5668         const char *file = "/sys/bus/event_source/devices/kprobe/type";
5669
5670         return parse_uint_from_file(file, "%d\n");
5671 }
5672
5673 static int determine_uprobe_perf_type(void)
5674 {
5675         const char *file = "/sys/bus/event_source/devices/uprobe/type";
5676
5677         return parse_uint_from_file(file, "%d\n");
5678 }
5679
5680 static int determine_kprobe_retprobe_bit(void)
5681 {
5682         const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
5683
5684         return parse_uint_from_file(file, "config:%d\n");
5685 }
5686
5687 static int determine_uprobe_retprobe_bit(void)
5688 {
5689         const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
5690
5691         return parse_uint_from_file(file, "config:%d\n");
5692 }
5693
5694 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
5695                                  uint64_t offset, int pid)
5696 {
5697         struct perf_event_attr attr = {};
5698         char errmsg[STRERR_BUFSIZE];
5699         int type, pfd, err;
5700
5701         type = uprobe ? determine_uprobe_perf_type()
5702                       : determine_kprobe_perf_type();
5703         if (type < 0) {
5704                 pr_warn("failed to determine %s perf type: %s\n",
5705                         uprobe ? "uprobe" : "kprobe",
5706                         libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
5707                 return type;
5708         }
5709         if (retprobe) {
5710                 int bit = uprobe ? determine_uprobe_retprobe_bit()
5711                                  : determine_kprobe_retprobe_bit();
5712
5713                 if (bit < 0) {
5714                         pr_warn("failed to determine %s retprobe bit: %s\n",
5715                                 uprobe ? "uprobe" : "kprobe",
5716                                 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
5717                         return bit;
5718                 }
5719                 attr.config |= 1 << bit;
5720         }
5721         attr.size = sizeof(attr);
5722         attr.type = type;
5723         attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
5724         attr.config2 = offset;           /* kprobe_addr or probe_offset */
5725
5726         /* pid filter is meaningful only for uprobes */
5727         pfd = syscall(__NR_perf_event_open, &attr,
5728                       pid < 0 ? -1 : pid /* pid */,
5729                       pid == -1 ? 0 : -1 /* cpu */,
5730                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
5731         if (pfd < 0) {
5732                 err = -errno;
5733                 pr_warn("%s perf_event_open() failed: %s\n",
5734                         uprobe ? "uprobe" : "kprobe",
5735                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5736                 return err;
5737         }
5738         return pfd;
5739 }
5740
5741 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
5742                                             bool retprobe,
5743                                             const char *func_name)
5744 {
5745         char errmsg[STRERR_BUFSIZE];
5746         struct bpf_link *link;
5747         int pfd, err;
5748
5749         pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,
5750                                     0 /* offset */, -1 /* pid */);
5751         if (pfd < 0) {
5752                 pr_warn("program '%s': failed to create %s '%s' perf event: %s\n",
5753                         bpf_program__title(prog, false),
5754                         retprobe ? "kretprobe" : "kprobe", func_name,
5755                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5756                 return ERR_PTR(pfd);
5757         }
5758         link = bpf_program__attach_perf_event(prog, pfd);
5759         if (IS_ERR(link)) {
5760                 close(pfd);
5761                 err = PTR_ERR(link);
5762                 pr_warn("program '%s': failed to attach to %s '%s': %s\n",
5763                         bpf_program__title(prog, false),
5764                         retprobe ? "kretprobe" : "kprobe", func_name,
5765                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5766                 return link;
5767         }
5768         return link;
5769 }
5770
5771 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
5772                                       struct bpf_program *prog)
5773 {
5774         const char *func_name;
5775         bool retprobe;
5776
5777         func_name = bpf_program__title(prog, false) + sec->len;
5778         retprobe = strcmp(sec->sec, "kretprobe/") == 0;
5779
5780         return bpf_program__attach_kprobe(prog, retprobe, func_name);
5781 }
5782
5783 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
5784                                             bool retprobe, pid_t pid,
5785                                             const char *binary_path,
5786                                             size_t func_offset)
5787 {
5788         char errmsg[STRERR_BUFSIZE];
5789         struct bpf_link *link;
5790         int pfd, err;
5791
5792         pfd = perf_event_open_probe(true /* uprobe */, retprobe,
5793                                     binary_path, func_offset, pid);
5794         if (pfd < 0) {
5795                 pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
5796                         bpf_program__title(prog, false),
5797                         retprobe ? "uretprobe" : "uprobe",
5798                         binary_path, func_offset,
5799                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5800                 return ERR_PTR(pfd);
5801         }
5802         link = bpf_program__attach_perf_event(prog, pfd);
5803         if (IS_ERR(link)) {
5804                 close(pfd);
5805                 err = PTR_ERR(link);
5806                 pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n",
5807                         bpf_program__title(prog, false),
5808                         retprobe ? "uretprobe" : "uprobe",
5809                         binary_path, func_offset,
5810                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5811                 return link;
5812         }
5813         return link;
5814 }
5815
5816 static int determine_tracepoint_id(const char *tp_category,
5817                                    const char *tp_name)
5818 {
5819         char file[PATH_MAX];
5820         int ret;
5821
5822         ret = snprintf(file, sizeof(file),
5823                        "/sys/kernel/debug/tracing/events/%s/%s/id",
5824                        tp_category, tp_name);
5825         if (ret < 0)
5826                 return -errno;
5827         if (ret >= sizeof(file)) {
5828                 pr_debug("tracepoint %s/%s path is too long\n",
5829                          tp_category, tp_name);
5830                 return -E2BIG;
5831         }
5832         return parse_uint_from_file(file, "%d\n");
5833 }
5834
5835 static int perf_event_open_tracepoint(const char *tp_category,
5836                                       const char *tp_name)
5837 {
5838         struct perf_event_attr attr = {};
5839         char errmsg[STRERR_BUFSIZE];
5840         int tp_id, pfd, err;
5841
5842         tp_id = determine_tracepoint_id(tp_category, tp_name);
5843         if (tp_id < 0) {
5844                 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
5845                         tp_category, tp_name,
5846                         libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
5847                 return tp_id;
5848         }
5849
5850         attr.type = PERF_TYPE_TRACEPOINT;
5851         attr.size = sizeof(attr);
5852         attr.config = tp_id;
5853
5854         pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
5855                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
5856         if (pfd < 0) {
5857                 err = -errno;
5858                 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
5859                         tp_category, tp_name,
5860                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5861                 return err;
5862         }
5863         return pfd;
5864 }
5865
5866 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
5867                                                 const char *tp_category,
5868                                                 const char *tp_name)
5869 {
5870         char errmsg[STRERR_BUFSIZE];
5871         struct bpf_link *link;
5872         int pfd, err;
5873
5874         pfd = perf_event_open_tracepoint(tp_category, tp_name);
5875         if (pfd < 0) {
5876                 pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
5877                         bpf_program__title(prog, false),
5878                         tp_category, tp_name,
5879                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5880                 return ERR_PTR(pfd);
5881         }
5882         link = bpf_program__attach_perf_event(prog, pfd);
5883         if (IS_ERR(link)) {
5884                 close(pfd);
5885                 err = PTR_ERR(link);
5886                 pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n",
5887                         bpf_program__title(prog, false),
5888                         tp_category, tp_name,
5889                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5890                 return link;
5891         }
5892         return link;
5893 }
5894
5895 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
5896                                   struct bpf_program *prog)
5897 {
5898         char *sec_name, *tp_cat, *tp_name;
5899         struct bpf_link *link;
5900
5901         sec_name = strdup(bpf_program__title(prog, false));
5902         if (!sec_name)
5903                 return ERR_PTR(-ENOMEM);
5904
5905         /* extract "tp/<category>/<name>" */
5906         tp_cat = sec_name + sec->len;
5907         tp_name = strchr(tp_cat, '/');
5908         if (!tp_name) {
5909                 link = ERR_PTR(-EINVAL);
5910                 goto out;
5911         }
5912         *tp_name = '\0';
5913         tp_name++;
5914
5915         link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
5916 out:
5917         free(sec_name);
5918         return link;
5919 }
5920
5921 static int bpf_link__destroy_fd(struct bpf_link *link)
5922 {
5923         struct bpf_link_fd *l = (void *)link;
5924
5925         return close(l->fd);
5926 }
5927
5928 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
5929                                                     const char *tp_name)
5930 {
5931         char errmsg[STRERR_BUFSIZE];
5932         struct bpf_link_fd *link;
5933         int prog_fd, pfd;
5934
5935         prog_fd = bpf_program__fd(prog);
5936         if (prog_fd < 0) {
5937                 pr_warn("program '%s': can't attach before loaded\n",
5938                         bpf_program__title(prog, false));
5939                 return ERR_PTR(-EINVAL);
5940         }
5941
5942         link = malloc(sizeof(*link));
5943         if (!link)
5944                 return ERR_PTR(-ENOMEM);
5945         link->link.destroy = &bpf_link__destroy_fd;
5946
5947         pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
5948         if (pfd < 0) {
5949                 pfd = -errno;
5950                 free(link);
5951                 pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n",
5952                         bpf_program__title(prog, false), tp_name,
5953                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5954                 return ERR_PTR(pfd);
5955         }
5956         link->fd = pfd;
5957         return (struct bpf_link *)link;
5958 }
5959
5960 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
5961                                       struct bpf_program *prog)
5962 {
5963         const char *tp_name = bpf_program__title(prog, false) + sec->len;
5964
5965         return bpf_program__attach_raw_tracepoint(prog, tp_name);
5966 }
5967
5968 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
5969 {
5970         char errmsg[STRERR_BUFSIZE];
5971         struct bpf_link_fd *link;
5972         int prog_fd, pfd;
5973
5974         prog_fd = bpf_program__fd(prog);
5975         if (prog_fd < 0) {
5976                 pr_warn("program '%s': can't attach before loaded\n",
5977                         bpf_program__title(prog, false));
5978                 return ERR_PTR(-EINVAL);
5979         }
5980
5981         link = malloc(sizeof(*link));
5982         if (!link)
5983                 return ERR_PTR(-ENOMEM);
5984         link->link.destroy = &bpf_link__destroy_fd;
5985
5986         pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
5987         if (pfd < 0) {
5988                 pfd = -errno;
5989                 free(link);
5990                 pr_warn("program '%s': failed to attach to trace: %s\n",
5991                         bpf_program__title(prog, false),
5992                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5993                 return ERR_PTR(pfd);
5994         }
5995         link->fd = pfd;
5996         return (struct bpf_link *)link;
5997 }
5998
5999 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
6000                                      struct bpf_program *prog)
6001 {
6002         return bpf_program__attach_trace(prog);
6003 }
6004
6005 struct bpf_link *bpf_program__attach(struct bpf_program *prog)
6006 {
6007         const struct bpf_sec_def *sec_def;
6008
6009         sec_def = find_sec_def(bpf_program__title(prog, false));
6010         if (!sec_def || !sec_def->attach_fn)
6011                 return ERR_PTR(-ESRCH);
6012
6013         return sec_def->attach_fn(sec_def, prog);
6014 }
6015
6016 enum bpf_perf_event_ret
6017 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
6018                            void **copy_mem, size_t *copy_size,
6019                            bpf_perf_event_print_t fn, void *private_data)
6020 {
6021         struct perf_event_mmap_page *header = mmap_mem;
6022         __u64 data_head = ring_buffer_read_head(header);
6023         __u64 data_tail = header->data_tail;
6024         void *base = ((__u8 *)header) + page_size;
6025         int ret = LIBBPF_PERF_EVENT_CONT;
6026         struct perf_event_header *ehdr;
6027         size_t ehdr_size;
6028
6029         while (data_head != data_tail) {
6030                 ehdr = base + (data_tail & (mmap_size - 1));
6031                 ehdr_size = ehdr->size;
6032
6033                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
6034                         void *copy_start = ehdr;
6035                         size_t len_first = base + mmap_size - copy_start;
6036                         size_t len_secnd = ehdr_size - len_first;
6037
6038                         if (*copy_size < ehdr_size) {
6039                                 free(*copy_mem);
6040                                 *copy_mem = malloc(ehdr_size);
6041                                 if (!*copy_mem) {
6042                                         *copy_size = 0;
6043                                         ret = LIBBPF_PERF_EVENT_ERROR;
6044                                         break;
6045                                 }
6046                                 *copy_size = ehdr_size;
6047                         }
6048
6049                         memcpy(*copy_mem, copy_start, len_first);
6050                         memcpy(*copy_mem + len_first, base, len_secnd);
6051                         ehdr = *copy_mem;
6052                 }
6053
6054                 ret = fn(ehdr, private_data);
6055                 data_tail += ehdr_size;
6056                 if (ret != LIBBPF_PERF_EVENT_CONT)
6057                         break;
6058         }
6059
6060         ring_buffer_write_tail(header, data_tail);
6061         return ret;
6062 }
6063
6064 struct perf_buffer;
6065
6066 struct perf_buffer_params {
6067         struct perf_event_attr *attr;
6068         /* if event_cb is specified, it takes precendence */
6069         perf_buffer_event_fn event_cb;
6070         /* sample_cb and lost_cb are higher-level common-case callbacks */
6071         perf_buffer_sample_fn sample_cb;
6072         perf_buffer_lost_fn lost_cb;
6073         void *ctx;
6074         int cpu_cnt;
6075         int *cpus;
6076         int *map_keys;
6077 };
6078
6079 struct perf_cpu_buf {
6080         struct perf_buffer *pb;
6081         void *base; /* mmap()'ed memory */
6082         void *buf; /* for reconstructing segmented data */
6083         size_t buf_size;
6084         int fd;
6085         int cpu;
6086         int map_key;
6087 };
6088
6089 struct perf_buffer {
6090         perf_buffer_event_fn event_cb;
6091         perf_buffer_sample_fn sample_cb;
6092         perf_buffer_lost_fn lost_cb;
6093         void *ctx; /* passed into callbacks */
6094
6095         size_t page_size;
6096         size_t mmap_size;
6097         struct perf_cpu_buf **cpu_bufs;
6098         struct epoll_event *events;
6099         int cpu_cnt; /* number of allocated CPU buffers */
6100         int epoll_fd; /* perf event FD */
6101         int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
6102 };
6103
6104 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
6105                                       struct perf_cpu_buf *cpu_buf)
6106 {
6107         if (!cpu_buf)
6108                 return;
6109         if (cpu_buf->base &&
6110             munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
6111                 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
6112         if (cpu_buf->fd >= 0) {
6113                 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
6114                 close(cpu_buf->fd);
6115         }
6116         free(cpu_buf->buf);
6117         free(cpu_buf);
6118 }
6119
6120 void perf_buffer__free(struct perf_buffer *pb)
6121 {
6122         int i;
6123
6124         if (!pb)
6125                 return;
6126         if (pb->cpu_bufs) {
6127                 for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) {
6128                         struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
6129
6130                         bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
6131                         perf_buffer__free_cpu_buf(pb, cpu_buf);
6132                 }
6133                 free(pb->cpu_bufs);
6134         }
6135         if (pb->epoll_fd >= 0)
6136                 close(pb->epoll_fd);
6137         free(pb->events);
6138         free(pb);
6139 }
6140
6141 static struct perf_cpu_buf *
6142 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
6143                           int cpu, int map_key)
6144 {
6145         struct perf_cpu_buf *cpu_buf;
6146         char msg[STRERR_BUFSIZE];
6147         int err;
6148
6149         cpu_buf = calloc(1, sizeof(*cpu_buf));
6150         if (!cpu_buf)
6151                 return ERR_PTR(-ENOMEM);
6152
6153         cpu_buf->pb = pb;
6154         cpu_buf->cpu = cpu;
6155         cpu_buf->map_key = map_key;
6156
6157         cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
6158                               -1, PERF_FLAG_FD_CLOEXEC);
6159         if (cpu_buf->fd < 0) {
6160                 err = -errno;
6161                 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
6162                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
6163                 goto error;
6164         }
6165
6166         cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
6167                              PROT_READ | PROT_WRITE, MAP_SHARED,
6168                              cpu_buf->fd, 0);
6169         if (cpu_buf->base == MAP_FAILED) {
6170                 cpu_buf->base = NULL;
6171                 err = -errno;
6172                 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
6173                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
6174                 goto error;
6175         }
6176
6177         if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
6178                 err = -errno;
6179                 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
6180                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
6181                 goto error;
6182         }
6183
6184         return cpu_buf;
6185
6186 error:
6187         perf_buffer__free_cpu_buf(pb, cpu_buf);
6188         return (struct perf_cpu_buf *)ERR_PTR(err);
6189 }
6190
6191 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
6192                                               struct perf_buffer_params *p);
6193
6194 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
6195                                      const struct perf_buffer_opts *opts)
6196 {
6197         struct perf_buffer_params p = {};
6198         struct perf_event_attr attr = { 0, };
6199
6200         attr.config = PERF_COUNT_SW_BPF_OUTPUT,
6201         attr.type = PERF_TYPE_SOFTWARE;
6202         attr.sample_type = PERF_SAMPLE_RAW;
6203         attr.sample_period = 1;
6204         attr.wakeup_events = 1;
6205
6206         p.attr = &attr;
6207         p.sample_cb = opts ? opts->sample_cb : NULL;
6208         p.lost_cb = opts ? opts->lost_cb : NULL;
6209         p.ctx = opts ? opts->ctx : NULL;
6210
6211         return __perf_buffer__new(map_fd, page_cnt, &p);
6212 }
6213
6214 struct perf_buffer *
6215 perf_buffer__new_raw(int map_fd, size_t page_cnt,
6216                      const struct perf_buffer_raw_opts *opts)
6217 {
6218         struct perf_buffer_params p = {};
6219
6220         p.attr = opts->attr;
6221         p.event_cb = opts->event_cb;
6222         p.ctx = opts->ctx;
6223         p.cpu_cnt = opts->cpu_cnt;
6224         p.cpus = opts->cpus;
6225         p.map_keys = opts->map_keys;
6226
6227         return __perf_buffer__new(map_fd, page_cnt, &p);
6228 }
6229
6230 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
6231                                               struct perf_buffer_params *p)
6232 {
6233         const char *online_cpus_file = "/sys/devices/system/cpu/online";
6234         struct bpf_map_info map = {};
6235         char msg[STRERR_BUFSIZE];
6236         struct perf_buffer *pb;
6237         bool *online = NULL;
6238         __u32 map_info_len;
6239         int err, i, j, n;
6240
6241         if (page_cnt & (page_cnt - 1)) {
6242                 pr_warn("page count should be power of two, but is %zu\n",
6243                         page_cnt);
6244                 return ERR_PTR(-EINVAL);
6245         }
6246
6247         map_info_len = sizeof(map);
6248         err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
6249         if (err) {
6250                 err = -errno;
6251                 pr_warn("failed to get map info for map FD %d: %s\n",
6252                         map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
6253                 return ERR_PTR(err);
6254         }
6255
6256         if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
6257                 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
6258                         map.name);
6259                 return ERR_PTR(-EINVAL);
6260         }
6261
6262         pb = calloc(1, sizeof(*pb));
6263         if (!pb)
6264                 return ERR_PTR(-ENOMEM);
6265
6266         pb->event_cb = p->event_cb;
6267         pb->sample_cb = p->sample_cb;
6268         pb->lost_cb = p->lost_cb;
6269         pb->ctx = p->ctx;
6270
6271         pb->page_size = getpagesize();
6272         pb->mmap_size = pb->page_size * page_cnt;
6273         pb->map_fd = map_fd;
6274
6275         pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
6276         if (pb->epoll_fd < 0) {
6277                 err = -errno;
6278                 pr_warn("failed to create epoll instance: %s\n",
6279                         libbpf_strerror_r(err, msg, sizeof(msg)));
6280                 goto error;
6281         }
6282
6283         if (p->cpu_cnt > 0) {
6284                 pb->cpu_cnt = p->cpu_cnt;
6285         } else {
6286                 pb->cpu_cnt = libbpf_num_possible_cpus();
6287                 if (pb->cpu_cnt < 0) {
6288                         err = pb->cpu_cnt;
6289                         goto error;
6290                 }
6291                 if (map.max_entries < pb->cpu_cnt)
6292                         pb->cpu_cnt = map.max_entries;
6293         }
6294
6295         pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
6296         if (!pb->events) {
6297                 err = -ENOMEM;
6298                 pr_warn("failed to allocate events: out of memory\n");
6299                 goto error;
6300         }
6301         pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
6302         if (!pb->cpu_bufs) {
6303                 err = -ENOMEM;
6304                 pr_warn("failed to allocate buffers: out of memory\n");
6305                 goto error;
6306         }
6307
6308         err = parse_cpu_mask_file(online_cpus_file, &online, &n);
6309         if (err) {
6310                 pr_warn("failed to get online CPU mask: %d\n", err);
6311                 goto error;
6312         }
6313
6314         for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
6315                 struct perf_cpu_buf *cpu_buf;
6316                 int cpu, map_key;
6317
6318                 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
6319                 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
6320
6321                 /* in case user didn't explicitly requested particular CPUs to
6322                  * be attached to, skip offline/not present CPUs
6323                  */
6324                 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
6325                         continue;
6326
6327                 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
6328                 if (IS_ERR(cpu_buf)) {
6329                         err = PTR_ERR(cpu_buf);
6330                         goto error;
6331                 }
6332
6333                 pb->cpu_bufs[j] = cpu_buf;
6334
6335                 err = bpf_map_update_elem(pb->map_fd, &map_key,
6336                                           &cpu_buf->fd, 0);
6337                 if (err) {
6338                         err = -errno;
6339                         pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
6340                                 cpu, map_key, cpu_buf->fd,
6341                                 libbpf_strerror_r(err, msg, sizeof(msg)));
6342                         goto error;
6343                 }
6344
6345                 pb->events[j].events = EPOLLIN;
6346                 pb->events[j].data.ptr = cpu_buf;
6347                 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
6348                               &pb->events[j]) < 0) {
6349                         err = -errno;
6350                         pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
6351                                 cpu, cpu_buf->fd,
6352                                 libbpf_strerror_r(err, msg, sizeof(msg)));
6353                         goto error;
6354                 }
6355                 j++;
6356         }
6357         pb->cpu_cnt = j;
6358         free(online);
6359
6360         return pb;
6361
6362 error:
6363         free(online);
6364         if (pb)
6365                 perf_buffer__free(pb);
6366         return ERR_PTR(err);
6367 }
6368
6369 struct perf_sample_raw {
6370         struct perf_event_header header;
6371         uint32_t size;
6372         char data[0];
6373 };
6374
6375 struct perf_sample_lost {
6376         struct perf_event_header header;
6377         uint64_t id;
6378         uint64_t lost;
6379         uint64_t sample_id;
6380 };
6381
6382 static enum bpf_perf_event_ret
6383 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
6384 {
6385         struct perf_cpu_buf *cpu_buf = ctx;
6386         struct perf_buffer *pb = cpu_buf->pb;
6387         void *data = e;
6388
6389         /* user wants full control over parsing perf event */
6390         if (pb->event_cb)
6391                 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
6392
6393         switch (e->type) {
6394         case PERF_RECORD_SAMPLE: {
6395                 struct perf_sample_raw *s = data;
6396
6397                 if (pb->sample_cb)
6398                         pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
6399                 break;
6400         }
6401         case PERF_RECORD_LOST: {
6402                 struct perf_sample_lost *s = data;
6403
6404                 if (pb->lost_cb)
6405                         pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
6406                 break;
6407         }
6408         default:
6409                 pr_warn("unknown perf sample type %d\n", e->type);
6410                 return LIBBPF_PERF_EVENT_ERROR;
6411         }
6412         return LIBBPF_PERF_EVENT_CONT;
6413 }
6414
6415 static int perf_buffer__process_records(struct perf_buffer *pb,
6416                                         struct perf_cpu_buf *cpu_buf)
6417 {
6418         enum bpf_perf_event_ret ret;
6419
6420         ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
6421                                          pb->page_size, &cpu_buf->buf,
6422                                          &cpu_buf->buf_size,
6423                                          perf_buffer__process_record, cpu_buf);
6424         if (ret != LIBBPF_PERF_EVENT_CONT)
6425                 return ret;
6426         return 0;
6427 }
6428
6429 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
6430 {
6431         int i, cnt, err;
6432
6433         cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
6434         for (i = 0; i < cnt; i++) {
6435                 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
6436
6437                 err = perf_buffer__process_records(pb, cpu_buf);
6438                 if (err) {
6439                         pr_warn("error while processing records: %d\n", err);
6440                         return err;
6441                 }
6442         }
6443         return cnt < 0 ? -errno : cnt;
6444 }
6445
6446 struct bpf_prog_info_array_desc {
6447         int     array_offset;   /* e.g. offset of jited_prog_insns */
6448         int     count_offset;   /* e.g. offset of jited_prog_len */
6449         int     size_offset;    /* > 0: offset of rec size,
6450                                  * < 0: fix size of -size_offset
6451                                  */
6452 };
6453
6454 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
6455         [BPF_PROG_INFO_JITED_INSNS] = {
6456                 offsetof(struct bpf_prog_info, jited_prog_insns),
6457                 offsetof(struct bpf_prog_info, jited_prog_len),
6458                 -1,
6459         },
6460         [BPF_PROG_INFO_XLATED_INSNS] = {
6461                 offsetof(struct bpf_prog_info, xlated_prog_insns),
6462                 offsetof(struct bpf_prog_info, xlated_prog_len),
6463                 -1,
6464         },
6465         [BPF_PROG_INFO_MAP_IDS] = {
6466                 offsetof(struct bpf_prog_info, map_ids),
6467                 offsetof(struct bpf_prog_info, nr_map_ids),
6468                 -(int)sizeof(__u32),
6469         },
6470         [BPF_PROG_INFO_JITED_KSYMS] = {
6471                 offsetof(struct bpf_prog_info, jited_ksyms),
6472                 offsetof(struct bpf_prog_info, nr_jited_ksyms),
6473                 -(int)sizeof(__u64),
6474         },
6475         [BPF_PROG_INFO_JITED_FUNC_LENS] = {
6476                 offsetof(struct bpf_prog_info, jited_func_lens),
6477                 offsetof(struct bpf_prog_info, nr_jited_func_lens),
6478                 -(int)sizeof(__u32),
6479         },
6480         [BPF_PROG_INFO_FUNC_INFO] = {
6481                 offsetof(struct bpf_prog_info, func_info),
6482                 offsetof(struct bpf_prog_info, nr_func_info),
6483                 offsetof(struct bpf_prog_info, func_info_rec_size),
6484         },
6485         [BPF_PROG_INFO_LINE_INFO] = {
6486                 offsetof(struct bpf_prog_info, line_info),
6487                 offsetof(struct bpf_prog_info, nr_line_info),
6488                 offsetof(struct bpf_prog_info, line_info_rec_size),
6489         },
6490         [BPF_PROG_INFO_JITED_LINE_INFO] = {
6491                 offsetof(struct bpf_prog_info, jited_line_info),
6492                 offsetof(struct bpf_prog_info, nr_jited_line_info),
6493                 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
6494         },
6495         [BPF_PROG_INFO_PROG_TAGS] = {
6496                 offsetof(struct bpf_prog_info, prog_tags),
6497                 offsetof(struct bpf_prog_info, nr_prog_tags),
6498                 -(int)sizeof(__u8) * BPF_TAG_SIZE,
6499         },
6500
6501 };
6502
6503 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
6504                                            int offset)
6505 {
6506         __u32 *array = (__u32 *)info;
6507
6508         if (offset >= 0)
6509                 return array[offset / sizeof(__u32)];
6510         return -(int)offset;
6511 }
6512
6513 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
6514                                            int offset)
6515 {
6516         __u64 *array = (__u64 *)info;
6517
6518         if (offset >= 0)
6519                 return array[offset / sizeof(__u64)];
6520         return -(int)offset;
6521 }
6522
6523 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
6524                                          __u32 val)
6525 {
6526         __u32 *array = (__u32 *)info;
6527
6528         if (offset >= 0)
6529                 array[offset / sizeof(__u32)] = val;
6530 }
6531
6532 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
6533                                          __u64 val)
6534 {
6535         __u64 *array = (__u64 *)info;
6536
6537         if (offset >= 0)
6538                 array[offset / sizeof(__u64)] = val;
6539 }
6540
6541 struct bpf_prog_info_linear *
6542 bpf_program__get_prog_info_linear(int fd, __u64 arrays)
6543 {
6544         struct bpf_prog_info_linear *info_linear;
6545         struct bpf_prog_info info = {};
6546         __u32 info_len = sizeof(info);
6547         __u32 data_len = 0;
6548         int i, err;
6549         void *ptr;
6550
6551         if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
6552                 return ERR_PTR(-EINVAL);
6553
6554         /* step 1: get array dimensions */
6555         err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
6556         if (err) {
6557                 pr_debug("can't get prog info: %s", strerror(errno));
6558                 return ERR_PTR(-EFAULT);
6559         }
6560
6561         /* step 2: calculate total size of all arrays */
6562         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
6563                 bool include_array = (arrays & (1UL << i)) > 0;
6564                 struct bpf_prog_info_array_desc *desc;
6565                 __u32 count, size;
6566
6567                 desc = bpf_prog_info_array_desc + i;
6568
6569                 /* kernel is too old to support this field */
6570                 if (info_len < desc->array_offset + sizeof(__u32) ||
6571                     info_len < desc->count_offset + sizeof(__u32) ||
6572                     (desc->size_offset > 0 && info_len < desc->size_offset))
6573                         include_array = false;
6574
6575                 if (!include_array) {
6576                         arrays &= ~(1UL << i);  /* clear the bit */
6577                         continue;
6578                 }
6579
6580                 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
6581                 size  = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
6582
6583                 data_len += count * size;
6584         }
6585
6586         /* step 3: allocate continuous memory */
6587         data_len = roundup(data_len, sizeof(__u64));
6588         info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
6589         if (!info_linear)
6590                 return ERR_PTR(-ENOMEM);
6591
6592         /* step 4: fill data to info_linear->info */
6593         info_linear->arrays = arrays;
6594         memset(&info_linear->info, 0, sizeof(info));
6595         ptr = info_linear->data;
6596
6597         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
6598                 struct bpf_prog_info_array_desc *desc;
6599                 __u32 count, size;
6600
6601                 if ((arrays & (1UL << i)) == 0)
6602                         continue;
6603
6604                 desc  = bpf_prog_info_array_desc + i;
6605                 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
6606                 size  = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
6607                 bpf_prog_info_set_offset_u32(&info_linear->info,
6608                                              desc->count_offset, count);
6609                 bpf_prog_info_set_offset_u32(&info_linear->info,
6610                                              desc->size_offset, size);
6611                 bpf_prog_info_set_offset_u64(&info_linear->info,
6612                                              desc->array_offset,
6613                                              ptr_to_u64(ptr));
6614                 ptr += count * size;
6615         }
6616
6617         /* step 5: call syscall again to get required arrays */
6618         err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
6619         if (err) {
6620                 pr_debug("can't get prog info: %s", strerror(errno));
6621                 free(info_linear);
6622                 return ERR_PTR(-EFAULT);
6623         }
6624
6625         /* step 6: verify the data */
6626         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
6627                 struct bpf_prog_info_array_desc *desc;
6628                 __u32 v1, v2;
6629
6630                 if ((arrays & (1UL << i)) == 0)
6631                         continue;
6632
6633                 desc = bpf_prog_info_array_desc + i;
6634                 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
6635                 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
6636                                                    desc->count_offset);
6637                 if (v1 != v2)
6638                         pr_warn("%s: mismatch in element count\n", __func__);
6639
6640                 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
6641                 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
6642                                                    desc->size_offset);
6643                 if (v1 != v2)
6644                         pr_warn("%s: mismatch in rec size\n", __func__);
6645         }
6646
6647         /* step 7: update info_len and data_len */
6648         info_linear->info_len = sizeof(struct bpf_prog_info);
6649         info_linear->data_len = data_len;
6650
6651         return info_linear;
6652 }
6653
6654 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
6655 {
6656         int i;
6657
6658         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
6659                 struct bpf_prog_info_array_desc *desc;
6660                 __u64 addr, offs;
6661
6662                 if ((info_linear->arrays & (1UL << i)) == 0)
6663                         continue;
6664
6665                 desc = bpf_prog_info_array_desc + i;
6666                 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
6667                                                      desc->array_offset);
6668                 offs = addr - ptr_to_u64(info_linear->data);
6669                 bpf_prog_info_set_offset_u64(&info_linear->info,
6670                                              desc->array_offset, offs);
6671         }
6672 }
6673
6674 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
6675 {
6676         int i;
6677
6678         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
6679                 struct bpf_prog_info_array_desc *desc;
6680                 __u64 addr, offs;
6681
6682                 if ((info_linear->arrays & (1UL << i)) == 0)
6683                         continue;
6684
6685                 desc = bpf_prog_info_array_desc + i;
6686                 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
6687                                                      desc->array_offset);
6688                 addr = offs + ptr_to_u64(info_linear->data);
6689                 bpf_prog_info_set_offset_u64(&info_linear->info,
6690                                              desc->array_offset, addr);
6691         }
6692 }
6693
6694 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
6695 {
6696         int err = 0, n, len, start, end = -1;
6697         bool *tmp;
6698
6699         *mask = NULL;
6700         *mask_sz = 0;
6701
6702         /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
6703         while (*s) {
6704                 if (*s == ',' || *s == '\n') {
6705                         s++;
6706                         continue;
6707                 }
6708                 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
6709                 if (n <= 0 || n > 2) {
6710                         pr_warn("Failed to get CPU range %s: %d\n", s, n);
6711                         err = -EINVAL;
6712                         goto cleanup;
6713                 } else if (n == 1) {
6714                         end = start;
6715                 }
6716                 if (start < 0 || start > end) {
6717                         pr_warn("Invalid CPU range [%d,%d] in %s\n",
6718                                 start, end, s);
6719                         err = -EINVAL;
6720                         goto cleanup;
6721                 }
6722                 tmp = realloc(*mask, end + 1);
6723                 if (!tmp) {
6724                         err = -ENOMEM;
6725                         goto cleanup;
6726                 }
6727                 *mask = tmp;
6728                 memset(tmp + *mask_sz, 0, start - *mask_sz);
6729                 memset(tmp + start, 1, end - start + 1);
6730                 *mask_sz = end + 1;
6731                 s += len;
6732         }
6733         if (!*mask_sz) {
6734                 pr_warn("Empty CPU range\n");
6735                 return -EINVAL;
6736         }
6737         return 0;
6738 cleanup:
6739         free(*mask);
6740         *mask = NULL;
6741         return err;
6742 }
6743
6744 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
6745 {
6746         int fd, err = 0, len;
6747         char buf[128];
6748
6749         fd = open(fcpu, O_RDONLY);
6750         if (fd < 0) {
6751                 err = -errno;
6752                 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
6753                 return err;
6754         }
6755         len = read(fd, buf, sizeof(buf));
6756         close(fd);
6757         if (len <= 0) {
6758                 err = len ? -errno : -EINVAL;
6759                 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
6760                 return err;
6761         }
6762         if (len >= sizeof(buf)) {
6763                 pr_warn("CPU mask is too big in file %s\n", fcpu);
6764                 return -E2BIG;
6765         }
6766         buf[len] = '\0';
6767
6768         return parse_cpu_mask_str(buf, mask, mask_sz);
6769 }
6770
6771 int libbpf_num_possible_cpus(void)
6772 {
6773         static const char *fcpu = "/sys/devices/system/cpu/possible";
6774         static int cpus;
6775         int err, n, i, tmp_cpus;
6776         bool *mask;
6777
6778         tmp_cpus = READ_ONCE(cpus);
6779         if (tmp_cpus > 0)
6780                 return tmp_cpus;
6781
6782         err = parse_cpu_mask_file(fcpu, &mask, &n);
6783         if (err)
6784                 return err;
6785
6786         tmp_cpus = 0;
6787         for (i = 0; i < n; i++) {
6788                 if (mask[i])
6789                         tmp_cpus++;
6790         }
6791         free(mask);
6792
6793         WRITE_ONCE(cpus, tmp_cpus);
6794         return tmp_cpus;
6795 }