tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  * Copyright (C) 2019 Isovalent, Inc.
  11  */
  12
  13 #ifndef _GNU_SOURCE
  14 #define _GNU_SOURCE
  15 #endif
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <stdarg.h>
  19 #include <libgen.h>
  20 #include <inttypes.h>
  21 #include <limits.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <endian.h>
  25 #include <fcntl.h>
  26 #include <errno.h>
  27 #include <ctype.h>
  28 #include <asm/unistd.h>
  29 #include <linux/err.h>
  30 #include <linux/kernel.h>
  31 #include <linux/bpf.h>
  32 #include <linux/btf.h>
  33 #include <linux/filter.h>
  34 #include <linux/list.h>
  35 #include <linux/limits.h>
  36 #include <linux/perf_event.h>
  37 #include <linux/ring_buffer.h>
  38 #include <linux/version.h>
  39 #include <sys/epoll.h>
  40 #include <sys/ioctl.h>
  41 #include <sys/mman.h>
  42 #include <sys/stat.h>
  43 #include <sys/types.h>
  44 #include <sys/vfs.h>
  45 #include <sys/utsname.h>
  46 #include <sys/resource.h>
  47 #include <libelf.h>
  48 #include <gelf.h>
  49 #include <zlib.h>
  50
  51 #include "libbpf.h"
  52 #include "bpf.h"
  53 #include "btf.h"
  54 #include "str_error.h"
  55 #include "libbpf_internal.h"
  56 #include "hashmap.h"
  57
  58 #ifndef BPF_FS_MAGIC
  59 #define BPF_FS_MAGIC            0xcafe4a11
  60 #endif
  61
  62 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
  63
  64 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
  65  * compilation if user enables corresponding warning. Disable it explicitly.
  66  */
  67 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
  68
  69 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  70
  71 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
  72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
  73
  74 static int __base_pr(enum libbpf_print_level level, const char *format,
  75                      va_list args)
  76 {
  77         if (level == LIBBPF_DEBUG)
  78                 return 0;
  79
  80         return vfprintf(stderr, format, args);
  81 }
  82
  83 static libbpf_print_fn_t __libbpf_pr = __base_pr;
  84
  85 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
  86 {
  87         libbpf_print_fn_t old_print_fn = __libbpf_pr;
  88
  89         __libbpf_pr = fn;
  90         return old_print_fn;
  91 }
  92
  93 __printf(2, 3)
  94 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
  95 {
  96         va_list args;
  97
  98         if (!__libbpf_pr)
  99                 return;
 100
 101         va_start(args, format);
 102         __libbpf_pr(level, format, args);
 103         va_end(args);
 104 }
 105
 106 static void pr_perm_msg(int err)
 107 {
 108         struct rlimit limit;
 109         char buf[100];
 110
 111         if (err != -EPERM || geteuid() != 0)
 112                 return;
 113
 114         err = getrlimit(RLIMIT_MEMLOCK, &limit);
 115         if (err)
 116                 return;
 117
 118         if (limit.rlim_cur == RLIM_INFINITY)
 119                 return;
 120
 121         if (limit.rlim_cur < 1024)
 122                 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
 123         else if (limit.rlim_cur < 1024*1024)
 124                 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
 125         else
 126                 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
 127
 128         pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
 129                 buf);
 130 }
 131
 132 #define STRERR_BUFSIZE  128
 133
 134 /* Copied from tools/perf/util/util.h */
 135 #ifndef zfree
 136 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 137 #endif
 138
 139 #ifndef zclose
 140 # define zclose(fd) ({                  \
 141         int ___err = 0;                 \
 142         if ((fd) >= 0)                  \
 143                 ___err = close((fd));   \
 144         fd = -1;                        \
 145         ___err; })
 146 #endif
 147
 148 static inline __u64 ptr_to_u64(const void *ptr)
 149 {
 150         return (__u64) (unsigned long) ptr;
 151 }
 152
 153 enum kern_feature_id {
 154         /* v4.14: kernel support for program & map names. */
 155         FEAT_PROG_NAME,
 156         /* v5.2: kernel support for global data sections. */
 157         FEAT_GLOBAL_DATA,
 158         /* BTF support */
 159         FEAT_BTF,
 160         /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
 161         FEAT_BTF_FUNC,
 162         /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
 163         FEAT_BTF_DATASEC,
 164         /* BTF_FUNC_GLOBAL is supported */
 165         FEAT_BTF_GLOBAL_FUNC,
 166         /* BPF_F_MMAPABLE is supported for arrays */
 167         FEAT_ARRAY_MMAP,
 168         /* kernel support for expected_attach_type in BPF_PROG_LOAD */
 169         FEAT_EXP_ATTACH_TYPE,
 170         /* bpf_probe_read_{kernel,user}[_str] helpers */
 171         FEAT_PROBE_READ_KERN,
 172         /* BPF_PROG_BIND_MAP is supported */
 173         FEAT_PROG_BIND_MAP,
 174         /* Kernel support for module BTFs */
 175         FEAT_MODULE_BTF,
 176         /* BTF_KIND_FLOAT support */
 177         FEAT_BTF_FLOAT,
 178         __FEAT_CNT,
 179 };
 180
 181 static bool kernel_supports(enum kern_feature_id feat_id);
 182
 183 enum reloc_type {
 184         RELO_LD64,
 185         RELO_CALL,
 186         RELO_DATA,
 187         RELO_EXTERN_VAR,
 188         RELO_EXTERN_FUNC,
 189         RELO_SUBPROG_ADDR,
 190 };
 191
 192 struct reloc_desc {
 193         enum reloc_type type;
 194         int insn_idx;
 195         int map_idx;
 196         int sym_off;
 197 };
 198
 199 struct bpf_sec_def;
 200
 201 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
 202                                         struct bpf_program *prog);
 203
 204 struct bpf_sec_def {
 205         const char *sec;
 206         size_t len;
 207         enum bpf_prog_type prog_type;
 208         enum bpf_attach_type expected_attach_type;
 209         bool is_exp_attach_type_optional;
 210         bool is_attachable;
 211         bool is_attach_btf;
 212         bool is_sleepable;
 213         attach_fn_t attach_fn;
 214 };
 215
 216 /*
 217  * bpf_prog should be a better name but it has been used in
 218  * linux/filter.h.
 219  */
 220 struct bpf_program {
 221         const struct bpf_sec_def *sec_def;
 222         char *sec_name;
 223         size_t sec_idx;
 224         /* this program's instruction offset (in number of instructions)
 225          * within its containing ELF section
 226          */
 227         size_t sec_insn_off;
 228         /* number of original instructions in ELF section belonging to this
 229          * program, not taking into account subprogram instructions possible
 230          * appended later during relocation
 231          */
 232         size_t sec_insn_cnt;
 233         /* Offset (in number of instructions) of the start of instruction
 234          * belonging to this BPF program  within its containing main BPF
 235          * program. For the entry-point (main) BPF program, this is always
 236          * zero. For a sub-program, this gets reset before each of main BPF
 237          * programs are processed and relocated and is used to determined
 238          * whether sub-program was already appended to the main program, and
 239          * if yes, at which instruction offset.
 240          */
 241         size_t sub_insn_off;
 242
 243         char *name;
 244         /* sec_name with / replaced by _; makes recursive pinning
 245          * in bpf_object__pin_programs easier
 246          */
 247         char *pin_name;
 248
 249         /* instructions that belong to BPF program; insns[0] is located at
 250          * sec_insn_off instruction within its ELF section in ELF file, so
 251          * when mapping ELF file instruction index to the local instruction,
 252          * one needs to subtract sec_insn_off; and vice versa.
 253          */
 254         struct bpf_insn *insns;
 255         /* actual number of instruction in this BPF program's image; for
 256          * entry-point BPF programs this includes the size of main program
 257          * itself plus all the used sub-programs, appended at the end
 258          */
 259         size_t insns_cnt;
 260
 261         struct reloc_desc *reloc_desc;
 262         int nr_reloc;
 263         int log_level;
 264
 265         struct {
 266                 int nr;
 267                 int *fds;
 268         } instances;
 269         bpf_program_prep_t preprocessor;
 270
 271         struct bpf_object *obj;
 272         void *priv;
 273         bpf_program_clear_priv_t clear_priv;
 274
 275         bool load;
 276         bool mark_btf_static;
 277         enum bpf_prog_type type;
 278         enum bpf_attach_type expected_attach_type;
 279         int prog_ifindex;
 280         __u32 attach_btf_obj_fd;
 281         __u32 attach_btf_id;
 282         __u32 attach_prog_fd;
 283         void *func_info;
 284         __u32 func_info_rec_size;
 285         __u32 func_info_cnt;
 286
 287         void *line_info;
 288         __u32 line_info_rec_size;
 289         __u32 line_info_cnt;
 290         __u32 prog_flags;
 291 };
 292
 293 struct bpf_struct_ops {
 294         const char *tname;
 295         const struct btf_type *type;
 296         struct bpf_program **progs;
 297         __u32 *kern_func_off;
 298         /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
 299         void *data;
 300         /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
 301          *      btf_vmlinux's format.
 302          * struct bpf_struct_ops_tcp_congestion_ops {
 303          *      [... some other kernel fields ...]
 304          *      struct tcp_congestion_ops data;
 305          * }
 306          * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
 307          * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
 308          * from "data".
 309          */
 310         void *kern_vdata;
 311         __u32 type_id;
 312 };
 313
 314 #define DATA_SEC ".data"
 315 #define BSS_SEC ".bss"
 316 #define RODATA_SEC ".rodata"
 317 #define KCONFIG_SEC ".kconfig"
 318 #define KSYMS_SEC ".ksyms"
 319 #define STRUCT_OPS_SEC ".struct_ops"
 320
 321 enum libbpf_map_type {
 322         LIBBPF_MAP_UNSPEC,
 323         LIBBPF_MAP_DATA,
 324         LIBBPF_MAP_BSS,
 325         LIBBPF_MAP_RODATA,
 326         LIBBPF_MAP_KCONFIG,
 327 };
 328
 329 static const char * const libbpf_type_to_btf_name[] = {
 330         [LIBBPF_MAP_DATA]       = DATA_SEC,
 331         [LIBBPF_MAP_BSS]        = BSS_SEC,
 332         [LIBBPF_MAP_RODATA]     = RODATA_SEC,
 333         [LIBBPF_MAP_KCONFIG]    = KCONFIG_SEC,
 334 };
 335
 336 struct bpf_map {
 337         char *name;
 338         int fd;
 339         int sec_idx;
 340         size_t sec_offset;
 341         int map_ifindex;
 342         int inner_map_fd;
 343         struct bpf_map_def def;
 344         __u32 numa_node;
 345         __u32 btf_var_idx;
 346         __u32 btf_key_type_id;
 347         __u32 btf_value_type_id;
 348         __u32 btf_vmlinux_value_type_id;
 349         void *priv;
 350         bpf_map_clear_priv_t clear_priv;
 351         enum libbpf_map_type libbpf_type;
 352         void *mmaped;
 353         struct bpf_struct_ops *st_ops;
 354         struct bpf_map *inner_map;
 355         void **init_slots;
 356         int init_slots_sz;
 357         char *pin_path;
 358         bool pinned;
 359         bool reused;
 360 };
 361
 362 enum extern_type {
 363         EXT_UNKNOWN,
 364         EXT_KCFG,
 365         EXT_KSYM,
 366 };
 367
 368 enum kcfg_type {
 369         KCFG_UNKNOWN,
 370         KCFG_CHAR,
 371         KCFG_BOOL,
 372         KCFG_INT,
 373         KCFG_TRISTATE,
 374         KCFG_CHAR_ARR,
 375 };
 376
 377 struct extern_desc {
 378         enum extern_type type;
 379         int sym_idx;
 380         int btf_id;
 381         int sec_btf_id;
 382         const char *name;
 383         bool is_set;
 384         bool is_weak;
 385         union {
 386                 struct {
 387                         enum kcfg_type type;
 388                         int sz;
 389                         int align;
 390                         int data_off;
 391                         bool is_signed;
 392                 } kcfg;
 393                 struct {
 394                         unsigned long long addr;
 395
 396                         /* target btf_id of the corresponding kernel var. */
 397                         int kernel_btf_obj_fd;
 398                         int kernel_btf_id;
 399
 400                         /* local btf_id of the ksym extern's type. */
 401                         __u32 type_id;
 402                 } ksym;
 403         };
 404 };
 405
 406 static LIST_HEAD(bpf_objects_list);
 407
 408 struct module_btf {
 409         struct btf *btf;
 410         char *name;
 411         __u32 id;
 412         int fd;
 413 };
 414
 415 struct bpf_object {
 416         char name[BPF_OBJ_NAME_LEN];
 417         char license[64];
 418         __u32 kern_version;
 419
 420         struct bpf_program *programs;
 421         size_t nr_programs;
 422         struct bpf_map *maps;
 423         size_t nr_maps;
 424         size_t maps_cap;
 425
 426         char *kconfig;
 427         struct extern_desc *externs;
 428         int nr_extern;
 429         int kconfig_map_idx;
 430         int rodata_map_idx;
 431
 432         bool loaded;
 433         bool has_subcalls;
 434
 435         /*
 436          * Information when doing elf related work. Only valid if fd
 437          * is valid.
 438          */
 439         struct {
 440                 int fd;
 441                 const void *obj_buf;
 442                 size_t obj_buf_sz;
 443                 Elf *elf;
 444                 GElf_Ehdr ehdr;
 445                 Elf_Data *symbols;
 446                 Elf_Data *data;
 447                 Elf_Data *rodata;
 448                 Elf_Data *bss;
 449                 Elf_Data *st_ops_data;
 450                 size_t shstrndx; /* section index for section name strings */
 451                 size_t strtabidx;
 452                 struct {
 453                         GElf_Shdr shdr;
 454                         Elf_Data *data;
 455                 } *reloc_sects;
 456                 int nr_reloc_sects;
 457                 int maps_shndx;
 458                 int btf_maps_shndx;
 459                 __u32 btf_maps_sec_btf_id;
 460                 int text_shndx;
 461                 int symbols_shndx;
 462                 int data_shndx;
 463                 int rodata_shndx;
 464                 int bss_shndx;
 465                 int st_ops_shndx;
 466         } efile;
 467         /*
 468          * All loaded bpf_object is linked in a list, which is
 469          * hidden to caller. bpf_objects__<func> handlers deal with
 470          * all objects.
 471          */
 472         struct list_head list;
 473
 474         struct btf *btf;
 475         struct btf_ext *btf_ext;
 476
 477         /* Parse and load BTF vmlinux if any of the programs in the object need
 478          * it at load time.
 479          */
 480         struct btf *btf_vmlinux;
 481         /* vmlinux BTF override for CO-RE relocations */
 482         struct btf *btf_vmlinux_override;
 483         /* Lazily initialized kernel module BTFs */
 484         struct module_btf *btf_modules;
 485         bool btf_modules_loaded;
 486         size_t btf_module_cnt;
 487         size_t btf_module_cap;
 488
 489         void *priv;
 490         bpf_object_clear_priv_t clear_priv;
 491
 492         char path[];
 493 };
 494 #define obj_elf_valid(o)        ((o)->efile.elf)
 495
 496 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
 497 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
 498 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
 499 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
 500 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr);
 501 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
 502 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
 503
 504 void bpf_program__unload(struct bpf_program *prog)
 505 {
 506         int i;
 507
 508         if (!prog)
 509                 return;
 510
 511         /*
 512          * If the object is opened but the program was never loaded,
 513          * it is possible that prog->instances.nr == -1.
 514          */
 515         if (prog->instances.nr > 0) {
 516                 for (i = 0; i < prog->instances.nr; i++)
 517                         zclose(prog->instances.fds[i]);
 518         } else if (prog->instances.nr != -1) {
 519                 pr_warn("Internal error: instances.nr is %d\n",
 520                         prog->instances.nr);
 521         }
 522
 523         prog->instances.nr = -1;
 524         zfree(&prog->instances.fds);
 525
 526         zfree(&prog->func_info);
 527         zfree(&prog->line_info);
 528 }
 529
 530 static void bpf_program__exit(struct bpf_program *prog)
 531 {
 532         if (!prog)
 533                 return;
 534
 535         if (prog->clear_priv)
 536                 prog->clear_priv(prog, prog->priv);
 537
 538         prog->priv = NULL;
 539         prog->clear_priv = NULL;
 540
 541         bpf_program__unload(prog);
 542         zfree(&prog->name);
 543         zfree(&prog->sec_name);
 544         zfree(&prog->pin_name);
 545         zfree(&prog->insns);
 546         zfree(&prog->reloc_desc);
 547
 548         prog->nr_reloc = 0;
 549         prog->insns_cnt = 0;
 550         prog->sec_idx = -1;
 551 }
 552
 553 static char *__bpf_program__pin_name(struct bpf_program *prog)
 554 {
 555         char *name, *p;
 556
 557         name = p = strdup(prog->sec_name);
 558         while ((p = strchr(p, '/')))
 559                 *p = '_';
 560
 561         return name;
 562 }
 563
 564 static bool insn_is_subprog_call(const struct bpf_insn *insn)
 565 {
 566         return BPF_CLASS(insn->code) == BPF_JMP &&
 567                BPF_OP(insn->code) == BPF_CALL &&
 568                BPF_SRC(insn->code) == BPF_K &&
 569                insn->src_reg == BPF_PSEUDO_CALL &&
 570                insn->dst_reg == 0 &&
 571                insn->off == 0;
 572 }
 573
 574 static bool is_ldimm64_insn(struct bpf_insn *insn)
 575 {
 576         return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
 577 }
 578
 579 static bool is_call_insn(const struct bpf_insn *insn)
 580 {
 581         return insn->code == (BPF_JMP | BPF_CALL);
 582 }
 583
 584 static bool insn_is_pseudo_func(struct bpf_insn *insn)
 585 {
 586         return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
 587 }
 588
 589 static int
 590 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
 591                       const char *name, size_t sec_idx, const char *sec_name,
 592                       size_t sec_off, void *insn_data, size_t insn_data_sz)
 593 {
 594         if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
 595                 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
 596                         sec_name, name, sec_off, insn_data_sz);
 597                 return -EINVAL;
 598         }
 599
 600         memset(prog, 0, sizeof(*prog));
 601         prog->obj = obj;
 602
 603         prog->sec_idx = sec_idx;
 604         prog->sec_insn_off = sec_off / BPF_INSN_SZ;
 605         prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
 606         /* insns_cnt can later be increased by appending used subprograms */
 607         prog->insns_cnt = prog->sec_insn_cnt;
 608
 609         prog->type = BPF_PROG_TYPE_UNSPEC;
 610         prog->load = true;
 611
 612         prog->instances.fds = NULL;
 613         prog->instances.nr = -1;
 614
 615         prog->sec_name = strdup(sec_name);
 616         if (!prog->sec_name)
 617                 goto errout;
 618
 619         prog->name = strdup(name);
 620         if (!prog->name)
 621                 goto errout;
 622
 623         prog->pin_name = __bpf_program__pin_name(prog);
 624         if (!prog->pin_name)
 625                 goto errout;
 626
 627         prog->insns = malloc(insn_data_sz);
 628         if (!prog->insns)
 629                 goto errout;
 630         memcpy(prog->insns, insn_data, insn_data_sz);
 631
 632         return 0;
 633 errout:
 634         pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
 635         bpf_program__exit(prog);
 636         return -ENOMEM;
 637 }
 638
 639 static int
 640 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
 641                          const char *sec_name, int sec_idx)
 642 {
 643         Elf_Data *symbols = obj->efile.symbols;
 644         struct bpf_program *prog, *progs;
 645         void *data = sec_data->d_buf;
 646         size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
 647         int nr_progs, err, i;
 648         const char *name;
 649         GElf_Sym sym;
 650
 651         progs = obj->programs;
 652         nr_progs = obj->nr_programs;
 653         nr_syms = symbols->d_size / sizeof(GElf_Sym);
 654         sec_off = 0;
 655
 656         for (i = 0; i < nr_syms; i++) {
 657                 if (!gelf_getsym(symbols, i, &sym))
 658                         continue;
 659                 if (sym.st_shndx != sec_idx)
 660                         continue;
 661                 if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
 662                         continue;
 663
 664                 prog_sz = sym.st_size;
 665                 sec_off = sym.st_value;
 666
 667                 name = elf_sym_str(obj, sym.st_name);
 668                 if (!name) {
 669                         pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
 670                                 sec_name, sec_off);
 671                         return -LIBBPF_ERRNO__FORMAT;
 672                 }
 673
 674                 if (sec_off + prog_sz > sec_sz) {
 675                         pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
 676                                 sec_name, sec_off);
 677                         return -LIBBPF_ERRNO__FORMAT;
 678                 }
 679
 680                 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
 681                          sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
 682
 683                 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
 684                 if (!progs) {
 685                         /*
 686                          * In this case the original obj->programs
 687                          * is still valid, so don't need special treat for
 688                          * bpf_close_object().
 689                          */
 690                         pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
 691                                 sec_name, name);
 692                         return -ENOMEM;
 693                 }
 694                 obj->programs = progs;
 695
 696                 prog = &progs[nr_progs];
 697
 698                 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
 699                                             sec_off, data + sec_off, prog_sz);
 700                 if (err)
 701                         return err;
 702
 703                 /* if function is a global/weak symbol, but has hidden
 704                  * visibility (STV_HIDDEN), mark its BTF FUNC as static to
 705                  * enable more permissive BPF verification mode with more
 706                  * outside context available to BPF verifier
 707                  */
 708                 if (GELF_ST_BIND(sym.st_info) != STB_LOCAL
 709                     && GELF_ST_VISIBILITY(sym.st_other) == STV_HIDDEN)
 710                         prog->mark_btf_static = true;
 711
 712                 nr_progs++;
 713                 obj->nr_programs = nr_progs;
 714         }
 715
 716         return 0;
 717 }
 718
 719 static __u32 get_kernel_version(void)
 720 {
 721         __u32 major, minor, patch;
 722         struct utsname info;
 723
 724         uname(&info);
 725         if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
 726                 return 0;
 727         return KERNEL_VERSION(major, minor, patch);
 728 }
 729
 730 static const struct btf_member *
 731 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
 732 {
 733         struct btf_member *m;
 734         int i;
 735
 736         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 737                 if (btf_member_bit_offset(t, i) == bit_offset)
 738                         return m;
 739         }
 740
 741         return NULL;
 742 }
 743
 744 static const struct btf_member *
 745 find_member_by_name(const struct btf *btf, const struct btf_type *t,
 746                     const char *name)
 747 {
 748         struct btf_member *m;
 749         int i;
 750
 751         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 752                 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
 753                         return m;
 754         }
 755
 756         return NULL;
 757 }
 758
 759 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
 760 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
 761                                    const char *name, __u32 kind);
 762
 763 static int
 764 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
 765                            const struct btf_type **type, __u32 *type_id,
 766                            const struct btf_type **vtype, __u32 *vtype_id,
 767                            const struct btf_member **data_member)
 768 {
 769         const struct btf_type *kern_type, *kern_vtype;
 770         const struct btf_member *kern_data_member;
 771         __s32 kern_vtype_id, kern_type_id;
 772         __u32 i;
 773
 774         kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
 775         if (kern_type_id < 0) {
 776                 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
 777                         tname);
 778                 return kern_type_id;
 779         }
 780         kern_type = btf__type_by_id(btf, kern_type_id);
 781
 782         /* Find the corresponding "map_value" type that will be used
 783          * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
 784          * find "struct bpf_struct_ops_tcp_congestion_ops" from the
 785          * btf_vmlinux.
 786          */
 787         kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
 788                                                 tname, BTF_KIND_STRUCT);
 789         if (kern_vtype_id < 0) {
 790                 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
 791                         STRUCT_OPS_VALUE_PREFIX, tname);
 792                 return kern_vtype_id;
 793         }
 794         kern_vtype = btf__type_by_id(btf, kern_vtype_id);
 795
 796         /* Find "struct tcp_congestion_ops" from
 797          * struct bpf_struct_ops_tcp_congestion_ops {
 798          *      [ ... ]
 799          *      struct tcp_congestion_ops data;
 800          * }
 801          */
 802         kern_data_member = btf_members(kern_vtype);
 803         for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
 804                 if (kern_data_member->type == kern_type_id)
 805                         break;
 806         }
 807         if (i == btf_vlen(kern_vtype)) {
 808                 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
 809                         tname, STRUCT_OPS_VALUE_PREFIX, tname);
 810                 return -EINVAL;
 811         }
 812
 813         *type = kern_type;
 814         *type_id = kern_type_id;
 815         *vtype = kern_vtype;
 816         *vtype_id = kern_vtype_id;
 817         *data_member = kern_data_member;
 818
 819         return 0;
 820 }
 821
 822 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
 823 {
 824         return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
 825 }
 826
 827 /* Init the map's fields that depend on kern_btf */
 828 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
 829                                          const struct btf *btf,
 830                                          const struct btf *kern_btf)
 831 {
 832         const struct btf_member *member, *kern_member, *kern_data_member;
 833         const struct btf_type *type, *kern_type, *kern_vtype;
 834         __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
 835         struct bpf_struct_ops *st_ops;
 836         void *data, *kern_data;
 837         const char *tname;
 838         int err;
 839
 840         st_ops = map->st_ops;
 841         type = st_ops->type;
 842         tname = st_ops->tname;
 843         err = find_struct_ops_kern_types(kern_btf, tname,
 844                                          &kern_type, &kern_type_id,
 845                                          &kern_vtype, &kern_vtype_id,
 846                                          &kern_data_member);
 847         if (err)
 848                 return err;
 849
 850         pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
 851                  map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
 852
 853         map->def.value_size = kern_vtype->size;
 854         map->btf_vmlinux_value_type_id = kern_vtype_id;
 855
 856         st_ops->kern_vdata = calloc(1, kern_vtype->size);
 857         if (!st_ops->kern_vdata)
 858                 return -ENOMEM;
 859
 860         data = st_ops->data;
 861         kern_data_off = kern_data_member->offset / 8;
 862         kern_data = st_ops->kern_vdata + kern_data_off;
 863
 864         member = btf_members(type);
 865         for (i = 0; i < btf_vlen(type); i++, member++) {
 866                 const struct btf_type *mtype, *kern_mtype;
 867                 __u32 mtype_id, kern_mtype_id;
 868                 void *mdata, *kern_mdata;
 869                 __s64 msize, kern_msize;
 870                 __u32 moff, kern_moff;
 871                 __u32 kern_member_idx;
 872                 const char *mname;
 873
 874                 mname = btf__name_by_offset(btf, member->name_off);
 875                 kern_member = find_member_by_name(kern_btf, kern_type, mname);
 876                 if (!kern_member) {
 877                         pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
 878                                 map->name, mname);
 879                         return -ENOTSUP;
 880                 }
 881
 882                 kern_member_idx = kern_member - btf_members(kern_type);
 883                 if (btf_member_bitfield_size(type, i) ||
 884                     btf_member_bitfield_size(kern_type, kern_member_idx)) {
 885                         pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
 886                                 map->name, mname);
 887                         return -ENOTSUP;
 888                 }
 889
 890                 moff = member->offset / 8;
 891                 kern_moff = kern_member->offset / 8;
 892
 893                 mdata = data + moff;
 894                 kern_mdata = kern_data + kern_moff;
 895
 896                 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
 897                 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
 898                                                     &kern_mtype_id);
 899                 if (BTF_INFO_KIND(mtype->info) !=
 900                     BTF_INFO_KIND(kern_mtype->info)) {
 901                         pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
 902                                 map->name, mname, BTF_INFO_KIND(mtype->info),
 903                                 BTF_INFO_KIND(kern_mtype->info));
 904                         return -ENOTSUP;
 905                 }
 906
 907                 if (btf_is_ptr(mtype)) {
 908                         struct bpf_program *prog;
 909
 910                         prog = st_ops->progs[i];
 911                         if (!prog)
 912                                 continue;
 913
 914                         kern_mtype = skip_mods_and_typedefs(kern_btf,
 915                                                             kern_mtype->type,
 916                                                             &kern_mtype_id);
 917
 918                         /* mtype->type must be a func_proto which was
 919                          * guaranteed in bpf_object__collect_st_ops_relos(),
 920                          * so only check kern_mtype for func_proto here.
 921                          */
 922                         if (!btf_is_func_proto(kern_mtype)) {
 923                                 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
 924                                         map->name, mname);
 925                                 return -ENOTSUP;
 926                         }
 927
 928                         prog->attach_btf_id = kern_type_id;
 929                         prog->expected_attach_type = kern_member_idx;
 930
 931                         st_ops->kern_func_off[i] = kern_data_off + kern_moff;
 932
 933                         pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
 934                                  map->name, mname, prog->name, moff,
 935                                  kern_moff);
 936
 937                         continue;
 938                 }
 939
 940                 msize = btf__resolve_size(btf, mtype_id);
 941                 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
 942                 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
 943                         pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
 944                                 map->name, mname, (ssize_t)msize,
 945                                 (ssize_t)kern_msize);
 946                         return -ENOTSUP;
 947                 }
 948
 949                 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
 950                          map->name, mname, (unsigned int)msize,
 951                          moff, kern_moff);
 952                 memcpy(kern_mdata, mdata, msize);
 953         }
 954
 955         return 0;
 956 }
 957
 958 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
 959 {
 960         struct bpf_map *map;
 961         size_t i;
 962         int err;
 963
 964         for (i = 0; i < obj->nr_maps; i++) {
 965                 map = &obj->maps[i];
 966
 967                 if (!bpf_map__is_struct_ops(map))
 968                         continue;
 969
 970                 err = bpf_map__init_kern_struct_ops(map, obj->btf,
 971                                                     obj->btf_vmlinux);
 972                 if (err)
 973                         return err;
 974         }
 975
 976         return 0;
 977 }
 978
 979 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
 980 {
 981         const struct btf_type *type, *datasec;
 982         const struct btf_var_secinfo *vsi;
 983         struct bpf_struct_ops *st_ops;
 984         const char *tname, *var_name;
 985         __s32 type_id, datasec_id;
 986         const struct btf *btf;
 987         struct bpf_map *map;
 988         __u32 i;
 989
 990         if (obj->efile.st_ops_shndx == -1)
 991                 return 0;
 992
 993         btf = obj->btf;
 994         datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
 995                                             BTF_KIND_DATASEC);
 996         if (datasec_id < 0) {
 997                 pr_warn("struct_ops init: DATASEC %s not found\n",
 998                         STRUCT_OPS_SEC);
 999                 return -EINVAL;
1000         }
1001
1002         datasec = btf__type_by_id(btf, datasec_id);
1003         vsi = btf_var_secinfos(datasec);
1004         for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1005                 type = btf__type_by_id(obj->btf, vsi->type);
1006                 var_name = btf__name_by_offset(obj->btf, type->name_off);
1007
1008                 type_id = btf__resolve_type(obj->btf, vsi->type);
1009                 if (type_id < 0) {
1010                         pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1011                                 vsi->type, STRUCT_OPS_SEC);
1012                         return -EINVAL;
1013                 }
1014
1015                 type = btf__type_by_id(obj->btf, type_id);
1016                 tname = btf__name_by_offset(obj->btf, type->name_off);
1017                 if (!tname[0]) {
1018                         pr_warn("struct_ops init: anonymous type is not supported\n");
1019                         return -ENOTSUP;
1020                 }
1021                 if (!btf_is_struct(type)) {
1022                         pr_warn("struct_ops init: %s is not a struct\n", tname);
1023                         return -EINVAL;
1024                 }
1025
1026                 map = bpf_object__add_map(obj);
1027                 if (IS_ERR(map))
1028                         return PTR_ERR(map);
1029
1030                 map->sec_idx = obj->efile.st_ops_shndx;
1031                 map->sec_offset = vsi->offset;
1032                 map->name = strdup(var_name);
1033                 if (!map->name)
1034                         return -ENOMEM;
1035
1036                 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1037                 map->def.key_size = sizeof(int);
1038                 map->def.value_size = type->size;
1039                 map->def.max_entries = 1;
1040
1041                 map->st_ops = calloc(1, sizeof(*map->st_ops));
1042                 if (!map->st_ops)
1043                         return -ENOMEM;
1044                 st_ops = map->st_ops;
1045                 st_ops->data = malloc(type->size);
1046                 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1047                 st_ops->kern_func_off = malloc(btf_vlen(type) *
1048                                                sizeof(*st_ops->kern_func_off));
1049                 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1050                         return -ENOMEM;
1051
1052                 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1053                         pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1054                                 var_name, STRUCT_OPS_SEC);
1055                         return -EINVAL;
1056                 }
1057
1058                 memcpy(st_ops->data,
1059                        obj->efile.st_ops_data->d_buf + vsi->offset,
1060                        type->size);
1061                 st_ops->tname = tname;
1062                 st_ops->type = type;
1063                 st_ops->type_id = type_id;
1064
1065                 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1066                          tname, type_id, var_name, vsi->offset);
1067         }
1068
1069         return 0;
1070 }
1071
1072 static struct bpf_object *bpf_object__new(const char *path,
1073                                           const void *obj_buf,
1074                                           size_t obj_buf_sz,
1075                                           const char *obj_name)
1076 {
1077         struct bpf_object *obj;
1078         char *end;
1079
1080         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1081         if (!obj) {
1082                 pr_warn("alloc memory failed for %s\n", path);
1083                 return ERR_PTR(-ENOMEM);
1084         }
1085
1086         strcpy(obj->path, path);
1087         if (obj_name) {
1088                 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
1089                 obj->name[sizeof(obj->name) - 1] = 0;
1090         } else {
1091                 /* Using basename() GNU version which doesn't modify arg. */
1092                 strncpy(obj->name, basename((void *)path),
1093                         sizeof(obj->name) - 1);
1094                 end = strchr(obj->name, '.');
1095                 if (end)
1096                         *end = 0;
1097         }
1098
1099         obj->efile.fd = -1;
1100         /*
1101          * Caller of this function should also call
1102          * bpf_object__elf_finish() after data collection to return
1103          * obj_buf to user. If not, we should duplicate the buffer to
1104          * avoid user freeing them before elf finish.
1105          */
1106         obj->efile.obj_buf = obj_buf;
1107         obj->efile.obj_buf_sz = obj_buf_sz;
1108         obj->efile.maps_shndx = -1;
1109         obj->efile.btf_maps_shndx = -1;
1110         obj->efile.data_shndx = -1;
1111         obj->efile.rodata_shndx = -1;
1112         obj->efile.bss_shndx = -1;
1113         obj->efile.st_ops_shndx = -1;
1114         obj->kconfig_map_idx = -1;
1115         obj->rodata_map_idx = -1;
1116
1117         obj->kern_version = get_kernel_version();
1118         obj->loaded = false;
1119
1120         INIT_LIST_HEAD(&obj->list);
1121         list_add(&obj->list, &bpf_objects_list);
1122         return obj;
1123 }
1124
1125 static void bpf_object__elf_finish(struct bpf_object *obj)
1126 {
1127         if (!obj_elf_valid(obj))
1128                 return;
1129
1130         if (obj->efile.elf) {
1131                 elf_end(obj->efile.elf);
1132                 obj->efile.elf = NULL;
1133         }
1134         obj->efile.symbols = NULL;
1135         obj->efile.data = NULL;
1136         obj->efile.rodata = NULL;
1137         obj->efile.bss = NULL;
1138         obj->efile.st_ops_data = NULL;
1139
1140         zfree(&obj->efile.reloc_sects);
1141         obj->efile.nr_reloc_sects = 0;
1142         zclose(obj->efile.fd);
1143         obj->efile.obj_buf = NULL;
1144         obj->efile.obj_buf_sz = 0;
1145 }
1146
1147 static int bpf_object__elf_init(struct bpf_object *obj)
1148 {
1149         int err = 0;
1150         GElf_Ehdr *ep;
1151
1152         if (obj_elf_valid(obj)) {
1153                 pr_warn("elf: init internal error\n");
1154                 return -LIBBPF_ERRNO__LIBELF;
1155         }
1156
1157         if (obj->efile.obj_buf_sz > 0) {
1158                 /*
1159                  * obj_buf should have been validated by
1160                  * bpf_object__open_buffer().
1161                  */
1162                 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1163                                             obj->efile.obj_buf_sz);
1164         } else {
1165                 obj->efile.fd = open(obj->path, O_RDONLY);
1166                 if (obj->efile.fd < 0) {
1167                         char errmsg[STRERR_BUFSIZE], *cp;
1168
1169                         err = -errno;
1170                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1171                         pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1172                         return err;
1173                 }
1174
1175                 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1176         }
1177
1178         if (!obj->efile.elf) {
1179                 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1180                 err = -LIBBPF_ERRNO__LIBELF;
1181                 goto errout;
1182         }
1183
1184         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1185                 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1186                 err = -LIBBPF_ERRNO__FORMAT;
1187                 goto errout;
1188         }
1189         ep = &obj->efile.ehdr;
1190
1191         if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) {
1192                 pr_warn("elf: failed to get section names section index for %s: %s\n",
1193                         obj->path, elf_errmsg(-1));
1194                 err = -LIBBPF_ERRNO__FORMAT;
1195                 goto errout;
1196         }
1197
1198         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1199         if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) {
1200                 pr_warn("elf: failed to get section names strings from %s: %s\n",
1201                         obj->path, elf_errmsg(-1));
1202                 err = -LIBBPF_ERRNO__FORMAT;
1203                 goto errout;
1204         }
1205
1206         /* Old LLVM set e_machine to EM_NONE */
1207         if (ep->e_type != ET_REL ||
1208             (ep->e_machine && ep->e_machine != EM_BPF)) {
1209                 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1210                 err = -LIBBPF_ERRNO__FORMAT;
1211                 goto errout;
1212         }
1213
1214         return 0;
1215 errout:
1216         bpf_object__elf_finish(obj);
1217         return err;
1218 }
1219
1220 static int bpf_object__check_endianness(struct bpf_object *obj)
1221 {
1222 #if __BYTE_ORDER == __LITTLE_ENDIAN
1223         if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1224                 return 0;
1225 #elif __BYTE_ORDER == __BIG_ENDIAN
1226         if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1227                 return 0;
1228 #else
1229 # error "Unrecognized __BYTE_ORDER__"
1230 #endif
1231         pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1232         return -LIBBPF_ERRNO__ENDIAN;
1233 }
1234
1235 static int
1236 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1237 {
1238         memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1239         pr_debug("license of %s is %s\n", obj->path, obj->license);
1240         return 0;
1241 }
1242
1243 static int
1244 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1245 {
1246         __u32 kver;
1247
1248         if (size != sizeof(kver)) {
1249                 pr_warn("invalid kver section in %s\n", obj->path);
1250                 return -LIBBPF_ERRNO__FORMAT;
1251         }
1252         memcpy(&kver, data, sizeof(kver));
1253         obj->kern_version = kver;
1254         pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1255         return 0;
1256 }
1257
1258 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1259 {
1260         if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1261             type == BPF_MAP_TYPE_HASH_OF_MAPS)
1262                 return true;
1263         return false;
1264 }
1265
1266 int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1267                              __u32 *size)
1268 {
1269         int ret = -ENOENT;
1270
1271         *size = 0;
1272         if (!name) {
1273                 return -EINVAL;
1274         } else if (!strcmp(name, DATA_SEC)) {
1275                 if (obj->efile.data)
1276                         *size = obj->efile.data->d_size;
1277         } else if (!strcmp(name, BSS_SEC)) {
1278                 if (obj->efile.bss)
1279                         *size = obj->efile.bss->d_size;
1280         } else if (!strcmp(name, RODATA_SEC)) {
1281                 if (obj->efile.rodata)
1282                         *size = obj->efile.rodata->d_size;
1283         } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1284                 if (obj->efile.st_ops_data)
1285                         *size = obj->efile.st_ops_data->d_size;
1286         } else {
1287                 Elf_Scn *scn = elf_sec_by_name(obj, name);
1288                 Elf_Data *data = elf_sec_data(obj, scn);
1289
1290                 if (data) {
1291                         ret = 0; /* found it */
1292                         *size = data->d_size;
1293                 }
1294         }
1295
1296         return *size ? 0 : ret;
1297 }
1298
1299 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1300                                 __u32 *off)
1301 {
1302         Elf_Data *symbols = obj->efile.symbols;
1303         const char *sname;
1304         size_t si;
1305
1306         if (!name || !off)
1307                 return -EINVAL;
1308
1309         for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1310                 GElf_Sym sym;
1311
1312                 if (!gelf_getsym(symbols, si, &sym))
1313                         continue;
1314                 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1315                     GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1316                         continue;
1317
1318                 sname = elf_sym_str(obj, sym.st_name);
1319                 if (!sname) {
1320                         pr_warn("failed to get sym name string for var %s\n",
1321                                 name);
1322                         return -EIO;
1323                 }
1324                 if (strcmp(name, sname) == 0) {
1325                         *off = sym.st_value;
1326                         return 0;
1327                 }
1328         }
1329
1330         return -ENOENT;
1331 }
1332
1333 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1334 {
1335         struct bpf_map *new_maps;
1336         size_t new_cap;
1337         int i;
1338
1339         if (obj->nr_maps < obj->maps_cap)
1340                 return &obj->maps[obj->nr_maps++];
1341
1342         new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1343         new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps));
1344         if (!new_maps) {
1345                 pr_warn("alloc maps for object failed\n");
1346                 return ERR_PTR(-ENOMEM);
1347         }
1348
1349         obj->maps_cap = new_cap;
1350         obj->maps = new_maps;
1351
1352         /* zero out new maps */
1353         memset(obj->maps + obj->nr_maps, 0,
1354                (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1355         /*
1356          * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
1357          * when failure (zclose won't close negative fd)).
1358          */
1359         for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1360                 obj->maps[i].fd = -1;
1361                 obj->maps[i].inner_map_fd = -1;
1362         }
1363
1364         return &obj->maps[obj->nr_maps++];
1365 }
1366
1367 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1368 {
1369         long page_sz = sysconf(_SC_PAGE_SIZE);
1370         size_t map_sz;
1371
1372         map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1373         map_sz = roundup(map_sz, page_sz);
1374         return map_sz;
1375 }
1376
1377 static char *internal_map_name(struct bpf_object *obj,
1378                                enum libbpf_map_type type)
1379 {
1380         char map_name[BPF_OBJ_NAME_LEN], *p;
1381         const char *sfx = libbpf_type_to_btf_name[type];
1382         int sfx_len = max((size_t)7, strlen(sfx));
1383         int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1384                           strlen(obj->name));
1385
1386         snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1387                  sfx_len, libbpf_type_to_btf_name[type]);
1388
1389         /* sanitise map name to characters allowed by kernel */
1390         for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1391                 if (!isalnum(*p) && *p != '_' && *p != '.')
1392                         *p = '_';
1393
1394         return strdup(map_name);
1395 }
1396
1397 static int
1398 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1399                               int sec_idx, void *data, size_t data_sz)
1400 {
1401         struct bpf_map_def *def;
1402         struct bpf_map *map;
1403         int err;
1404
1405         map = bpf_object__add_map(obj);
1406         if (IS_ERR(map))
1407                 return PTR_ERR(map);
1408
1409         map->libbpf_type = type;
1410         map->sec_idx = sec_idx;
1411         map->sec_offset = 0;
1412         map->name = internal_map_name(obj, type);
1413         if (!map->name) {
1414                 pr_warn("failed to alloc map name\n");
1415                 return -ENOMEM;
1416         }
1417
1418         def = &map->def;
1419         def->type = BPF_MAP_TYPE_ARRAY;
1420         def->key_size = sizeof(int);
1421         def->value_size = data_sz;
1422         def->max_entries = 1;
1423         def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1424                          ? BPF_F_RDONLY_PROG : 0;
1425         def->map_flags |= BPF_F_MMAPABLE;
1426
1427         pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1428                  map->name, map->sec_idx, map->sec_offset, def->map_flags);
1429
1430         map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1431                            MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1432         if (map->mmaped == MAP_FAILED) {
1433                 err = -errno;
1434                 map->mmaped = NULL;
1435                 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1436                         map->name, err);
1437                 zfree(&map->name);
1438                 return err;
1439         }
1440
1441         if (data)
1442                 memcpy(map->mmaped, data, data_sz);
1443
1444         pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1445         return 0;
1446 }
1447
1448 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1449 {
1450         int err;
1451
1452         /*
1453          * Populate obj->maps with libbpf internal maps.
1454          */
1455         if (obj->efile.data_shndx >= 0) {
1456                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1457                                                     obj->efile.data_shndx,
1458                                                     obj->efile.data->d_buf,
1459                                                     obj->efile.data->d_size);
1460                 if (err)
1461                         return err;
1462         }
1463         if (obj->efile.rodata_shndx >= 0) {
1464                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1465                                                     obj->efile.rodata_shndx,
1466                                                     obj->efile.rodata->d_buf,
1467                                                     obj->efile.rodata->d_size);
1468                 if (err)
1469                         return err;
1470
1471                 obj->rodata_map_idx = obj->nr_maps - 1;
1472         }
1473         if (obj->efile.bss_shndx >= 0) {
1474                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1475                                                     obj->efile.bss_shndx,
1476                                                     NULL,
1477                                                     obj->efile.bss->d_size);
1478                 if (err)
1479                         return err;
1480         }
1481         return 0;
1482 }
1483
1484
1485 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1486                                                const void *name)
1487 {
1488         int i;
1489
1490         for (i = 0; i < obj->nr_extern; i++) {
1491                 if (strcmp(obj->externs[i].name, name) == 0)
1492                         return &obj->externs[i];
1493         }
1494         return NULL;
1495 }
1496
1497 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1498                               char value)
1499 {
1500         switch (ext->kcfg.type) {
1501         case KCFG_BOOL:
1502                 if (value == 'm') {
1503                         pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
1504                                 ext->name, value);
1505                         return -EINVAL;
1506                 }
1507                 *(bool *)ext_val = value == 'y' ? true : false;
1508                 break;
1509         case KCFG_TRISTATE:
1510                 if (value == 'y')
1511                         *(enum libbpf_tristate *)ext_val = TRI_YES;
1512                 else if (value == 'm')
1513                         *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1514                 else /* value == 'n' */
1515                         *(enum libbpf_tristate *)ext_val = TRI_NO;
1516                 break;
1517         case KCFG_CHAR:
1518                 *(char *)ext_val = value;
1519                 break;
1520         case KCFG_UNKNOWN:
1521         case KCFG_INT:
1522         case KCFG_CHAR_ARR:
1523         default:
1524                 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
1525                         ext->name, value);
1526                 return -EINVAL;
1527         }
1528         ext->is_set = true;
1529         return 0;
1530 }
1531
1532 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1533                               const char *value)
1534 {
1535         size_t len;
1536
1537         if (ext->kcfg.type != KCFG_CHAR_ARR) {
1538                 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
1539                 return -EINVAL;
1540         }
1541
1542         len = strlen(value);
1543         if (value[len - 1] != '"') {
1544                 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1545                         ext->name, value);
1546                 return -EINVAL;
1547         }
1548
1549         /* strip quotes */
1550         len -= 2;
1551         if (len >= ext->kcfg.sz) {
1552                 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1553                         ext->name, value, len, ext->kcfg.sz - 1);
1554                 len = ext->kcfg.sz - 1;
1555         }
1556         memcpy(ext_val, value + 1, len);
1557         ext_val[len] = '\0';
1558         ext->is_set = true;
1559         return 0;
1560 }
1561
1562 static int parse_u64(const char *value, __u64 *res)
1563 {
1564         char *value_end;
1565         int err;
1566
1567         errno = 0;
1568         *res = strtoull(value, &value_end, 0);
1569         if (errno) {
1570                 err = -errno;
1571                 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1572                 return err;
1573         }
1574         if (*value_end) {
1575                 pr_warn("failed to parse '%s' as integer completely\n", value);
1576                 return -EINVAL;
1577         }
1578         return 0;
1579 }
1580
1581 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1582 {
1583         int bit_sz = ext->kcfg.sz * 8;
1584
1585         if (ext->kcfg.sz == 8)
1586                 return true;
1587
1588         /* Validate that value stored in u64 fits in integer of `ext->sz`
1589          * bytes size without any loss of information. If the target integer
1590          * is signed, we rely on the following limits of integer type of
1591          * Y bits and subsequent transformation:
1592          *
1593          *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
1594          *            0 <= X + 2^(Y-1) <= 2^Y - 1
1595          *            0 <= X + 2^(Y-1) <  2^Y
1596          *
1597          *  For unsigned target integer, check that all the (64 - Y) bits are
1598          *  zero.
1599          */
1600         if (ext->kcfg.is_signed)
1601                 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1602         else
1603                 return (v >> bit_sz) == 0;
1604 }
1605
1606 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1607                               __u64 value)
1608 {
1609         if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1610                 pr_warn("extern (kcfg) %s=%llu should be integer\n",
1611                         ext->name, (unsigned long long)value);
1612                 return -EINVAL;
1613         }
1614         if (!is_kcfg_value_in_range(ext, value)) {
1615                 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
1616                         ext->name, (unsigned long long)value, ext->kcfg.sz);
1617                 return -ERANGE;
1618         }
1619         switch (ext->kcfg.sz) {
1620                 case 1: *(__u8 *)ext_val = value; break;
1621                 case 2: *(__u16 *)ext_val = value; break;
1622                 case 4: *(__u32 *)ext_val = value; break;
1623                 case 8: *(__u64 *)ext_val = value; break;
1624                 default:
1625                         return -EINVAL;
1626         }
1627         ext->is_set = true;
1628         return 0;
1629 }
1630
1631 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1632                                             char *buf, void *data)
1633 {
1634         struct extern_desc *ext;
1635         char *sep, *value;
1636         int len, err = 0;
1637         void *ext_val;
1638         __u64 num;
1639
1640         if (strncmp(buf, "CONFIG_", 7))
1641                 return 0;
1642
1643         sep = strchr(buf, '=');
1644         if (!sep) {
1645                 pr_warn("failed to parse '%s': no separator\n", buf);
1646                 return -EINVAL;
1647         }
1648
1649         /* Trim ending '\n' */
1650         len = strlen(buf);
1651         if (buf[len - 1] == '\n')
1652                 buf[len - 1] = '\0';
1653         /* Split on '=' and ensure that a value is present. */
1654         *sep = '\0';
1655         if (!sep[1]) {
1656                 *sep = '=';
1657                 pr_warn("failed to parse '%s': no value\n", buf);
1658                 return -EINVAL;
1659         }
1660
1661         ext = find_extern_by_name(obj, buf);
1662         if (!ext || ext->is_set)
1663                 return 0;
1664
1665         ext_val = data + ext->kcfg.data_off;
1666         value = sep + 1;
1667
1668         switch (*value) {
1669         case 'y': case 'n': case 'm':
1670                 err = set_kcfg_value_tri(ext, ext_val, *value);
1671                 break;
1672         case '"':
1673                 err = set_kcfg_value_str(ext, ext_val, value);
1674                 break;
1675         default:
1676                 /* assume integer */
1677                 err = parse_u64(value, &num);
1678                 if (err) {
1679                         pr_warn("extern (kcfg) %s=%s should be integer\n",
1680                                 ext->name, value);
1681                         return err;
1682                 }
1683                 err = set_kcfg_value_num(ext, ext_val, num);
1684                 break;
1685         }
1686         if (err)
1687                 return err;
1688         pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
1689         return 0;
1690 }
1691
1692 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1693 {
1694         char buf[PATH_MAX];
1695         struct utsname uts;
1696         int len, err = 0;
1697         gzFile file;
1698
1699         uname(&uts);
1700         len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1701         if (len < 0)
1702                 return -EINVAL;
1703         else if (len >= PATH_MAX)
1704                 return -ENAMETOOLONG;
1705
1706         /* gzopen also accepts uncompressed files. */
1707         file = gzopen(buf, "r");
1708         if (!file)
1709                 file = gzopen("/proc/config.gz", "r");
1710
1711         if (!file) {
1712                 pr_warn("failed to open system Kconfig\n");
1713                 return -ENOENT;
1714         }
1715
1716         while (gzgets(file, buf, sizeof(buf))) {
1717                 err = bpf_object__process_kconfig_line(obj, buf, data);
1718                 if (err) {
1719                         pr_warn("error parsing system Kconfig line '%s': %d\n",
1720                                 buf, err);
1721                         goto out;
1722                 }
1723         }
1724
1725 out:
1726         gzclose(file);
1727         return err;
1728 }
1729
1730 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1731                                         const char *config, void *data)
1732 {
1733         char buf[PATH_MAX];
1734         int err = 0;
1735         FILE *file;
1736
1737         file = fmemopen((void *)config, strlen(config), "r");
1738         if (!file) {
1739                 err = -errno;
1740                 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1741                 return err;
1742         }
1743
1744         while (fgets(buf, sizeof(buf), file)) {
1745                 err = bpf_object__process_kconfig_line(obj, buf, data);
1746                 if (err) {
1747                         pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1748                                 buf, err);
1749                         break;
1750                 }
1751         }
1752
1753         fclose(file);
1754         return err;
1755 }
1756
1757 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1758 {
1759         struct extern_desc *last_ext = NULL, *ext;
1760         size_t map_sz;
1761         int i, err;
1762
1763         for (i = 0; i < obj->nr_extern; i++) {
1764                 ext = &obj->externs[i];
1765                 if (ext->type == EXT_KCFG)
1766                         last_ext = ext;
1767         }
1768
1769         if (!last_ext)
1770                 return 0;
1771
1772         map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1773         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1774                                             obj->efile.symbols_shndx,
1775                                             NULL, map_sz);
1776         if (err)
1777                 return err;
1778
1779         obj->kconfig_map_idx = obj->nr_maps - 1;
1780
1781         return 0;
1782 }
1783
1784 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1785 {
1786         Elf_Data *symbols = obj->efile.symbols;
1787         int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1788         Elf_Data *data = NULL;
1789         Elf_Scn *scn;
1790
1791         if (obj->efile.maps_shndx < 0)
1792                 return 0;
1793
1794         if (!symbols)
1795                 return -EINVAL;
1796
1797
1798         scn = elf_sec_by_idx(obj, obj->efile.maps_shndx);
1799         data = elf_sec_data(obj, scn);
1800         if (!scn || !data) {
1801                 pr_warn("elf: failed to get legacy map definitions for %s\n",
1802                         obj->path);
1803                 return -EINVAL;
1804         }
1805
1806         /*
1807          * Count number of maps. Each map has a name.
1808          * Array of maps is not supported: only the first element is
1809          * considered.
1810          *
1811          * TODO: Detect array of map and report error.
1812          */
1813         nr_syms = symbols->d_size / sizeof(GElf_Sym);
1814         for (i = 0; i < nr_syms; i++) {
1815                 GElf_Sym sym;
1816
1817                 if (!gelf_getsym(symbols, i, &sym))
1818                         continue;
1819                 if (sym.st_shndx != obj->efile.maps_shndx)
1820                         continue;
1821                 nr_maps++;
1822         }
1823         /* Assume equally sized map definitions */
1824         pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n",
1825                  nr_maps, data->d_size, obj->path);
1826
1827         if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1828                 pr_warn("elf: unable to determine legacy map definition size in %s\n",
1829                         obj->path);
1830                 return -EINVAL;
1831         }
1832         map_def_sz = data->d_size / nr_maps;
1833
1834         /* Fill obj->maps using data in "maps" section.  */
1835         for (i = 0; i < nr_syms; i++) {
1836                 GElf_Sym sym;
1837                 const char *map_name;
1838                 struct bpf_map_def *def;
1839                 struct bpf_map *map;
1840
1841                 if (!gelf_getsym(symbols, i, &sym))
1842                         continue;
1843                 if (sym.st_shndx != obj->efile.maps_shndx)
1844                         continue;
1845
1846                 map = bpf_object__add_map(obj);
1847                 if (IS_ERR(map))
1848                         return PTR_ERR(map);
1849
1850                 map_name = elf_sym_str(obj, sym.st_name);
1851                 if (!map_name) {
1852                         pr_warn("failed to get map #%d name sym string for obj %s\n",
1853                                 i, obj->path);
1854                         return -LIBBPF_ERRNO__FORMAT;
1855                 }
1856
1857                 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1858                 map->sec_idx = sym.st_shndx;
1859                 map->sec_offset = sym.st_value;
1860                 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1861                          map_name, map->sec_idx, map->sec_offset);
1862                 if (sym.st_value + map_def_sz > data->d_size) {
1863                         pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1864                                 obj->path, map_name);
1865                         return -EINVAL;
1866                 }
1867
1868                 map->name = strdup(map_name);
1869                 if (!map->name) {
1870                         pr_warn("failed to alloc map name\n");
1871                         return -ENOMEM;
1872                 }
1873                 pr_debug("map %d is \"%s\"\n", i, map->name);
1874                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1875                 /*
1876                  * If the definition of the map in the object file fits in
1877                  * bpf_map_def, copy it.  Any extra fields in our version
1878                  * of bpf_map_def will default to zero as a result of the
1879                  * calloc above.
1880                  */
1881                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1882                         memcpy(&map->def, def, map_def_sz);
1883                 } else {
1884                         /*
1885                          * Here the map structure being read is bigger than what
1886                          * we expect, truncate if the excess bits are all zero.
1887                          * If they are not zero, reject this map as
1888                          * incompatible.
1889                          */
1890                         char *b;
1891
1892                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
1893                              b < ((char *)def) + map_def_sz; b++) {
1894                                 if (*b != 0) {
1895                                         pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1896                                                 obj->path, map_name);
1897                                         if (strict)
1898                                                 return -EINVAL;
1899                                 }
1900                         }
1901                         memcpy(&map->def, def, sizeof(struct bpf_map_def));
1902                 }
1903         }
1904         return 0;
1905 }
1906
1907 const struct btf_type *
1908 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1909 {
1910         const struct btf_type *t = btf__type_by_id(btf, id);
1911
1912         if (res_id)
1913                 *res_id = id;
1914
1915         while (btf_is_mod(t) || btf_is_typedef(t)) {
1916                 if (res_id)
1917                         *res_id = t->type;
1918                 t = btf__type_by_id(btf, t->type);
1919         }
1920
1921         return t;
1922 }
1923
1924 static const struct btf_type *
1925 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1926 {
1927         const struct btf_type *t;
1928
1929         t = skip_mods_and_typedefs(btf, id, NULL);
1930         if (!btf_is_ptr(t))
1931                 return NULL;
1932
1933         t = skip_mods_and_typedefs(btf, t->type, res_id);
1934
1935         return btf_is_func_proto(t) ? t : NULL;
1936 }
1937
1938 static const char *__btf_kind_str(__u16 kind)
1939 {
1940         switch (kind) {
1941         case BTF_KIND_UNKN: return "void";
1942         case BTF_KIND_INT: return "int";
1943         case BTF_KIND_PTR: return "ptr";
1944         case BTF_KIND_ARRAY: return "array";
1945         case BTF_KIND_STRUCT: return "struct";
1946         case BTF_KIND_UNION: return "union";
1947         case BTF_KIND_ENUM: return "enum";
1948         case BTF_KIND_FWD: return "fwd";
1949         case BTF_KIND_TYPEDEF: return "typedef";
1950         case BTF_KIND_VOLATILE: return "volatile";
1951         case BTF_KIND_CONST: return "const";
1952         case BTF_KIND_RESTRICT: return "restrict";
1953         case BTF_KIND_FUNC: return "func";
1954         case BTF_KIND_FUNC_PROTO: return "func_proto";
1955         case BTF_KIND_VAR: return "var";
1956         case BTF_KIND_DATASEC: return "datasec";
1957         case BTF_KIND_FLOAT: return "float";
1958         default: return "unknown";
1959         }
1960 }
1961
1962 const char *btf_kind_str(const struct btf_type *t)
1963 {
1964         return __btf_kind_str(btf_kind(t));
1965 }
1966
1967 /*
1968  * Fetch integer attribute of BTF map definition. Such attributes are
1969  * represented using a pointer to an array, in which dimensionality of array
1970  * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
1971  * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
1972  * type definition, while using only sizeof(void *) space in ELF data section.
1973  */
1974 static bool get_map_field_int(const char *map_name, const struct btf *btf,
1975                               const struct btf_member *m, __u32 *res)
1976 {
1977         const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1978         const char *name = btf__name_by_offset(btf, m->name_off);
1979         const struct btf_array *arr_info;
1980         const struct btf_type *arr_t;
1981
1982         if (!btf_is_ptr(t)) {
1983                 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
1984                         map_name, name, btf_kind_str(t));
1985                 return false;
1986         }
1987
1988         arr_t = btf__type_by_id(btf, t->type);
1989         if (!arr_t) {
1990                 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1991                         map_name, name, t->type);
1992                 return false;
1993         }
1994         if (!btf_is_array(arr_t)) {
1995                 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
1996                         map_name, name, btf_kind_str(arr_t));
1997                 return false;
1998         }
1999         arr_info = btf_array(arr_t);
2000         *res = arr_info->nelems;
2001         return true;
2002 }
2003
2004 static int build_map_pin_path(struct bpf_map *map, const char *path)
2005 {
2006         char buf[PATH_MAX];
2007         int len;
2008
2009         if (!path)
2010                 path = "/sys/fs/bpf";
2011
2012         len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
2013         if (len < 0)
2014                 return -EINVAL;
2015         else if (len >= PATH_MAX)
2016                 return -ENAMETOOLONG;
2017
2018         return bpf_map__set_pin_path(map, buf);
2019 }
2020
2021 int parse_btf_map_def(const char *map_name, struct btf *btf,
2022                       const struct btf_type *def_t, bool strict,
2023                       struct btf_map_def *map_def, struct btf_map_def *inner_def)
2024 {
2025         const struct btf_type *t;
2026         const struct btf_member *m;
2027         bool is_inner = inner_def == NULL;
2028         int vlen, i;
2029
2030         vlen = btf_vlen(def_t);
2031         m = btf_members(def_t);
2032         for (i = 0; i < vlen; i++, m++) {
2033                 const char *name = btf__name_by_offset(btf, m->name_off);
2034
2035                 if (!name) {
2036                         pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2037                         return -EINVAL;
2038                 }
2039                 if (strcmp(name, "type") == 0) {
2040                         if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2041                                 return -EINVAL;
2042                         map_def->parts |= MAP_DEF_MAP_TYPE;
2043                 } else if (strcmp(name, "max_entries") == 0) {
2044                         if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2045                                 return -EINVAL;
2046                         map_def->parts |= MAP_DEF_MAX_ENTRIES;
2047                 } else if (strcmp(name, "map_flags") == 0) {
2048                         if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2049                                 return -EINVAL;
2050                         map_def->parts |= MAP_DEF_MAP_FLAGS;
2051                 } else if (strcmp(name, "numa_node") == 0) {
2052                         if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2053                                 return -EINVAL;
2054                         map_def->parts |= MAP_DEF_NUMA_NODE;
2055                 } else if (strcmp(name, "key_size") == 0) {
2056                         __u32 sz;
2057
2058                         if (!get_map_field_int(map_name, btf, m, &sz))
2059                                 return -EINVAL;
2060                         if (map_def->key_size && map_def->key_size != sz) {
2061                                 pr_warn("map '%s': conflicting key size %u != %u.\n",
2062                                         map_name, map_def->key_size, sz);
2063                                 return -EINVAL;
2064                         }
2065                         map_def->key_size = sz;
2066                         map_def->parts |= MAP_DEF_KEY_SIZE;
2067                 } else if (strcmp(name, "key") == 0) {
2068                         __s64 sz;
2069
2070                         t = btf__type_by_id(btf, m->type);
2071                         if (!t) {
2072                                 pr_warn("map '%s': key type [%d] not found.\n",
2073                                         map_name, m->type);
2074                                 return -EINVAL;
2075                         }
2076                         if (!btf_is_ptr(t)) {
2077                                 pr_warn("map '%s': key spec is not PTR: %s.\n",
2078                                         map_name, btf_kind_str(t));
2079                                 return -EINVAL;
2080                         }
2081                         sz = btf__resolve_size(btf, t->type);
2082                         if (sz < 0) {
2083                                 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2084                                         map_name, t->type, (ssize_t)sz);
2085                                 return sz;
2086                         }
2087                         if (map_def->key_size && map_def->key_size != sz) {
2088                                 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2089                                         map_name, map_def->key_size, (ssize_t)sz);
2090                                 return -EINVAL;
2091                         }
2092                         map_def->key_size = sz;
2093                         map_def->key_type_id = t->type;
2094                         map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2095                 } else if (strcmp(name, "value_size") == 0) {
2096                         __u32 sz;
2097
2098                         if (!get_map_field_int(map_name, btf, m, &sz))
2099                                 return -EINVAL;
2100                         if (map_def->value_size && map_def->value_size != sz) {
2101                                 pr_warn("map '%s': conflicting value size %u != %u.\n",
2102                                         map_name, map_def->value_size, sz);
2103                                 return -EINVAL;
2104                         }
2105                         map_def->value_size = sz;
2106                         map_def->parts |= MAP_DEF_VALUE_SIZE;
2107                 } else if (strcmp(name, "value") == 0) {
2108                         __s64 sz;
2109
2110                         t = btf__type_by_id(btf, m->type);
2111                         if (!t) {
2112                                 pr_warn("map '%s': value type [%d] not found.\n",
2113                                         map_name, m->type);
2114                                 return -EINVAL;
2115                         }
2116                         if (!btf_is_ptr(t)) {
2117                                 pr_warn("map '%s': value spec is not PTR: %s.\n",
2118                                         map_name, btf_kind_str(t));
2119                                 return -EINVAL;
2120                         }
2121                         sz = btf__resolve_size(btf, t->type);
2122                         if (sz < 0) {
2123                                 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2124                                         map_name, t->type, (ssize_t)sz);
2125                                 return sz;
2126                         }
2127                         if (map_def->value_size && map_def->value_size != sz) {
2128                                 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2129                                         map_name, map_def->value_size, (ssize_t)sz);
2130                                 return -EINVAL;
2131                         }
2132                         map_def->value_size = sz;
2133                         map_def->value_type_id = t->type;
2134                         map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2135                 }
2136                 else if (strcmp(name, "values") == 0) {
2137                         char inner_map_name[128];
2138                         int err;
2139
2140                         if (is_inner) {
2141                                 pr_warn("map '%s': multi-level inner maps not supported.\n",
2142                                         map_name);
2143                                 return -ENOTSUP;
2144                         }
2145                         if (i != vlen - 1) {
2146                                 pr_warn("map '%s': '%s' member should be last.\n",
2147                                         map_name, name);
2148                                 return -EINVAL;
2149                         }
2150                         if (!bpf_map_type__is_map_in_map(map_def->map_type)) {
2151                                 pr_warn("map '%s': should be map-in-map.\n",
2152                                         map_name);
2153                                 return -ENOTSUP;
2154                         }
2155                         if (map_def->value_size && map_def->value_size != 4) {
2156                                 pr_warn("map '%s': conflicting value size %u != 4.\n",
2157                                         map_name, map_def->value_size);
2158                                 return -EINVAL;
2159                         }
2160                         map_def->value_size = 4;
2161                         t = btf__type_by_id(btf, m->type);
2162                         if (!t) {
2163                                 pr_warn("map '%s': map-in-map inner type [%d] not found.\n",
2164                                         map_name, m->type);
2165                                 return -EINVAL;
2166                         }
2167                         if (!btf_is_array(t) || btf_array(t)->nelems) {
2168                                 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",
2169                                         map_name);
2170                                 return -EINVAL;
2171                         }
2172                         t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2173                         if (!btf_is_ptr(t)) {
2174                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2175                                         map_name, btf_kind_str(t));
2176                                 return -EINVAL;
2177                         }
2178                         t = skip_mods_and_typedefs(btf, t->type, NULL);
2179                         if (!btf_is_struct(t)) {
2180                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2181                                         map_name, btf_kind_str(t));
2182                                 return -EINVAL;
2183                         }
2184
2185                         snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2186                         err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2187                         if (err)
2188                                 return err;
2189
2190                         map_def->parts |= MAP_DEF_INNER_MAP;
2191                 } else if (strcmp(name, "pinning") == 0) {
2192                         __u32 val;
2193
2194                         if (is_inner) {
2195                                 pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2196                                 return -EINVAL;
2197                         }
2198                         if (!get_map_field_int(map_name, btf, m, &val))
2199                                 return -EINVAL;
2200                         if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2201                                 pr_warn("map '%s': invalid pinning value %u.\n",
2202                                         map_name, val);
2203                                 return -EINVAL;
2204                         }
2205                         map_def->pinning = val;
2206                         map_def->parts |= MAP_DEF_PINNING;
2207                 } else {
2208                         if (strict) {
2209                                 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2210                                 return -ENOTSUP;
2211                         }
2212                         pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2213                 }
2214         }
2215
2216         if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2217                 pr_warn("map '%s': map type isn't specified.\n", map_name);
2218                 return -EINVAL;
2219         }
2220
2221         return 0;
2222 }
2223
2224 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2225 {
2226         map->def.type = def->map_type;
2227         map->def.key_size = def->key_size;
2228         map->def.value_size = def->value_size;
2229         map->def.max_entries = def->max_entries;
2230         map->def.map_flags = def->map_flags;
2231
2232         map->numa_node = def->numa_node;
2233         map->btf_key_type_id = def->key_type_id;
2234         map->btf_value_type_id = def->value_type_id;
2235
2236         if (def->parts & MAP_DEF_MAP_TYPE)
2237                 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2238
2239         if (def->parts & MAP_DEF_KEY_TYPE)
2240                 pr_debug("map '%s': found key [%u], sz = %u.\n",
2241                          map->name, def->key_type_id, def->key_size);
2242         else if (def->parts & MAP_DEF_KEY_SIZE)
2243                 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2244
2245         if (def->parts & MAP_DEF_VALUE_TYPE)
2246                 pr_debug("map '%s': found value [%u], sz = %u.\n",
2247                          map->name, def->value_type_id, def->value_size);
2248         else if (def->parts & MAP_DEF_VALUE_SIZE)
2249                 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2250
2251         if (def->parts & MAP_DEF_MAX_ENTRIES)
2252                 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2253         if (def->parts & MAP_DEF_MAP_FLAGS)
2254                 pr_debug("map '%s': found map_flags = %u.\n", map->name, def->map_flags);
2255         if (def->parts & MAP_DEF_PINNING)
2256                 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2257         if (def->parts & MAP_DEF_NUMA_NODE)
2258                 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2259
2260         if (def->parts & MAP_DEF_INNER_MAP)
2261                 pr_debug("map '%s': found inner map definition.\n", map->name);
2262 }
2263
2264 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2265                                          const struct btf_type *sec,
2266                                          int var_idx, int sec_idx,
2267                                          const Elf_Data *data, bool strict,
2268                                          const char *pin_root_path)
2269 {
2270         struct btf_map_def map_def = {}, inner_def = {};
2271         const struct btf_type *var, *def;
2272         const struct btf_var_secinfo *vi;
2273         const struct btf_var *var_extra;
2274         const char *map_name;
2275         struct bpf_map *map;
2276         int err;
2277
2278         vi = btf_var_secinfos(sec) + var_idx;
2279         var = btf__type_by_id(obj->btf, vi->type);
2280         var_extra = btf_var(var);
2281         map_name = btf__name_by_offset(obj->btf, var->name_off);
2282
2283         if (map_name == NULL || map_name[0] == '\0') {
2284                 pr_warn("map #%d: empty name.\n", var_idx);
2285                 return -EINVAL;
2286         }
2287         if ((__u64)vi->offset + vi->size > data->d_size) {
2288                 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2289                 return -EINVAL;
2290         }
2291         if (!btf_is_var(var)) {
2292                 pr_warn("map '%s': unexpected var kind %s.\n",
2293                         map_name, btf_kind_str(var));
2294                 return -EINVAL;
2295         }
2296         if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2297             var_extra->linkage != BTF_VAR_STATIC) {
2298                 pr_warn("map '%s': unsupported var linkage %u.\n",
2299                         map_name, var_extra->linkage);
2300                 return -EOPNOTSUPP;
2301         }
2302
2303         def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2304         if (!btf_is_struct(def)) {
2305                 pr_warn("map '%s': unexpected def kind %s.\n",
2306                         map_name, btf_kind_str(var));
2307                 return -EINVAL;
2308         }
2309         if (def->size > vi->size) {
2310                 pr_warn("map '%s': invalid def size.\n", map_name);
2311                 return -EINVAL;
2312         }
2313
2314         map = bpf_object__add_map(obj);
2315         if (IS_ERR(map))
2316                 return PTR_ERR(map);
2317         map->name = strdup(map_name);
2318         if (!map->name) {
2319                 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2320                 return -ENOMEM;
2321         }
2322         map->libbpf_type = LIBBPF_MAP_UNSPEC;
2323         map->def.type = BPF_MAP_TYPE_UNSPEC;
2324         map->sec_idx = sec_idx;
2325         map->sec_offset = vi->offset;
2326         map->btf_var_idx = var_idx;
2327         pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2328                  map_name, map->sec_idx, map->sec_offset);
2329
2330         err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2331         if (err)
2332                 return err;
2333
2334         fill_map_from_def(map, &map_def);
2335
2336         if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2337                 err = build_map_pin_path(map, pin_root_path);
2338                 if (err) {
2339                         pr_warn("map '%s': couldn't build pin path.\n", map->name);
2340                         return err;
2341                 }
2342         }
2343
2344         if (map_def.parts & MAP_DEF_INNER_MAP) {
2345                 map->inner_map = calloc(1, sizeof(*map->inner_map));
2346                 if (!map->inner_map)
2347                         return -ENOMEM;
2348                 map->inner_map->fd = -1;
2349                 map->inner_map->sec_idx = sec_idx;
2350                 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2351                 if (!map->inner_map->name)
2352                         return -ENOMEM;
2353                 sprintf(map->inner_map->name, "%s.inner", map_name);
2354
2355                 fill_map_from_def(map->inner_map, &inner_def);
2356         }
2357
2358         return 0;
2359 }
2360
2361 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2362                                           const char *pin_root_path)
2363 {
2364         const struct btf_type *sec = NULL;
2365         int nr_types, i, vlen, err;
2366         const struct btf_type *t;
2367         const char *name;
2368         Elf_Data *data;
2369         Elf_Scn *scn;
2370
2371         if (obj->efile.btf_maps_shndx < 0)
2372                 return 0;
2373
2374         scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2375         data = elf_sec_data(obj, scn);
2376         if (!scn || !data) {
2377                 pr_warn("elf: failed to get %s map definitions for %s\n",
2378                         MAPS_ELF_SEC, obj->path);
2379                 return -EINVAL;
2380         }
2381
2382         nr_types = btf__get_nr_types(obj->btf);
2383         for (i = 1; i <= nr_types; i++) {
2384                 t = btf__type_by_id(obj->btf, i);
2385                 if (!btf_is_datasec(t))
2386                         continue;
2387                 name = btf__name_by_offset(obj->btf, t->name_off);
2388                 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2389                         sec = t;
2390                         obj->efile.btf_maps_sec_btf_id = i;
2391                         break;
2392                 }
2393         }
2394
2395         if (!sec) {
2396                 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2397                 return -ENOENT;
2398         }
2399
2400         vlen = btf_vlen(sec);
2401         for (i = 0; i < vlen; i++) {
2402                 err = bpf_object__init_user_btf_map(obj, sec, i,
2403                                                     obj->efile.btf_maps_shndx,
2404                                                     data, strict,
2405                                                     pin_root_path);
2406                 if (err)
2407                         return err;
2408         }
2409
2410         return 0;
2411 }
2412
2413 static int bpf_object__init_maps(struct bpf_object *obj,
2414                                  const struct bpf_object_open_opts *opts)
2415 {
2416         const char *pin_root_path;
2417         bool strict;
2418         int err;
2419
2420         strict = !OPTS_GET(opts, relaxed_maps, false);
2421         pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2422
2423         err = bpf_object__init_user_maps(obj, strict);
2424         err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2425         err = err ?: bpf_object__init_global_data_maps(obj);
2426         err = err ?: bpf_object__init_kconfig_map(obj);
2427         err = err ?: bpf_object__init_struct_ops_maps(obj);
2428         if (err)
2429                 return err;
2430
2431         return 0;
2432 }
2433
2434 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2435 {
2436         GElf_Shdr sh;
2437
2438         if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh))
2439                 return false;
2440
2441         return sh.sh_flags & SHF_EXECINSTR;
2442 }
2443
2444 static bool btf_needs_sanitization(struct bpf_object *obj)
2445 {
2446         bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2447         bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2448         bool has_float = kernel_supports(FEAT_BTF_FLOAT);
2449         bool has_func = kernel_supports(FEAT_BTF_FUNC);
2450
2451         return !has_func || !has_datasec || !has_func_global || !has_float;
2452 }
2453
2454 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2455 {
2456         bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2457         bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2458         bool has_float = kernel_supports(FEAT_BTF_FLOAT);
2459         bool has_func = kernel_supports(FEAT_BTF_FUNC);
2460         struct btf_type *t;
2461         int i, j, vlen;
2462
2463         for (i = 1; i <= btf__get_nr_types(btf); i++) {
2464                 t = (struct btf_type *)btf__type_by_id(btf, i);
2465
2466                 if (!has_datasec && btf_is_var(t)) {
2467                         /* replace VAR with INT */
2468                         t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2469                         /*
2470                          * using size = 1 is the safest choice, 4 will be too
2471                          * big and cause kernel BTF validation failure if
2472                          * original variable took less than 4 bytes
2473                          */
2474                         t->size = 1;
2475                         *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2476                 } else if (!has_datasec && btf_is_datasec(t)) {
2477                         /* replace DATASEC with STRUCT */
2478                         const struct btf_var_secinfo *v = btf_var_secinfos(t);
2479                         struct btf_member *m = btf_members(t);
2480                         struct btf_type *vt;
2481                         char *name;
2482
2483                         name = (char *)btf__name_by_offset(btf, t->name_off);
2484                         while (*name) {
2485                                 if (*name == '.')
2486                                         *name = '_';
2487                                 name++;
2488                         }
2489
2490                         vlen = btf_vlen(t);
2491                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2492                         for (j = 0; j < vlen; j++, v++, m++) {
2493                                 /* order of field assignments is important */
2494                                 m->offset = v->offset * 8;
2495                                 m->type = v->type;
2496                                 /* preserve variable name as member name */
2497                                 vt = (void *)btf__type_by_id(btf, v->type);
2498                                 m->name_off = vt->name_off;
2499                         }
2500                 } else if (!has_func && btf_is_func_proto(t)) {
2501                         /* replace FUNC_PROTO with ENUM */
2502                         vlen = btf_vlen(t);
2503                         t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2504                         t->size = sizeof(__u32); /* kernel enforced */
2505                 } else if (!has_func && btf_is_func(t)) {
2506                         /* replace FUNC with TYPEDEF */
2507                         t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2508                 } else if (!has_func_global && btf_is_func(t)) {
2509                         /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2510                         t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2511                 } else if (!has_float && btf_is_float(t)) {
2512                         /* replace FLOAT with an equally-sized empty STRUCT;
2513                          * since C compilers do not accept e.g. "float" as a
2514                          * valid struct name, make it anonymous
2515                          */
2516                         t->name_off = 0;
2517                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2518                 }
2519         }
2520 }
2521
2522 static bool libbpf_needs_btf(const struct bpf_object *obj)
2523 {
2524         return obj->efile.btf_maps_shndx >= 0 ||
2525                obj->efile.st_ops_shndx >= 0 ||
2526                obj->nr_extern > 0;
2527 }
2528
2529 static bool kernel_needs_btf(const struct bpf_object *obj)
2530 {
2531         return obj->efile.st_ops_shndx >= 0;
2532 }
2533
2534 static int bpf_object__init_btf(struct bpf_object *obj,
2535                                 Elf_Data *btf_data,
2536                                 Elf_Data *btf_ext_data)
2537 {
2538         int err = -ENOENT;
2539
2540         if (btf_data) {
2541                 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2542                 if (IS_ERR(obj->btf)) {
2543                         err = PTR_ERR(obj->btf);
2544                         obj->btf = NULL;
2545                         pr_warn("Error loading ELF section %s: %d.\n",
2546                                 BTF_ELF_SEC, err);
2547                         goto out;
2548                 }
2549                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2550                 btf__set_pointer_size(obj->btf, 8);
2551                 err = 0;
2552         }
2553         if (btf_ext_data) {
2554                 if (!obj->btf) {
2555                         pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2556                                  BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2557                         goto out;
2558                 }
2559                 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
2560                                             btf_ext_data->d_size);
2561                 if (IS_ERR(obj->btf_ext)) {
2562                         pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
2563                                 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
2564                         obj->btf_ext = NULL;
2565                         goto out;
2566                 }
2567         }
2568 out:
2569         if (err && libbpf_needs_btf(obj)) {
2570                 pr_warn("BTF is required, but is missing or corrupted.\n");
2571                 return err;
2572         }
2573         return 0;
2574 }
2575
2576 static int bpf_object__finalize_btf(struct bpf_object *obj)
2577 {
2578         int err;
2579
2580         if (!obj->btf)
2581                 return 0;
2582
2583         err = btf__finalize_data(obj, obj->btf);
2584         if (err) {
2585                 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2586                 return err;
2587         }
2588
2589         return 0;
2590 }
2591
2592 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
2593 {
2594         if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2595             prog->type == BPF_PROG_TYPE_LSM)
2596                 return true;
2597
2598         /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2599          * also need vmlinux BTF
2600          */
2601         if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2602                 return true;
2603
2604         return false;
2605 }
2606
2607 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
2608 {
2609         struct bpf_program *prog;
2610         int i;
2611
2612         /* CO-RE relocations need kernel BTF */
2613         if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
2614                 return true;
2615
2616         /* Support for typed ksyms needs kernel BTF */
2617         for (i = 0; i < obj->nr_extern; i++) {
2618                 const struct extern_desc *ext;
2619
2620                 ext = &obj->externs[i];
2621                 if (ext->type == EXT_KSYM && ext->ksym.type_id)
2622                         return true;
2623         }
2624
2625         bpf_object__for_each_program(prog, obj) {
2626                 if (!prog->load)
2627                         continue;
2628                 if (prog_needs_vmlinux_btf(prog))
2629                         return true;
2630         }
2631
2632         return false;
2633 }
2634
2635 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
2636 {
2637         int err;
2638
2639         /* btf_vmlinux could be loaded earlier */
2640         if (obj->btf_vmlinux)
2641                 return 0;
2642
2643         if (!force && !obj_needs_vmlinux_btf(obj))
2644                 return 0;
2645
2646         obj->btf_vmlinux = libbpf_find_kernel_btf();
2647         if (IS_ERR(obj->btf_vmlinux)) {
2648                 err = PTR_ERR(obj->btf_vmlinux);
2649                 pr_warn("Error loading vmlinux BTF: %d\n", err);
2650                 obj->btf_vmlinux = NULL;
2651                 return err;
2652         }
2653         return 0;
2654 }
2655
2656 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2657 {
2658         struct btf *kern_btf = obj->btf;
2659         bool btf_mandatory, sanitize;
2660         int i, err = 0;
2661
2662         if (!obj->btf)
2663                 return 0;
2664
2665         if (!kernel_supports(FEAT_BTF)) {
2666                 if (kernel_needs_btf(obj)) {
2667                         err = -EOPNOTSUPP;
2668                         goto report;
2669                 }
2670                 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
2671                 return 0;
2672         }
2673
2674         /* Even though some subprogs are global/weak, user might prefer more
2675          * permissive BPF verification process that BPF verifier performs for
2676          * static functions, taking into account more context from the caller
2677          * functions. In such case, they need to mark such subprogs with
2678          * __attribute__((visibility("hidden"))) and libbpf will adjust
2679          * corresponding FUNC BTF type to be marked as static and trigger more
2680          * involved BPF verification process.
2681          */
2682         for (i = 0; i < obj->nr_programs; i++) {
2683                 struct bpf_program *prog = &obj->programs[i];
2684                 struct btf_type *t;
2685                 const char *name;
2686                 int j, n;
2687
2688                 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
2689                         continue;
2690
2691                 n = btf__get_nr_types(obj->btf);
2692                 for (j = 1; j <= n; j++) {
2693                         t = btf_type_by_id(obj->btf, j);
2694                         if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
2695                                 continue;
2696
2697                         name = btf__str_by_offset(obj->btf, t->name_off);
2698                         if (strcmp(name, prog->name) != 0)
2699                                 continue;
2700
2701                         t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
2702                         break;
2703                 }
2704         }
2705
2706         sanitize = btf_needs_sanitization(obj);
2707         if (sanitize) {
2708                 const void *raw_data;
2709                 __u32 sz;
2710
2711                 /* clone BTF to sanitize a copy and leave the original intact */
2712                 raw_data = btf__get_raw_data(obj->btf, &sz);
2713                 kern_btf = btf__new(raw_data, sz);
2714                 if (IS_ERR(kern_btf))
2715                         return PTR_ERR(kern_btf);
2716
2717                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2718                 btf__set_pointer_size(obj->btf, 8);
2719                 bpf_object__sanitize_btf(obj, kern_btf);
2720         }
2721
2722         err = btf__load(kern_btf);
2723         if (sanitize) {
2724                 if (!err) {
2725                         /* move fd to libbpf's BTF */
2726                         btf__set_fd(obj->btf, btf__fd(kern_btf));
2727                         btf__set_fd(kern_btf, -1);
2728                 }
2729                 btf__free(kern_btf);
2730         }
2731 report:
2732         if (err) {
2733                 btf_mandatory = kernel_needs_btf(obj);
2734                 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
2735                         btf_mandatory ? "BTF is mandatory, can't proceed."
2736                                       : "BTF is optional, ignoring.");
2737                 if (!btf_mandatory)
2738                         err = 0;
2739         }
2740         return err;
2741 }
2742
2743 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
2744 {
2745         const char *name;
2746
2747         name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
2748         if (!name) {
2749                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2750                         off, obj->path, elf_errmsg(-1));
2751                 return NULL;
2752         }
2753
2754         return name;
2755 }
2756
2757 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
2758 {
2759         const char *name;
2760
2761         name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
2762         if (!name) {
2763                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2764                         off, obj->path, elf_errmsg(-1));
2765                 return NULL;
2766         }
2767
2768         return name;
2769 }
2770
2771 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
2772 {
2773         Elf_Scn *scn;
2774
2775         scn = elf_getscn(obj->efile.elf, idx);
2776         if (!scn) {
2777                 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
2778                         idx, obj->path, elf_errmsg(-1));
2779                 return NULL;
2780         }
2781         return scn;
2782 }
2783
2784 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
2785 {
2786         Elf_Scn *scn = NULL;
2787         Elf *elf = obj->efile.elf;
2788         const char *sec_name;
2789
2790         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2791                 sec_name = elf_sec_name(obj, scn);
2792                 if (!sec_name)
2793                         return NULL;
2794
2795                 if (strcmp(sec_name, name) != 0)
2796                         continue;
2797
2798                 return scn;
2799         }
2800         return NULL;
2801 }
2802
2803 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr)
2804 {
2805         if (!scn)
2806                 return -EINVAL;
2807
2808         if (gelf_getshdr(scn, hdr) != hdr) {
2809                 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
2810                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2811                 return -EINVAL;
2812         }
2813
2814         return 0;
2815 }
2816
2817 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
2818 {
2819         const char *name;
2820         GElf_Shdr sh;
2821
2822         if (!scn)
2823                 return NULL;
2824
2825         if (elf_sec_hdr(obj, scn, &sh))
2826                 return NULL;
2827
2828         name = elf_sec_str(obj, sh.sh_name);
2829         if (!name) {
2830                 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
2831                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2832                 return NULL;
2833         }
2834
2835         return name;
2836 }
2837
2838 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
2839 {
2840         Elf_Data *data;
2841
2842         if (!scn)
2843                 return NULL;
2844
2845         data = elf_getdata(scn, 0);
2846         if (!data) {
2847                 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
2848                         elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
2849                         obj->path, elf_errmsg(-1));
2850                 return NULL;
2851         }
2852
2853         return data;
2854 }
2855
2856 static bool is_sec_name_dwarf(const char *name)
2857 {
2858         /* approximation, but the actual list is too long */
2859         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
2860 }
2861
2862 static bool ignore_elf_section(GElf_Shdr *hdr, const char *name)
2863 {
2864         /* no special handling of .strtab */
2865         if (hdr->sh_type == SHT_STRTAB)
2866                 return true;
2867
2868         /* ignore .llvm_addrsig section as well */
2869         if (hdr->sh_type == SHT_LLVM_ADDRSIG)
2870                 return true;
2871
2872         /* no subprograms will lead to an empty .text section, ignore it */
2873         if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
2874             strcmp(name, ".text") == 0)
2875                 return true;
2876
2877         /* DWARF sections */
2878         if (is_sec_name_dwarf(name))
2879                 return true;
2880
2881         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
2882                 name += sizeof(".rel") - 1;
2883                 /* DWARF section relocations */
2884                 if (is_sec_name_dwarf(name))
2885                         return true;
2886
2887                 /* .BTF and .BTF.ext don't need relocations */
2888                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
2889                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
2890                         return true;
2891         }
2892
2893         return false;
2894 }
2895
2896 static int cmp_progs(const void *_a, const void *_b)
2897 {
2898         const struct bpf_program *a = _a;
2899         const struct bpf_program *b = _b;
2900
2901         if (a->sec_idx != b->sec_idx)
2902                 return a->sec_idx < b->sec_idx ? -1 : 1;
2903
2904         /* sec_insn_off can't be the same within the section */
2905         return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
2906 }
2907
2908 static int bpf_object__elf_collect(struct bpf_object *obj)
2909 {
2910         Elf *elf = obj->efile.elf;
2911         Elf_Data *btf_ext_data = NULL;
2912         Elf_Data *btf_data = NULL;
2913         int idx = 0, err = 0;
2914         const char *name;
2915         Elf_Data *data;
2916         Elf_Scn *scn;
2917         GElf_Shdr sh;
2918
2919         /* a bunch of ELF parsing functionality depends on processing symbols,
2920          * so do the first pass and find the symbol table
2921          */
2922         scn = NULL;
2923         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2924                 if (elf_sec_hdr(obj, scn, &sh))
2925                         return -LIBBPF_ERRNO__FORMAT;
2926
2927                 if (sh.sh_type == SHT_SYMTAB) {
2928                         if (obj->efile.symbols) {
2929                                 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
2930                                 return -LIBBPF_ERRNO__FORMAT;
2931                         }
2932
2933                         data = elf_sec_data(obj, scn);
2934                         if (!data)
2935                                 return -LIBBPF_ERRNO__FORMAT;
2936
2937                         obj->efile.symbols = data;
2938                         obj->efile.symbols_shndx = elf_ndxscn(scn);
2939                         obj->efile.strtabidx = sh.sh_link;
2940                 }
2941         }
2942
2943         scn = NULL;
2944         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2945                 idx++;
2946
2947                 if (elf_sec_hdr(obj, scn, &sh))
2948                         return -LIBBPF_ERRNO__FORMAT;
2949
2950                 name = elf_sec_str(obj, sh.sh_name);
2951                 if (!name)
2952                         return -LIBBPF_ERRNO__FORMAT;
2953
2954                 if (ignore_elf_section(&sh, name))
2955                         continue;
2956
2957                 data = elf_sec_data(obj, scn);
2958                 if (!data)
2959                         return -LIBBPF_ERRNO__FORMAT;
2960
2961                 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
2962                          idx, name, (unsigned long)data->d_size,
2963                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
2964                          (int)sh.sh_type);
2965
2966                 if (strcmp(name, "license") == 0) {
2967                         err = bpf_object__init_license(obj, data->d_buf, data->d_size);
2968                         if (err)
2969                                 return err;
2970                 } else if (strcmp(name, "version") == 0) {
2971                         err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
2972                         if (err)
2973                                 return err;
2974                 } else if (strcmp(name, "maps") == 0) {
2975                         obj->efile.maps_shndx = idx;
2976                 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
2977                         obj->efile.btf_maps_shndx = idx;
2978                 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
2979                         btf_data = data;
2980                 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
2981                         btf_ext_data = data;
2982                 } else if (sh.sh_type == SHT_SYMTAB) {
2983                         /* already processed during the first pass above */
2984                 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
2985                         if (sh.sh_flags & SHF_EXECINSTR) {
2986                                 if (strcmp(name, ".text") == 0)
2987                                         obj->efile.text_shndx = idx;
2988                                 err = bpf_object__add_programs(obj, data, name, idx);
2989                                 if (err)
2990                                         return err;
2991                         } else if (strcmp(name, DATA_SEC) == 0) {
2992                                 obj->efile.data = data;
2993                                 obj->efile.data_shndx = idx;
2994                         } else if (strcmp(name, RODATA_SEC) == 0) {
2995                                 obj->efile.rodata = data;
2996                                 obj->efile.rodata_shndx = idx;
2997                         } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
2998                                 obj->efile.st_ops_data = data;
2999                                 obj->efile.st_ops_shndx = idx;
3000                         } else {
3001                                 pr_info("elf: skipping unrecognized data section(%d) %s\n",
3002                                         idx, name);
3003                         }
3004                 } else if (sh.sh_type == SHT_REL) {
3005                         int nr_sects = obj->efile.nr_reloc_sects;
3006                         void *sects = obj->efile.reloc_sects;
3007                         int sec = sh.sh_info; /* points to other section */
3008
3009                         /* Only do relo for section with exec instructions */
3010                         if (!section_have_execinstr(obj, sec) &&
3011                             strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3012                             strcmp(name, ".rel" MAPS_ELF_SEC)) {
3013                                 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3014                                         idx, name, sec,
3015                                         elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>");
3016                                 continue;
3017                         }
3018
3019                         sects = libbpf_reallocarray(sects, nr_sects + 1,
3020                                                     sizeof(*obj->efile.reloc_sects));
3021                         if (!sects)
3022                                 return -ENOMEM;
3023
3024                         obj->efile.reloc_sects = sects;
3025                         obj->efile.nr_reloc_sects++;
3026
3027                         obj->efile.reloc_sects[nr_sects].shdr = sh;
3028                         obj->efile.reloc_sects[nr_sects].data = data;
3029                 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
3030                         obj->efile.bss = data;
3031                         obj->efile.bss_shndx = idx;
3032                 } else {
3033                         pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3034                                 (size_t)sh.sh_size);
3035                 }
3036         }
3037
3038         if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3039                 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3040                 return -LIBBPF_ERRNO__FORMAT;
3041         }
3042
3043         /* sort BPF programs by section name and in-section instruction offset
3044          * for faster search */
3045         qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3046
3047         return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3048 }
3049
3050 static bool sym_is_extern(const GElf_Sym *sym)
3051 {
3052         int bind = GELF_ST_BIND(sym->st_info);
3053         /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
3054         return sym->st_shndx == SHN_UNDEF &&
3055                (bind == STB_GLOBAL || bind == STB_WEAK) &&
3056                GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
3057 }
3058
3059 static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx)
3060 {
3061         int bind = GELF_ST_BIND(sym->st_info);
3062         int type = GELF_ST_TYPE(sym->st_info);
3063
3064         /* in .text section */
3065         if (sym->st_shndx != text_shndx)
3066                 return false;
3067
3068         /* local function */
3069         if (bind == STB_LOCAL && type == STT_SECTION)
3070                 return true;
3071
3072         /* global function */
3073         return bind == STB_GLOBAL && type == STT_FUNC;
3074 }
3075
3076 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3077 {
3078         const struct btf_type *t;
3079         const char *tname;
3080         int i, n;
3081
3082         if (!btf)
3083                 return -ESRCH;
3084
3085         n = btf__get_nr_types(btf);
3086         for (i = 1; i <= n; i++) {
3087                 t = btf__type_by_id(btf, i);
3088
3089                 if (!btf_is_var(t) && !btf_is_func(t))
3090                         continue;
3091
3092                 tname = btf__name_by_offset(btf, t->name_off);
3093                 if (strcmp(tname, ext_name))
3094                         continue;
3095
3096                 if (btf_is_var(t) &&
3097                     btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3098                         return -EINVAL;
3099
3100                 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3101                         return -EINVAL;
3102
3103                 return i;
3104         }
3105
3106         return -ENOENT;
3107 }
3108
3109 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3110         const struct btf_var_secinfo *vs;
3111         const struct btf_type *t;
3112         int i, j, n;
3113
3114         if (!btf)
3115                 return -ESRCH;
3116
3117         n = btf__get_nr_types(btf);
3118         for (i = 1; i <= n; i++) {
3119                 t = btf__type_by_id(btf, i);
3120
3121                 if (!btf_is_datasec(t))
3122                         continue;
3123
3124                 vs = btf_var_secinfos(t);
3125                 for (j = 0; j < btf_vlen(t); j++, vs++) {
3126                         if (vs->type == ext_btf_id)
3127                                 return i;
3128                 }
3129         }
3130
3131         return -ENOENT;
3132 }
3133
3134 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3135                                      bool *is_signed)
3136 {
3137         const struct btf_type *t;
3138         const char *name;
3139
3140         t = skip_mods_and_typedefs(btf, id, NULL);
3141         name = btf__name_by_offset(btf, t->name_off);
3142
3143         if (is_signed)
3144                 *is_signed = false;
3145         switch (btf_kind(t)) {
3146         case BTF_KIND_INT: {
3147                 int enc = btf_int_encoding(t);
3148
3149                 if (enc & BTF_INT_BOOL)
3150                         return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3151                 if (is_signed)
3152                         *is_signed = enc & BTF_INT_SIGNED;
3153                 if (t->size == 1)
3154                         return KCFG_CHAR;
3155                 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3156                         return KCFG_UNKNOWN;
3157                 return KCFG_INT;
3158         }
3159         case BTF_KIND_ENUM:
3160                 if (t->size != 4)
3161                         return KCFG_UNKNOWN;
3162                 if (strcmp(name, "libbpf_tristate"))
3163                         return KCFG_UNKNOWN;
3164                 return KCFG_TRISTATE;
3165         case BTF_KIND_ARRAY:
3166                 if (btf_array(t)->nelems == 0)
3167                         return KCFG_UNKNOWN;
3168                 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3169                         return KCFG_UNKNOWN;
3170                 return KCFG_CHAR_ARR;
3171         default:
3172                 return KCFG_UNKNOWN;
3173         }
3174 }
3175
3176 static int cmp_externs(const void *_a, const void *_b)
3177 {
3178         const struct extern_desc *a = _a;
3179         const struct extern_desc *b = _b;
3180
3181         if (a->type != b->type)
3182                 return a->type < b->type ? -1 : 1;
3183
3184         if (a->type == EXT_KCFG) {
3185                 /* descending order by alignment requirements */
3186                 if (a->kcfg.align != b->kcfg.align)
3187                         return a->kcfg.align > b->kcfg.align ? -1 : 1;
3188                 /* ascending order by size, within same alignment class */
3189                 if (a->kcfg.sz != b->kcfg.sz)
3190                         return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3191         }
3192
3193         /* resolve ties by name */
3194         return strcmp(a->name, b->name);
3195 }
3196
3197 static int find_int_btf_id(const struct btf *btf)
3198 {
3199         const struct btf_type *t;
3200         int i, n;
3201
3202         n = btf__get_nr_types(btf);
3203         for (i = 1; i <= n; i++) {
3204                 t = btf__type_by_id(btf, i);
3205
3206                 if (btf_is_int(t) && btf_int_bits(t) == 32)
3207                         return i;
3208         }
3209
3210         return 0;
3211 }
3212
3213 static int add_dummy_ksym_var(struct btf *btf)
3214 {
3215         int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
3216         const struct btf_var_secinfo *vs;
3217         const struct btf_type *sec;
3218
3219         sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
3220                                             BTF_KIND_DATASEC);
3221         if (sec_btf_id < 0)
3222                 return 0;
3223
3224         sec = btf__type_by_id(btf, sec_btf_id);
3225         vs = btf_var_secinfos(sec);
3226         for (i = 0; i < btf_vlen(sec); i++, vs++) {
3227                 const struct btf_type *vt;
3228
3229                 vt = btf__type_by_id(btf, vs->type);
3230                 if (btf_is_func(vt))
3231                         break;
3232         }
3233
3234         /* No func in ksyms sec.  No need to add dummy var. */
3235         if (i == btf_vlen(sec))
3236                 return 0;
3237
3238         int_btf_id = find_int_btf_id(btf);
3239         dummy_var_btf_id = btf__add_var(btf,
3240                                         "dummy_ksym",
3241                                         BTF_VAR_GLOBAL_ALLOCATED,
3242                                         int_btf_id);
3243         if (dummy_var_btf_id < 0)
3244                 pr_warn("cannot create a dummy_ksym var\n");
3245
3246         return dummy_var_btf_id;
3247 }
3248
3249 static int bpf_object__collect_externs(struct bpf_object *obj)
3250 {
3251         struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3252         const struct btf_type *t;
3253         struct extern_desc *ext;
3254         int i, n, off, dummy_var_btf_id;
3255         const char *ext_name, *sec_name;
3256         Elf_Scn *scn;
3257         GElf_Shdr sh;
3258
3259         if (!obj->efile.symbols)
3260                 return 0;
3261
3262         scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3263         if (elf_sec_hdr(obj, scn, &sh))
3264                 return -LIBBPF_ERRNO__FORMAT;
3265
3266         dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3267         if (dummy_var_btf_id < 0)
3268                 return dummy_var_btf_id;
3269
3270         n = sh.sh_size / sh.sh_entsize;
3271         pr_debug("looking for externs among %d symbols...\n", n);
3272
3273         for (i = 0; i < n; i++) {
3274                 GElf_Sym sym;
3275
3276                 if (!gelf_getsym(obj->efile.symbols, i, &sym))
3277                         return -LIBBPF_ERRNO__FORMAT;
3278                 if (!sym_is_extern(&sym))
3279                         continue;
3280                 ext_name = elf_sym_str(obj, sym.st_name);
3281                 if (!ext_name || !ext_name[0])
3282                         continue;
3283
3284                 ext = obj->externs;
3285                 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3286                 if (!ext)
3287                         return -ENOMEM;
3288                 obj->externs = ext;
3289                 ext = &ext[obj->nr_extern];
3290                 memset(ext, 0, sizeof(*ext));
3291                 obj->nr_extern++;
3292
3293                 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3294                 if (ext->btf_id <= 0) {
3295                         pr_warn("failed to find BTF for extern '%s': %d\n",
3296                                 ext_name, ext->btf_id);
3297                         return ext->btf_id;
3298                 }
3299                 t = btf__type_by_id(obj->btf, ext->btf_id);
3300                 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3301                 ext->sym_idx = i;
3302                 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
3303
3304                 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3305                 if (ext->sec_btf_id <= 0) {
3306                         pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3307                                 ext_name, ext->btf_id, ext->sec_btf_id);
3308                         return ext->sec_btf_id;
3309                 }
3310                 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3311                 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3312
3313                 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3314                         if (btf_is_func(t)) {
3315                                 pr_warn("extern function %s is unsupported under %s section\n",
3316                                         ext->name, KCONFIG_SEC);
3317                                 return -ENOTSUP;
3318                         }
3319                         kcfg_sec = sec;
3320                         ext->type = EXT_KCFG;
3321                         ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3322                         if (ext->kcfg.sz <= 0) {
3323                                 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3324                                         ext_name, ext->kcfg.sz);
3325                                 return ext->kcfg.sz;
3326                         }
3327                         ext->kcfg.align = btf__align_of(obj->btf, t->type);
3328                         if (ext->kcfg.align <= 0) {
3329                                 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3330                                         ext_name, ext->kcfg.align);
3331                                 return -EINVAL;
3332                         }
3333                         ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3334                                                         &ext->kcfg.is_signed);
3335                         if (ext->kcfg.type == KCFG_UNKNOWN) {
3336                                 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name);
3337                                 return -ENOTSUP;
3338                         }
3339                 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3340                         if (btf_is_func(t) && ext->is_weak) {
3341                                 pr_warn("extern weak function %s is unsupported\n",
3342                                         ext->name);
3343                                 return -ENOTSUP;
3344                         }
3345                         ksym_sec = sec;
3346                         ext->type = EXT_KSYM;
3347                         skip_mods_and_typedefs(obj->btf, t->type,
3348                                                &ext->ksym.type_id);
3349                 } else {
3350                         pr_warn("unrecognized extern section '%s'\n", sec_name);
3351                         return -ENOTSUP;
3352                 }
3353         }
3354         pr_debug("collected %d externs total\n", obj->nr_extern);
3355
3356         if (!obj->nr_extern)
3357                 return 0;
3358
3359         /* sort externs by type, for kcfg ones also by (align, size, name) */
3360         qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3361
3362         /* for .ksyms section, we need to turn all externs into allocated
3363          * variables in BTF to pass kernel verification; we do this by
3364          * pretending that each extern is a 8-byte variable
3365          */
3366         if (ksym_sec) {
3367                 /* find existing 4-byte integer type in BTF to use for fake
3368                  * extern variables in DATASEC
3369                  */
3370                 int int_btf_id = find_int_btf_id(obj->btf);
3371                 /* For extern function, a dummy_var added earlier
3372                  * will be used to replace the vs->type and
3373                  * its name string will be used to refill
3374                  * the missing param's name.
3375                  */
3376                 const struct btf_type *dummy_var;
3377
3378                 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3379                 for (i = 0; i < obj->nr_extern; i++) {
3380                         ext = &obj->externs[i];
3381                         if (ext->type != EXT_KSYM)
3382                                 continue;
3383                         pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3384                                  i, ext->sym_idx, ext->name);
3385                 }
3386
3387                 sec = ksym_sec;
3388                 n = btf_vlen(sec);
3389                 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3390                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3391                         struct btf_type *vt;
3392
3393                         vt = (void *)btf__type_by_id(obj->btf, vs->type);
3394                         ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3395                         ext = find_extern_by_name(obj, ext_name);
3396                         if (!ext) {
3397                                 pr_warn("failed to find extern definition for BTF %s '%s'\n",
3398                                         btf_kind_str(vt), ext_name);
3399                                 return -ESRCH;
3400                         }
3401                         if (btf_is_func(vt)) {
3402                                 const struct btf_type *func_proto;
3403                                 struct btf_param *param;
3404                                 int j;
3405
3406                                 func_proto = btf__type_by_id(obj->btf,
3407                                                              vt->type);
3408                                 param = btf_params(func_proto);
3409                                 /* Reuse the dummy_var string if the
3410                                  * func proto does not have param name.
3411                                  */
3412                                 for (j = 0; j < btf_vlen(func_proto); j++)
3413                                         if (param[j].type && !param[j].name_off)
3414                                                 param[j].name_off =
3415                                                         dummy_var->name_off;
3416                                 vs->type = dummy_var_btf_id;
3417                                 vt->info &= ~0xffff;
3418                                 vt->info |= BTF_FUNC_GLOBAL;
3419                         } else {
3420                                 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3421                                 vt->type = int_btf_id;
3422                         }
3423                         vs->offset = off;
3424                         vs->size = sizeof(int);
3425                 }
3426                 sec->size = off;
3427         }
3428
3429         if (kcfg_sec) {
3430                 sec = kcfg_sec;
3431                 /* for kcfg externs calculate their offsets within a .kconfig map */
3432                 off = 0;
3433                 for (i = 0; i < obj->nr_extern; i++) {
3434                         ext = &obj->externs[i];
3435                         if (ext->type != EXT_KCFG)
3436                                 continue;
3437
3438                         ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3439                         off = ext->kcfg.data_off + ext->kcfg.sz;
3440                         pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3441                                  i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3442                 }
3443                 sec->size = off;
3444                 n = btf_vlen(sec);
3445                 for (i = 0; i < n; i++) {
3446                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3447
3448                         t = btf__type_by_id(obj->btf, vs->type);
3449                         ext_name = btf__name_by_offset(obj->btf, t->name_off);
3450                         ext = find_extern_by_name(obj, ext_name);
3451                         if (!ext) {
3452                                 pr_warn("failed to find extern definition for BTF var '%s'\n",
3453                                         ext_name);
3454                                 return -ESRCH;
3455                         }
3456                         btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3457                         vs->offset = ext->kcfg.data_off;
3458                 }
3459         }
3460         return 0;
3461 }
3462
3463 struct bpf_program *
3464 bpf_object__find_program_by_title(const struct bpf_object *obj,
3465                                   const char *title)
3466 {
3467         struct bpf_program *pos;
3468
3469         bpf_object__for_each_program(pos, obj) {
3470                 if (pos->sec_name && !strcmp(pos->sec_name, title))
3471                         return pos;
3472         }
3473         return NULL;
3474 }
3475
3476 static bool prog_is_subprog(const struct bpf_object *obj,
3477                             const struct bpf_program *prog)
3478 {
3479         /* For legacy reasons, libbpf supports an entry-point BPF programs
3480          * without SEC() attribute, i.e., those in the .text section. But if
3481          * there are 2 or more such programs in the .text section, they all
3482          * must be subprograms called from entry-point BPF programs in
3483          * designated SEC()'tions, otherwise there is no way to distinguish
3484          * which of those programs should be loaded vs which are a subprogram.
3485          * Similarly, if there is a function/program in .text and at least one
3486          * other BPF program with custom SEC() attribute, then we just assume
3487          * .text programs are subprograms (even if they are not called from
3488          * other programs), because libbpf never explicitly supported mixing
3489          * SEC()-designated BPF programs and .text entry-point BPF programs.
3490          */
3491         return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3492 }
3493
3494 struct bpf_program *
3495 bpf_object__find_program_by_name(const struct bpf_object *obj,
3496                                  const char *name)
3497 {
3498         struct bpf_program *prog;
3499
3500         bpf_object__for_each_program(prog, obj) {
3501                 if (prog_is_subprog(obj, prog))
3502                         continue;
3503                 if (!strcmp(prog->name, name))
3504                         return prog;
3505         }
3506         return NULL;
3507 }
3508
3509 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3510                                       int shndx)
3511 {
3512         return shndx == obj->efile.data_shndx ||
3513                shndx == obj->efile.bss_shndx ||
3514                shndx == obj->efile.rodata_shndx;
3515 }
3516
3517 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3518                                       int shndx)
3519 {
3520         return shndx == obj->efile.maps_shndx ||
3521                shndx == obj->efile.btf_maps_shndx;
3522 }
3523
3524 static enum libbpf_map_type
3525 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3526 {
3527         if (shndx == obj->efile.data_shndx)
3528                 return LIBBPF_MAP_DATA;
3529         else if (shndx == obj->efile.bss_shndx)
3530                 return LIBBPF_MAP_BSS;
3531         else if (shndx == obj->efile.rodata_shndx)
3532                 return LIBBPF_MAP_RODATA;
3533         else if (shndx == obj->efile.symbols_shndx)
3534                 return LIBBPF_MAP_KCONFIG;
3535         else
3536                 return LIBBPF_MAP_UNSPEC;
3537 }
3538
3539 static int bpf_program__record_reloc(struct bpf_program *prog,
3540                                      struct reloc_desc *reloc_desc,
3541                                      __u32 insn_idx, const char *sym_name,
3542                                      const GElf_Sym *sym, const GElf_Rel *rel)
3543 {
3544         struct bpf_insn *insn = &prog->insns[insn_idx];
3545         size_t map_idx, nr_maps = prog->obj->nr_maps;
3546         struct bpf_object *obj = prog->obj;
3547         __u32 shdr_idx = sym->st_shndx;
3548         enum libbpf_map_type type;
3549         const char *sym_sec_name;
3550         struct bpf_map *map;
3551
3552         if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
3553                 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3554                         prog->name, sym_name, insn_idx, insn->code);
3555                 return -LIBBPF_ERRNO__RELOC;
3556         }
3557
3558         if (sym_is_extern(sym)) {
3559                 int sym_idx = GELF_R_SYM(rel->r_info);
3560                 int i, n = obj->nr_extern;
3561                 struct extern_desc *ext;
3562
3563                 for (i = 0; i < n; i++) {
3564                         ext = &obj->externs[i];
3565                         if (ext->sym_idx == sym_idx)
3566                                 break;
3567                 }
3568                 if (i >= n) {
3569                         pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3570                                 prog->name, sym_name, sym_idx);
3571                         return -LIBBPF_ERRNO__RELOC;
3572                 }
3573                 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3574                          prog->name, i, ext->name, ext->sym_idx, insn_idx);
3575                 if (insn->code == (BPF_JMP | BPF_CALL))
3576                         reloc_desc->type = RELO_EXTERN_FUNC;
3577                 else
3578                         reloc_desc->type = RELO_EXTERN_VAR;
3579                 reloc_desc->insn_idx = insn_idx;
3580                 reloc_desc->sym_off = i; /* sym_off stores extern index */
3581                 return 0;
3582         }
3583
3584         /* sub-program call relocation */
3585         if (is_call_insn(insn)) {
3586                 if (insn->src_reg != BPF_PSEUDO_CALL) {
3587                         pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3588                         return -LIBBPF_ERRNO__RELOC;
3589                 }
3590                 /* text_shndx can be 0, if no default "main" program exists */
3591                 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3592                         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3593                         pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3594                                 prog->name, sym_name, sym_sec_name);
3595                         return -LIBBPF_ERRNO__RELOC;
3596                 }
3597                 if (sym->st_value % BPF_INSN_SZ) {
3598                         pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3599                                 prog->name, sym_name, (size_t)sym->st_value);
3600                         return -LIBBPF_ERRNO__RELOC;
3601                 }
3602                 reloc_desc->type = RELO_CALL;
3603                 reloc_desc->insn_idx = insn_idx;
3604                 reloc_desc->sym_off = sym->st_value;
3605                 return 0;
3606         }
3607
3608         if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
3609                 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
3610                         prog->name, sym_name, shdr_idx);
3611                 return -LIBBPF_ERRNO__RELOC;
3612         }
3613
3614         /* loading subprog addresses */
3615         if (sym_is_subprog(sym, obj->efile.text_shndx)) {
3616                 /* global_func: sym->st_value = offset in the section, insn->imm = 0.
3617                  * local_func: sym->st_value = 0, insn->imm = offset in the section.
3618                  */
3619                 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
3620                         pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
3621                                 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
3622                         return -LIBBPF_ERRNO__RELOC;
3623                 }
3624
3625                 reloc_desc->type = RELO_SUBPROG_ADDR;
3626                 reloc_desc->insn_idx = insn_idx;
3627                 reloc_desc->sym_off = sym->st_value;
3628                 return 0;
3629         }
3630
3631         type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
3632         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3633
3634         /* generic map reference relocation */
3635         if (type == LIBBPF_MAP_UNSPEC) {
3636                 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
3637                         pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
3638                                 prog->name, sym_name, sym_sec_name);
3639                         return -LIBBPF_ERRNO__RELOC;
3640                 }
3641                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3642                         map = &obj->maps[map_idx];
3643                         if (map->libbpf_type != type ||
3644                             map->sec_idx != sym->st_shndx ||
3645                             map->sec_offset != sym->st_value)
3646                                 continue;
3647                         pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
3648                                  prog->name, map_idx, map->name, map->sec_idx,
3649                                  map->sec_offset, insn_idx);
3650                         break;
3651                 }
3652                 if (map_idx >= nr_maps) {
3653                         pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
3654                                 prog->name, sym_sec_name, (size_t)sym->st_value);
3655                         return -LIBBPF_ERRNO__RELOC;
3656                 }
3657                 reloc_desc->type = RELO_LD64;
3658                 reloc_desc->insn_idx = insn_idx;
3659                 reloc_desc->map_idx = map_idx;
3660                 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
3661                 return 0;
3662         }
3663
3664         /* global data map relocation */
3665         if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
3666                 pr_warn("prog '%s': bad data relo against section '%s'\n",
3667                         prog->name, sym_sec_name);
3668                 return -LIBBPF_ERRNO__RELOC;
3669         }
3670         for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3671                 map = &obj->maps[map_idx];
3672                 if (map->libbpf_type != type)
3673                         continue;
3674                 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
3675                          prog->name, map_idx, map->name, map->sec_idx,
3676                          map->sec_offset, insn_idx);
3677                 break;
3678         }
3679         if (map_idx >= nr_maps) {
3680                 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
3681                         prog->name, sym_sec_name);
3682                 return -LIBBPF_ERRNO__RELOC;
3683         }
3684
3685         reloc_desc->type = RELO_DATA;
3686         reloc_desc->insn_idx = insn_idx;
3687         reloc_desc->map_idx = map_idx;
3688         reloc_desc->sym_off = sym->st_value;
3689         return 0;
3690 }
3691
3692 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
3693 {
3694         return insn_idx >= prog->sec_insn_off &&
3695                insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
3696 }
3697
3698 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
3699                                                  size_t sec_idx, size_t insn_idx)
3700 {
3701         int l = 0, r = obj->nr_programs - 1, m;
3702         struct bpf_program *prog;
3703
3704         while (l < r) {
3705                 m = l + (r - l + 1) / 2;
3706                 prog = &obj->programs[m];
3707
3708                 if (prog->sec_idx < sec_idx ||
3709                     (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
3710                         l = m;
3711                 else
3712                         r = m - 1;
3713         }
3714         /* matching program could be at index l, but it still might be the
3715          * wrong one, so we need to double check conditions for the last time
3716          */
3717         prog = &obj->programs[l];
3718         if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
3719                 return prog;
3720         return NULL;
3721 }
3722
3723 static int
3724 bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data)
3725 {
3726         Elf_Data *symbols = obj->efile.symbols;
3727         const char *relo_sec_name, *sec_name;
3728         size_t sec_idx = shdr->sh_info;
3729         struct bpf_program *prog;
3730         struct reloc_desc *relos;
3731         int err, i, nrels;
3732         const char *sym_name;
3733         __u32 insn_idx;
3734         Elf_Scn *scn;
3735         Elf_Data *scn_data;
3736         GElf_Sym sym;
3737         GElf_Rel rel;
3738
3739         scn = elf_sec_by_idx(obj, sec_idx);
3740         scn_data = elf_sec_data(obj, scn);
3741
3742         relo_sec_name = elf_sec_str(obj, shdr->sh_name);
3743         sec_name = elf_sec_name(obj, scn);
3744         if (!relo_sec_name || !sec_name)
3745                 return -EINVAL;
3746
3747         pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
3748                  relo_sec_name, sec_idx, sec_name);
3749         nrels = shdr->sh_size / shdr->sh_entsize;
3750
3751         for (i = 0; i < nrels; i++) {
3752                 if (!gelf_getrel(data, i, &rel)) {
3753                         pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
3754                         return -LIBBPF_ERRNO__FORMAT;
3755                 }
3756                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3757                         pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n",
3758                                 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3759                         return -LIBBPF_ERRNO__FORMAT;
3760                 }
3761
3762                 if (rel.r_offset % BPF_INSN_SZ || rel.r_offset >= scn_data->d_size) {
3763                         pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
3764                                 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3765                         return -LIBBPF_ERRNO__FORMAT;
3766                 }
3767
3768                 insn_idx = rel.r_offset / BPF_INSN_SZ;
3769                 /* relocations against static functions are recorded as
3770                  * relocations against the section that contains a function;
3771                  * in such case, symbol will be STT_SECTION and sym.st_name
3772                  * will point to empty string (0), so fetch section name
3773                  * instead
3774                  */
3775                 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0)
3776                         sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx));
3777                 else
3778                         sym_name = elf_sym_str(obj, sym.st_name);
3779                 sym_name = sym_name ?: "<?";
3780
3781                 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
3782                          relo_sec_name, i, insn_idx, sym_name);
3783
3784                 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
3785                 if (!prog) {
3786                         pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
3787                                 relo_sec_name, i, sec_name, insn_idx);
3788                         continue;
3789                 }
3790
3791                 relos = libbpf_reallocarray(prog->reloc_desc,
3792                                             prog->nr_reloc + 1, sizeof(*relos));
3793                 if (!relos)
3794                         return -ENOMEM;
3795                 prog->reloc_desc = relos;
3796
3797                 /* adjust insn_idx to local BPF program frame of reference */
3798                 insn_idx -= prog->sec_insn_off;
3799                 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
3800                                                 insn_idx, sym_name, &sym, &rel);
3801                 if (err)
3802                         return err;
3803
3804                 prog->nr_reloc++;
3805         }
3806         return 0;
3807 }
3808
3809 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3810 {
3811         struct bpf_map_def *def = &map->def;
3812         __u32 key_type_id = 0, value_type_id = 0;
3813         int ret;
3814
3815         /* if it's BTF-defined map, we don't need to search for type IDs.
3816          * For struct_ops map, it does not need btf_key_type_id and
3817          * btf_value_type_id.
3818          */
3819         if (map->sec_idx == obj->efile.btf_maps_shndx ||
3820             bpf_map__is_struct_ops(map))
3821                 return 0;
3822
3823         if (!bpf_map__is_internal(map)) {
3824                 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3825                                            def->value_size, &key_type_id,
3826                                            &value_type_id);
3827         } else {
3828                 /*
3829                  * LLVM annotates global data differently in BTF, that is,
3830                  * only as '.data', '.bss' or '.rodata'.
3831                  */
3832                 ret = btf__find_by_name(obj->btf,
3833                                 libbpf_type_to_btf_name[map->libbpf_type]);
3834         }
3835         if (ret < 0)
3836                 return ret;
3837
3838         map->btf_key_type_id = key_type_id;
3839         map->btf_value_type_id = bpf_map__is_internal(map) ?
3840                                  ret : value_type_id;
3841         return 0;
3842 }
3843
3844 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3845 {
3846         struct bpf_map_info info = {};
3847         __u32 len = sizeof(info);
3848         int new_fd, err;
3849         char *new_name;
3850
3851         err = bpf_obj_get_info_by_fd(fd, &info, &len);
3852         if (err)
3853                 return err;
3854
3855         new_name = strdup(info.name);
3856         if (!new_name)
3857                 return -errno;
3858
3859         new_fd = open("/", O_RDONLY | O_CLOEXEC);
3860         if (new_fd < 0) {
3861                 err = -errno;
3862                 goto err_free_new_name;
3863         }
3864
3865         new_fd = dup3(fd, new_fd, O_CLOEXEC);
3866         if (new_fd < 0) {
3867                 err = -errno;
3868                 goto err_close_new_fd;
3869         }
3870
3871         err = zclose(map->fd);
3872         if (err) {
3873                 err = -errno;
3874                 goto err_close_new_fd;
3875         }
3876         free(map->name);
3877
3878         map->fd = new_fd;
3879         map->name = new_name;
3880         map->def.type = info.type;
3881         map->def.key_size = info.key_size;
3882         map->def.value_size = info.value_size;
3883         map->def.max_entries = info.max_entries;
3884         map->def.map_flags = info.map_flags;
3885         map->btf_key_type_id = info.btf_key_type_id;
3886         map->btf_value_type_id = info.btf_value_type_id;
3887         map->reused = true;
3888
3889         return 0;
3890
3891 err_close_new_fd:
3892         close(new_fd);
3893 err_free_new_name:
3894         free(new_name);
3895         return err;
3896 }
3897
3898 __u32 bpf_map__max_entries(const struct bpf_map *map)
3899 {
3900         return map->def.max_entries;
3901 }
3902
3903 struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
3904 {
3905         if (!bpf_map_type__is_map_in_map(map->def.type))
3906                 return NULL;
3907
3908         return map->inner_map;
3909 }
3910
3911 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
3912 {
3913         if (map->fd >= 0)
3914                 return -EBUSY;
3915         map->def.max_entries = max_entries;
3916         return 0;
3917 }
3918
3919 int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
3920 {
3921         if (!map || !max_entries)
3922                 return -EINVAL;
3923
3924         return bpf_map__set_max_entries(map, max_entries);
3925 }
3926
3927 static int
3928 bpf_object__probe_loading(struct bpf_object *obj)
3929 {
3930         struct bpf_load_program_attr attr;
3931         char *cp, errmsg[STRERR_BUFSIZE];
3932         struct bpf_insn insns[] = {
3933                 BPF_MOV64_IMM(BPF_REG_0, 0),
3934                 BPF_EXIT_INSN(),
3935         };
3936         int ret;
3937
3938         /* make sure basic loading works */
3939
3940         memset(&attr, 0, sizeof(attr));
3941         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3942         attr.insns = insns;
3943         attr.insns_cnt = ARRAY_SIZE(insns);
3944         attr.license = "GPL";
3945
3946         ret = bpf_load_program_xattr(&attr, NULL, 0);
3947         if (ret < 0) {
3948                 ret = errno;
3949                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3950                 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
3951                         "program. Make sure your kernel supports BPF "
3952                         "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
3953                         "set to big enough value.\n", __func__, cp, ret);
3954                 return -ret;
3955         }
3956         close(ret);
3957
3958         return 0;
3959 }
3960
3961 static int probe_fd(int fd)
3962 {
3963         if (fd >= 0)
3964                 close(fd);
3965         return fd >= 0;
3966 }
3967
3968 static int probe_kern_prog_name(void)
3969 {
3970         struct bpf_load_program_attr attr;
3971         struct bpf_insn insns[] = {
3972                 BPF_MOV64_IMM(BPF_REG_0, 0),
3973                 BPF_EXIT_INSN(),
3974         };
3975         int ret;
3976
3977         /* make sure loading with name works */
3978
3979         memset(&attr, 0, sizeof(attr));
3980         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3981         attr.insns = insns;
3982         attr.insns_cnt = ARRAY_SIZE(insns);
3983         attr.license = "GPL";
3984         attr.name = "test";
3985         ret = bpf_load_program_xattr(&attr, NULL, 0);
3986         return probe_fd(ret);
3987 }
3988
3989 static int probe_kern_global_data(void)
3990 {
3991         struct bpf_load_program_attr prg_attr;
3992         struct bpf_create_map_attr map_attr;
3993         char *cp, errmsg[STRERR_BUFSIZE];
3994         struct bpf_insn insns[] = {
3995                 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
3996                 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
3997                 BPF_MOV64_IMM(BPF_REG_0, 0),
3998                 BPF_EXIT_INSN(),
3999         };
4000         int ret, map;
4001
4002         memset(&map_attr, 0, sizeof(map_attr));
4003         map_attr.map_type = BPF_MAP_TYPE_ARRAY;
4004         map_attr.key_size = sizeof(int);
4005         map_attr.value_size = 32;
4006         map_attr.max_entries = 1;
4007
4008         map = bpf_create_map_xattr(&map_attr);
4009         if (map < 0) {
4010                 ret = -errno;
4011                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4012                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4013                         __func__, cp, -ret);
4014                 return ret;
4015         }
4016
4017         insns[0].imm = map;
4018
4019         memset(&prg_attr, 0, sizeof(prg_attr));
4020         prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4021         prg_attr.insns = insns;
4022         prg_attr.insns_cnt = ARRAY_SIZE(insns);
4023         prg_attr.license = "GPL";
4024
4025         ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
4026         close(map);
4027         return probe_fd(ret);
4028 }
4029
4030 static int probe_kern_btf(void)
4031 {
4032         static const char strs[] = "\0int";
4033         __u32 types[] = {
4034                 /* int */
4035                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4036         };
4037
4038         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4039                                              strs, sizeof(strs)));
4040 }
4041
4042 static int probe_kern_btf_func(void)
4043 {
4044         static const char strs[] = "\0int\0x\0a";
4045         /* void x(int a) {} */
4046         __u32 types[] = {
4047                 /* int */
4048                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4049                 /* FUNC_PROTO */                                /* [2] */
4050                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4051                 BTF_PARAM_ENC(7, 1),
4052                 /* FUNC x */                                    /* [3] */
4053                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
4054         };
4055
4056         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4057                                              strs, sizeof(strs)));
4058 }
4059
4060 static int probe_kern_btf_func_global(void)
4061 {
4062         static const char strs[] = "\0int\0x\0a";
4063         /* static void x(int a) {} */
4064         __u32 types[] = {
4065                 /* int */
4066                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4067                 /* FUNC_PROTO */                                /* [2] */
4068                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4069                 BTF_PARAM_ENC(7, 1),
4070                 /* FUNC x BTF_FUNC_GLOBAL */                    /* [3] */
4071                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
4072         };
4073
4074         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4075                                              strs, sizeof(strs)));
4076 }
4077
4078 static int probe_kern_btf_datasec(void)
4079 {
4080         static const char strs[] = "\0x\0.data";
4081         /* static int a; */
4082         __u32 types[] = {
4083                 /* int */
4084                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4085                 /* VAR x */                                     /* [2] */
4086                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4087                 BTF_VAR_STATIC,
4088                 /* DATASEC val */                               /* [3] */
4089                 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
4090                 BTF_VAR_SECINFO_ENC(2, 0, 4),
4091         };
4092
4093         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4094                                              strs, sizeof(strs)));
4095 }
4096
4097 static int probe_kern_btf_float(void)
4098 {
4099         static const char strs[] = "\0float";
4100         __u32 types[] = {
4101                 /* float */
4102                 BTF_TYPE_FLOAT_ENC(1, 4),
4103         };
4104
4105         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4106                                              strs, sizeof(strs)));
4107 }
4108
4109 static int probe_kern_array_mmap(void)
4110 {
4111         struct bpf_create_map_attr attr = {
4112                 .map_type = BPF_MAP_TYPE_ARRAY,
4113                 .map_flags = BPF_F_MMAPABLE,
4114                 .key_size = sizeof(int),
4115                 .value_size = sizeof(int),
4116                 .max_entries = 1,
4117         };
4118
4119         return probe_fd(bpf_create_map_xattr(&attr));
4120 }
4121
4122 static int probe_kern_exp_attach_type(void)
4123 {
4124         struct bpf_load_program_attr attr;
4125         struct bpf_insn insns[] = {
4126                 BPF_MOV64_IMM(BPF_REG_0, 0),
4127                 BPF_EXIT_INSN(),
4128         };
4129
4130         memset(&attr, 0, sizeof(attr));
4131         /* use any valid combination of program type and (optional)
4132          * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
4133          * to see if kernel supports expected_attach_type field for
4134          * BPF_PROG_LOAD command
4135          */
4136         attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK;
4137         attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE;
4138         attr.insns = insns;
4139         attr.insns_cnt = ARRAY_SIZE(insns);
4140         attr.license = "GPL";
4141
4142         return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
4143 }
4144
4145 static int probe_kern_probe_read_kernel(void)
4146 {
4147         struct bpf_load_program_attr attr;
4148         struct bpf_insn insns[] = {
4149                 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),   /* r1 = r10 (fp) */
4150                 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),  /* r1 += -8 */
4151                 BPF_MOV64_IMM(BPF_REG_2, 8),            /* r2 = 8 */
4152                 BPF_MOV64_IMM(BPF_REG_3, 0),            /* r3 = 0 */
4153                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
4154                 BPF_EXIT_INSN(),
4155         };
4156
4157         memset(&attr, 0, sizeof(attr));
4158         attr.prog_type = BPF_PROG_TYPE_KPROBE;
4159         attr.insns = insns;
4160         attr.insns_cnt = ARRAY_SIZE(insns);
4161         attr.license = "GPL";
4162
4163         return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
4164 }
4165
4166 static int probe_prog_bind_map(void)
4167 {
4168         struct bpf_load_program_attr prg_attr;
4169         struct bpf_create_map_attr map_attr;
4170         char *cp, errmsg[STRERR_BUFSIZE];
4171         struct bpf_insn insns[] = {
4172                 BPF_MOV64_IMM(BPF_REG_0, 0),
4173                 BPF_EXIT_INSN(),
4174         };
4175         int ret, map, prog;
4176
4177         memset(&map_attr, 0, sizeof(map_attr));
4178         map_attr.map_type = BPF_MAP_TYPE_ARRAY;
4179         map_attr.key_size = sizeof(int);
4180         map_attr.value_size = 32;
4181         map_attr.max_entries = 1;
4182
4183         map = bpf_create_map_xattr(&map_attr);
4184         if (map < 0) {
4185                 ret = -errno;
4186                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4187                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4188                         __func__, cp, -ret);
4189                 return ret;
4190         }
4191
4192         memset(&prg_attr, 0, sizeof(prg_attr));
4193         prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4194         prg_attr.insns = insns;
4195         prg_attr.insns_cnt = ARRAY_SIZE(insns);
4196         prg_attr.license = "GPL";
4197
4198         prog = bpf_load_program_xattr(&prg_attr, NULL, 0);
4199         if (prog < 0) {
4200                 close(map);
4201                 return 0;
4202         }
4203
4204         ret = bpf_prog_bind_map(prog, map, NULL);
4205
4206         close(map);
4207         close(prog);
4208
4209         return ret >= 0;
4210 }
4211
4212 static int probe_module_btf(void)
4213 {
4214         static const char strs[] = "\0int";
4215         __u32 types[] = {
4216                 /* int */
4217                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4218         };
4219         struct bpf_btf_info info;
4220         __u32 len = sizeof(info);
4221         char name[16];
4222         int fd, err;
4223
4224         fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4225         if (fd < 0)
4226                 return 0; /* BTF not supported at all */
4227
4228         memset(&info, 0, sizeof(info));
4229         info.name = ptr_to_u64(name);
4230         info.name_len = sizeof(name);
4231
4232         /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4233          * kernel's module BTF support coincides with support for
4234          * name/name_len fields in struct bpf_btf_info.
4235          */
4236         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4237         close(fd);
4238         return !err;
4239 }
4240
4241 enum kern_feature_result {
4242         FEAT_UNKNOWN = 0,
4243         FEAT_SUPPORTED = 1,
4244         FEAT_MISSING = 2,
4245 };
4246
4247 typedef int (*feature_probe_fn)(void);
4248
4249 static struct kern_feature_desc {
4250         const char *desc;
4251         feature_probe_fn probe;
4252         enum kern_feature_result res;
4253 } feature_probes[__FEAT_CNT] = {
4254         [FEAT_PROG_NAME] = {
4255                 "BPF program name", probe_kern_prog_name,
4256         },
4257         [FEAT_GLOBAL_DATA] = {
4258                 "global variables", probe_kern_global_data,
4259         },
4260         [FEAT_BTF] = {
4261                 "minimal BTF", probe_kern_btf,
4262         },
4263         [FEAT_BTF_FUNC] = {
4264                 "BTF functions", probe_kern_btf_func,
4265         },
4266         [FEAT_BTF_GLOBAL_FUNC] = {
4267                 "BTF global function", probe_kern_btf_func_global,
4268         },
4269         [FEAT_BTF_DATASEC] = {
4270                 "BTF data section and variable", probe_kern_btf_datasec,
4271         },
4272         [FEAT_ARRAY_MMAP] = {
4273                 "ARRAY map mmap()", probe_kern_array_mmap,
4274         },
4275         [FEAT_EXP_ATTACH_TYPE] = {
4276                 "BPF_PROG_LOAD expected_attach_type attribute",
4277                 probe_kern_exp_attach_type,
4278         },
4279         [FEAT_PROBE_READ_KERN] = {
4280                 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4281         },
4282         [FEAT_PROG_BIND_MAP] = {
4283                 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4284         },
4285         [FEAT_MODULE_BTF] = {
4286                 "module BTF support", probe_module_btf,
4287         },
4288         [FEAT_BTF_FLOAT] = {
4289                 "BTF_KIND_FLOAT support", probe_kern_btf_float,
4290         },
4291 };
4292
4293 static bool kernel_supports(enum kern_feature_id feat_id)
4294 {
4295         struct kern_feature_desc *feat = &feature_probes[feat_id];
4296         int ret;
4297
4298         if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4299                 ret = feat->probe();
4300                 if (ret > 0) {
4301                         WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4302                 } else if (ret == 0) {
4303                         WRITE_ONCE(feat->res, FEAT_MISSING);
4304                 } else {
4305                         pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4306                         WRITE_ONCE(feat->res, FEAT_MISSING);
4307                 }
4308         }
4309
4310         return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4311 }
4312
4313 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4314 {
4315         struct bpf_map_info map_info = {};
4316         char msg[STRERR_BUFSIZE];
4317         __u32 map_info_len;
4318
4319         map_info_len = sizeof(map_info);
4320
4321         if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
4322                 pr_warn("failed to get map info for map FD %d: %s\n",
4323                         map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
4324                 return false;
4325         }
4326
4327         return (map_info.type == map->def.type &&
4328                 map_info.key_size == map->def.key_size &&
4329                 map_info.value_size == map->def.value_size &&
4330                 map_info.max_entries == map->def.max_entries &&
4331                 map_info.map_flags == map->def.map_flags);
4332 }
4333
4334 static int
4335 bpf_object__reuse_map(struct bpf_map *map)
4336 {
4337         char *cp, errmsg[STRERR_BUFSIZE];
4338         int err, pin_fd;
4339
4340         pin_fd = bpf_obj_get(map->pin_path);
4341         if (pin_fd < 0) {
4342                 err = -errno;
4343                 if (err == -ENOENT) {
4344                         pr_debug("found no pinned map to reuse at '%s'\n",
4345                                  map->pin_path);
4346                         return 0;
4347                 }
4348
4349                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4350                 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4351                         map->pin_path, cp);
4352                 return err;
4353         }
4354
4355         if (!map_is_reuse_compat(map, pin_fd)) {
4356                 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4357                         map->pin_path);
4358                 close(pin_fd);
4359                 return -EINVAL;
4360         }
4361
4362         err = bpf_map__reuse_fd(map, pin_fd);
4363         if (err) {
4364                 close(pin_fd);
4365                 return err;
4366         }
4367         map->pinned = true;
4368         pr_debug("reused pinned map at '%s'\n", map->pin_path);
4369
4370         return 0;
4371 }
4372
4373 static int
4374 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4375 {
4376         enum libbpf_map_type map_type = map->libbpf_type;
4377         char *cp, errmsg[STRERR_BUFSIZE];
4378         int err, zero = 0;
4379
4380         err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4381         if (err) {
4382                 err = -errno;
4383                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4384                 pr_warn("Error setting initial map(%s) contents: %s\n",
4385                         map->name, cp);
4386                 return err;
4387         }
4388
4389         /* Freeze .rodata and .kconfig map as read-only from syscall side. */
4390         if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4391                 err = bpf_map_freeze(map->fd);
4392                 if (err) {
4393                         err = -errno;
4394                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4395                         pr_warn("Error freezing map(%s) as read-only: %s\n",
4396                                 map->name, cp);
4397                         return err;
4398                 }
4399         }
4400         return 0;
4401 }
4402
4403 static void bpf_map__destroy(struct bpf_map *map);
4404
4405 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map)
4406 {
4407         struct bpf_create_map_attr create_attr;
4408         struct bpf_map_def *def = &map->def;
4409
4410         memset(&create_attr, 0, sizeof(create_attr));
4411
4412         if (kernel_supports(FEAT_PROG_NAME))
4413                 create_attr.name = map->name;
4414         create_attr.map_ifindex = map->map_ifindex;
4415         create_attr.map_type = def->type;
4416         create_attr.map_flags = def->map_flags;
4417         create_attr.key_size = def->key_size;
4418         create_attr.value_size = def->value_size;
4419         create_attr.numa_node = map->numa_node;
4420
4421         if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) {
4422                 int nr_cpus;
4423
4424                 nr_cpus = libbpf_num_possible_cpus();
4425                 if (nr_cpus < 0) {
4426                         pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
4427                                 map->name, nr_cpus);
4428                         return nr_cpus;
4429                 }
4430                 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
4431                 create_attr.max_entries = nr_cpus;
4432         } else {
4433                 create_attr.max_entries = def->max_entries;
4434         }
4435
4436         if (bpf_map__is_struct_ops(map))
4437                 create_attr.btf_vmlinux_value_type_id =
4438                         map->btf_vmlinux_value_type_id;
4439
4440         create_attr.btf_fd = 0;
4441         create_attr.btf_key_type_id = 0;
4442         create_attr.btf_value_type_id = 0;
4443         if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) {
4444                 create_attr.btf_fd = btf__fd(obj->btf);
4445                 create_attr.btf_key_type_id = map->btf_key_type_id;
4446                 create_attr.btf_value_type_id = map->btf_value_type_id;
4447         }
4448
4449         if (bpf_map_type__is_map_in_map(def->type)) {
4450                 if (map->inner_map) {
4451                         int err;
4452
4453                         err = bpf_object__create_map(obj, map->inner_map);
4454                         if (err) {
4455                                 pr_warn("map '%s': failed to create inner map: %d\n",
4456                                         map->name, err);
4457                                 return err;
4458                         }
4459                         map->inner_map_fd = bpf_map__fd(map->inner_map);
4460                 }
4461                 if (map->inner_map_fd >= 0)
4462                         create_attr.inner_map_fd = map->inner_map_fd;
4463         }
4464
4465         map->fd = bpf_create_map_xattr(&create_attr);
4466         if (map->fd < 0 && (create_attr.btf_key_type_id ||
4467                             create_attr.btf_value_type_id)) {
4468                 char *cp, errmsg[STRERR_BUFSIZE];
4469                 int err = -errno;
4470
4471                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4472                 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
4473                         map->name, cp, err);
4474                 create_attr.btf_fd = 0;
4475                 create_attr.btf_key_type_id = 0;
4476                 create_attr.btf_value_type_id = 0;
4477                 map->btf_key_type_id = 0;
4478                 map->btf_value_type_id = 0;
4479                 map->fd = bpf_create_map_xattr(&create_attr);
4480         }
4481
4482         if (map->fd < 0)
4483                 return -errno;
4484
4485         if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
4486                 bpf_map__destroy(map->inner_map);
4487                 zfree(&map->inner_map);
4488         }
4489
4490         return 0;
4491 }
4492
4493 static int init_map_slots(struct bpf_map *map)
4494 {
4495         const struct bpf_map *targ_map;
4496         unsigned int i;
4497         int fd, err;
4498
4499         for (i = 0; i < map->init_slots_sz; i++) {
4500                 if (!map->init_slots[i])
4501                         continue;
4502
4503                 targ_map = map->init_slots[i];
4504                 fd = bpf_map__fd(targ_map);
4505                 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
4506                 if (err) {
4507                         err = -errno;
4508                         pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
4509                                 map->name, i, targ_map->name,
4510                                 fd, err);
4511                         return err;
4512                 }
4513                 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
4514                          map->name, i, targ_map->name, fd);
4515         }
4516
4517         zfree(&map->init_slots);
4518         map->init_slots_sz = 0;
4519
4520         return 0;
4521 }
4522
4523 static int
4524 bpf_object__create_maps(struct bpf_object *obj)
4525 {
4526         struct bpf_map *map;
4527         char *cp, errmsg[STRERR_BUFSIZE];
4528         unsigned int i, j;
4529         int err;
4530
4531         for (i = 0; i < obj->nr_maps; i++) {
4532                 map = &obj->maps[i];
4533
4534                 if (map->pin_path) {
4535                         err = bpf_object__reuse_map(map);
4536                         if (err) {
4537                                 pr_warn("map '%s': error reusing pinned map\n",
4538                                         map->name);
4539                                 goto err_out;
4540                         }
4541                 }
4542
4543                 if (map->fd >= 0) {
4544                         pr_debug("map '%s': skipping creation (preset fd=%d)\n",
4545                                  map->name, map->fd);
4546                 } else {
4547                         err = bpf_object__create_map(obj, map);
4548                         if (err)
4549                                 goto err_out;
4550
4551                         pr_debug("map '%s': created successfully, fd=%d\n",
4552                                  map->name, map->fd);
4553
4554                         if (bpf_map__is_internal(map)) {
4555                                 err = bpf_object__populate_internal_map(obj, map);
4556                                 if (err < 0) {
4557                                         zclose(map->fd);
4558                                         goto err_out;
4559                                 }
4560                         }
4561
4562                         if (map->init_slots_sz) {
4563                                 err = init_map_slots(map);
4564                                 if (err < 0) {
4565                                         zclose(map->fd);
4566                                         goto err_out;
4567                                 }
4568                         }
4569                 }
4570
4571                 if (map->pin_path && !map->pinned) {
4572                         err = bpf_map__pin(map, NULL);
4573                         if (err) {
4574                                 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
4575                                         map->name, map->pin_path, err);
4576                                 zclose(map->fd);
4577                                 goto err_out;
4578                         }
4579                 }
4580         }
4581
4582         return 0;
4583
4584 err_out:
4585         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4586         pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
4587         pr_perm_msg(err);
4588         for (j = 0; j < i; j++)
4589                 zclose(obj->maps[j].fd);
4590         return err;
4591 }
4592
4593 #define BPF_CORE_SPEC_MAX_LEN 64
4594
4595 /* represents BPF CO-RE field or array element accessor */
4596 struct bpf_core_accessor {
4597         __u32 type_id;          /* struct/union type or array element type */
4598         __u32 idx;              /* field index or array index */
4599         const char *name;       /* field name or NULL for array accessor */
4600 };
4601
4602 struct bpf_core_spec {
4603         const struct btf *btf;
4604         /* high-level spec: named fields and array indices only */
4605         struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
4606         /* original unresolved (no skip_mods_or_typedefs) root type ID */
4607         __u32 root_type_id;
4608         /* CO-RE relocation kind */
4609         enum bpf_core_relo_kind relo_kind;
4610         /* high-level spec length */
4611         int len;
4612         /* raw, low-level spec: 1-to-1 with accessor spec string */
4613         int raw_spec[BPF_CORE_SPEC_MAX_LEN];
4614         /* raw spec length */
4615         int raw_len;
4616         /* field bit offset represented by spec */
4617         __u32 bit_offset;
4618 };
4619
4620 static bool str_is_empty(const char *s)
4621 {
4622         return !s || !s[0];
4623 }
4624
4625 static bool is_flex_arr(const struct btf *btf,
4626                         const struct bpf_core_accessor *acc,
4627                         const struct btf_array *arr)
4628 {
4629         const struct btf_type *t;
4630
4631         /* not a flexible array, if not inside a struct or has non-zero size */
4632         if (!acc->name || arr->nelems > 0)
4633                 return false;
4634
4635         /* has to be the last member of enclosing struct */
4636         t = btf__type_by_id(btf, acc->type_id);
4637         return acc->idx == btf_vlen(t) - 1;
4638 }
4639
4640 static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
4641 {
4642         switch (kind) {
4643         case BPF_FIELD_BYTE_OFFSET: return "byte_off";
4644         case BPF_FIELD_BYTE_SIZE: return "byte_sz";
4645         case BPF_FIELD_EXISTS: return "field_exists";
4646         case BPF_FIELD_SIGNED: return "signed";
4647         case BPF_FIELD_LSHIFT_U64: return "lshift_u64";
4648         case BPF_FIELD_RSHIFT_U64: return "rshift_u64";
4649         case BPF_TYPE_ID_LOCAL: return "local_type_id";
4650         case BPF_TYPE_ID_TARGET: return "target_type_id";
4651         case BPF_TYPE_EXISTS: return "type_exists";
4652         case BPF_TYPE_SIZE: return "type_size";
4653         case BPF_ENUMVAL_EXISTS: return "enumval_exists";
4654         case BPF_ENUMVAL_VALUE: return "enumval_value";
4655         default: return "unknown";
4656         }
4657 }
4658
4659 static bool core_relo_is_field_based(enum bpf_core_relo_kind kind)
4660 {
4661         switch (kind) {
4662         case BPF_FIELD_BYTE_OFFSET:
4663         case BPF_FIELD_BYTE_SIZE:
4664         case BPF_FIELD_EXISTS:
4665         case BPF_FIELD_SIGNED:
4666         case BPF_FIELD_LSHIFT_U64:
4667         case BPF_FIELD_RSHIFT_U64:
4668                 return true;
4669         default:
4670                 return false;
4671         }
4672 }
4673
4674 static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
4675 {
4676         switch (kind) {
4677         case BPF_TYPE_ID_LOCAL:
4678         case BPF_TYPE_ID_TARGET:
4679         case BPF_TYPE_EXISTS:
4680         case BPF_TYPE_SIZE:
4681                 return true;
4682         default:
4683                 return false;
4684         }
4685 }
4686
4687 static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
4688 {
4689         switch (kind) {
4690         case BPF_ENUMVAL_EXISTS:
4691         case BPF_ENUMVAL_VALUE:
4692                 return true;
4693         default:
4694                 return false;
4695         }
4696 }
4697
4698 /*
4699  * Turn bpf_core_relo into a low- and high-level spec representation,
4700  * validating correctness along the way, as well as calculating resulting
4701  * field bit offset, specified by accessor string. Low-level spec captures
4702  * every single level of nestedness, including traversing anonymous
4703  * struct/union members. High-level one only captures semantically meaningful
4704  * "turning points": named fields and array indicies.
4705  * E.g., for this case:
4706  *
4707  *   struct sample {
4708  *       int __unimportant;
4709  *       struct {
4710  *           int __1;
4711  *           int __2;
4712  *           int a[7];
4713  *       };
4714  *   };
4715  *
4716  *   struct sample *s = ...;
4717  *
4718  *   int x = &s->a[3]; // access string = '0:1:2:3'
4719  *
4720  * Low-level spec has 1:1 mapping with each element of access string (it's
4721  * just a parsed access string representation): [0, 1, 2, 3].
4722  *
4723  * High-level spec will capture only 3 points:
4724  *   - intial zero-index access by pointer (&s->... is the same as &s[0]...);
4725  *   - field 'a' access (corresponds to '2' in low-level spec);
4726  *   - array element #3 access (corresponds to '3' in low-level spec).
4727  *
4728  * Type-based relocations (TYPE_EXISTS/TYPE_SIZE,
4729  * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their
4730  * spec and raw_spec are kept empty.
4731  *
4732  * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access
4733  * string to specify enumerator's value index that need to be relocated.
4734  */
4735 static int bpf_core_parse_spec(const struct btf *btf,
4736                                __u32 type_id,
4737                                const char *spec_str,
4738                                enum bpf_core_relo_kind relo_kind,
4739                                struct bpf_core_spec *spec)
4740 {
4741         int access_idx, parsed_len, i;
4742         struct bpf_core_accessor *acc;
4743         const struct btf_type *t;
4744         const char *name;
4745         __u32 id;
4746         __s64 sz;
4747
4748         if (str_is_empty(spec_str) || *spec_str == ':')
4749                 return -EINVAL;
4750
4751         memset(spec, 0, sizeof(*spec));
4752         spec->btf = btf;
4753         spec->root_type_id = type_id;
4754         spec->relo_kind = relo_kind;
4755
4756         /* type-based relocations don't have a field access string */
4757         if (core_relo_is_type_based(relo_kind)) {
4758                 if (strcmp(spec_str, "0"))
4759                         return -EINVAL;
4760                 return 0;
4761         }
4762
4763         /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
4764         while (*spec_str) {
4765                 if (*spec_str == ':')
4766                         ++spec_str;
4767                 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
4768                         return -EINVAL;
4769                 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4770                         return -E2BIG;
4771                 spec_str += parsed_len;
4772                 spec->raw_spec[spec->raw_len++] = access_idx;
4773         }
4774
4775         if (spec->raw_len == 0)
4776                 return -EINVAL;
4777
4778         t = skip_mods_and_typedefs(btf, type_id, &id);
4779         if (!t)
4780                 return -EINVAL;
4781
4782         access_idx = spec->raw_spec[0];
4783         acc = &spec->spec[0];
4784         acc->type_id = id;
4785         acc->idx = access_idx;
4786         spec->len++;
4787
4788         if (core_relo_is_enumval_based(relo_kind)) {
4789                 if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
4790                         return -EINVAL;
4791
4792                 /* record enumerator name in a first accessor */
4793                 acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off);
4794                 return 0;
4795         }
4796
4797         if (!core_relo_is_field_based(relo_kind))
4798                 return -EINVAL;
4799
4800         sz = btf__resolve_size(btf, id);
4801         if (sz < 0)
4802                 return sz;
4803         spec->bit_offset = access_idx * sz * 8;
4804
4805         for (i = 1; i < spec->raw_len; i++) {
4806                 t = skip_mods_and_typedefs(btf, id, &id);
4807                 if (!t)
4808                         return -EINVAL;
4809
4810                 access_idx = spec->raw_spec[i];
4811                 acc = &spec->spec[spec->len];
4812
4813                 if (btf_is_composite(t)) {
4814                         const struct btf_member *m;
4815                         __u32 bit_offset;
4816
4817                         if (access_idx >= btf_vlen(t))
4818                                 return -EINVAL;
4819
4820                         bit_offset = btf_member_bit_offset(t, access_idx);
4821                         spec->bit_offset += bit_offset;
4822
4823                         m = btf_members(t) + access_idx;
4824                         if (m->name_off) {
4825                                 name = btf__name_by_offset(btf, m->name_off);
4826                                 if (str_is_empty(name))
4827                                         return -EINVAL;
4828
4829                                 acc->type_id = id;
4830                                 acc->idx = access_idx;
4831                                 acc->name = name;
4832                                 spec->len++;
4833                         }
4834
4835                         id = m->type;
4836                 } else if (btf_is_array(t)) {
4837                         const struct btf_array *a = btf_array(t);
4838                         bool flex;
4839
4840                         t = skip_mods_and_typedefs(btf, a->type, &id);
4841                         if (!t)
4842                                 return -EINVAL;
4843
4844                         flex = is_flex_arr(btf, acc - 1, a);
4845                         if (!flex && access_idx >= a->nelems)
4846                                 return -EINVAL;
4847
4848                         spec->spec[spec->len].type_id = id;
4849                         spec->spec[spec->len].idx = access_idx;
4850                         spec->len++;
4851
4852                         sz = btf__resolve_size(btf, id);
4853                         if (sz < 0)
4854                                 return sz;
4855                         spec->bit_offset += access_idx * sz * 8;
4856                 } else {
4857                         pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
4858                                 type_id, spec_str, i, id, btf_kind_str(t));
4859                         return -EINVAL;
4860                 }
4861         }
4862
4863         return 0;
4864 }
4865
4866 static bool bpf_core_is_flavor_sep(const char *s)
4867 {
4868         /* check X___Y name pattern, where X and Y are not underscores */
4869         return s[0] != '_' &&                                 /* X */
4870                s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */
4871                s[4] != '_';                                   /* Y */
4872 }
4873
4874 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
4875  * before last triple underscore. Struct name part after last triple
4876  * underscore is ignored by BPF CO-RE relocation during relocation matching.
4877  */
4878 static size_t bpf_core_essential_name_len(const char *name)
4879 {
4880         size_t n = strlen(name);
4881         int i;
4882
4883         for (i = n - 5; i >= 0; i--) {
4884                 if (bpf_core_is_flavor_sep(name + i))
4885                         return i + 1;
4886         }
4887         return n;
4888 }
4889
4890 struct core_cand
4891 {
4892         const struct btf *btf;
4893         const struct btf_type *t;
4894         const char *name;
4895         __u32 id;
4896 };
4897
4898 /* dynamically sized list of type IDs and its associated struct btf */
4899 struct core_cand_list {
4900         struct core_cand *cands;
4901         int len;
4902 };
4903
4904 static void bpf_core_free_cands(struct core_cand_list *cands)
4905 {
4906         free(cands->cands);
4907         free(cands);
4908 }
4909
4910 static int bpf_core_add_cands(struct core_cand *local_cand,
4911                               size_t local_essent_len,
4912                               const struct btf *targ_btf,
4913                               const char *targ_btf_name,
4914                               int targ_start_id,
4915                               struct core_cand_list *cands)
4916 {
4917         struct core_cand *new_cands, *cand;
4918         const struct btf_type *t;
4919         const char *targ_name;
4920         size_t targ_essent_len;
4921         int n, i;
4922
4923         n = btf__get_nr_types(targ_btf);
4924         for (i = targ_start_id; i <= n; i++) {
4925                 t = btf__type_by_id(targ_btf, i);
4926                 if (btf_kind(t) != btf_kind(local_cand->t))
4927                         continue;
4928
4929                 targ_name = btf__name_by_offset(targ_btf, t->name_off);
4930                 if (str_is_empty(targ_name))
4931                         continue;
4932
4933                 targ_essent_len = bpf_core_essential_name_len(targ_name);
4934                 if (targ_essent_len != local_essent_len)
4935                         continue;
4936
4937                 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
4938                         continue;
4939
4940                 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
4941                          local_cand->id, btf_kind_str(local_cand->t),
4942                          local_cand->name, i, btf_kind_str(t), targ_name,
4943                          targ_btf_name);
4944                 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
4945                                               sizeof(*cands->cands));
4946                 if (!new_cands)
4947                         return -ENOMEM;
4948
4949                 cand = &new_cands[cands->len];
4950                 cand->btf = targ_btf;
4951                 cand->t = t;
4952                 cand->name = targ_name;
4953                 cand->id = i;
4954
4955                 cands->cands = new_cands;
4956                 cands->len++;
4957         }
4958         return 0;
4959 }
4960
4961 static int load_module_btfs(struct bpf_object *obj)
4962 {
4963         struct bpf_btf_info info;
4964         struct module_btf *mod_btf;
4965         struct btf *btf;
4966         char name[64];
4967         __u32 id = 0, len;
4968         int err, fd;
4969
4970         if (obj->btf_modules_loaded)
4971                 return 0;
4972
4973         /* don't do this again, even if we find no module BTFs */
4974         obj->btf_modules_loaded = true;
4975
4976         /* kernel too old to support module BTFs */
4977         if (!kernel_supports(FEAT_MODULE_BTF))
4978                 return 0;
4979
4980         while (true) {
4981                 err = bpf_btf_get_next_id(id, &id);
4982                 if (err && errno == ENOENT)
4983                         return 0;
4984                 if (err) {
4985                         err = -errno;
4986                         pr_warn("failed to iterate BTF objects: %d\n", err);
4987                         return err;
4988                 }
4989
4990                 fd = bpf_btf_get_fd_by_id(id);
4991                 if (fd < 0) {
4992                         if (errno == ENOENT)
4993                                 continue; /* expected race: BTF was unloaded */
4994                         err = -errno;
4995                         pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
4996                         return err;
4997                 }
4998
4999                 len = sizeof(info);
5000                 memset(&info, 0, sizeof(info));
5001                 info.name = ptr_to_u64(name);
5002                 info.name_len = sizeof(name);
5003
5004                 err = bpf_obj_get_info_by_fd(fd, &info, &len);
5005                 if (err) {
5006                         err = -errno;
5007                         pr_warn("failed to get BTF object #%d info: %d\n", id, err);
5008                         goto err_out;
5009                 }
5010
5011                 /* ignore non-module BTFs */
5012                 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
5013                         close(fd);
5014                         continue;
5015                 }
5016
5017                 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5018                 if (IS_ERR(btf)) {
5019                         pr_warn("failed to load module [%s]'s BTF object #%d: %ld\n",
5020                                 name, id, PTR_ERR(btf));
5021                         err = PTR_ERR(btf);
5022                         goto err_out;
5023                 }
5024
5025                 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5026                                         sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5027                 if (err)
5028                         goto err_out;
5029
5030                 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5031
5032                 mod_btf->btf = btf;
5033                 mod_btf->id = id;
5034                 mod_btf->fd = fd;
5035                 mod_btf->name = strdup(name);
5036                 if (!mod_btf->name) {
5037                         err = -ENOMEM;
5038                         goto err_out;
5039                 }
5040                 continue;
5041
5042 err_out:
5043                 close(fd);
5044                 return err;
5045         }
5046
5047         return 0;
5048 }
5049
5050 static struct core_cand_list *
5051 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
5052 {
5053         struct core_cand local_cand = {};
5054         struct core_cand_list *cands;
5055         const struct btf *main_btf;
5056         size_t local_essent_len;
5057         int err, i;
5058
5059         local_cand.btf = local_btf;
5060         local_cand.t = btf__type_by_id(local_btf, local_type_id);
5061         if (!local_cand.t)
5062                 return ERR_PTR(-EINVAL);
5063
5064         local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
5065         if (str_is_empty(local_cand.name))
5066                 return ERR_PTR(-EINVAL);
5067         local_essent_len = bpf_core_essential_name_len(local_cand.name);
5068
5069         cands = calloc(1, sizeof(*cands));
5070         if (!cands)
5071                 return ERR_PTR(-ENOMEM);
5072
5073         /* Attempt to find target candidates in vmlinux BTF first */
5074         main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5075         err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
5076         if (err)
5077                 goto err_out;
5078
5079         /* if vmlinux BTF has any candidate, don't got for module BTFs */
5080         if (cands->len)
5081                 return cands;
5082
5083         /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
5084         if (obj->btf_vmlinux_override)
5085                 return cands;
5086
5087         /* now look through module BTFs, trying to still find candidates */
5088         err = load_module_btfs(obj);
5089         if (err)
5090                 goto err_out;
5091
5092         for (i = 0; i < obj->btf_module_cnt; i++) {
5093                 err = bpf_core_add_cands(&local_cand, local_essent_len,
5094                                          obj->btf_modules[i].btf,
5095                                          obj->btf_modules[i].name,
5096                                          btf__get_nr_types(obj->btf_vmlinux) + 1,
5097                                          cands);
5098                 if (err)
5099                         goto err_out;
5100         }
5101
5102         return cands;
5103 err_out:
5104         bpf_core_free_cands(cands);
5105         return ERR_PTR(err);
5106 }
5107
5108 /* Check two types for compatibility for the purpose of field access
5109  * relocation. const/volatile/restrict and typedefs are skipped to ensure we
5110  * are relocating semantically compatible entities:
5111  *   - any two STRUCTs/UNIONs are compatible and can be mixed;
5112  *   - any two FWDs are compatible, if their names match (modulo flavor suffix);
5113  *   - any two PTRs are always compatible;
5114  *   - for ENUMs, names should be the same (ignoring flavor suffix) or at
5115  *     least one of enums should be anonymous;
5116  *   - for ENUMs, check sizes, names are ignored;
5117  *   - for INT, size and signedness are ignored;
5118  *   - for ARRAY, dimensionality is ignored, element types are checked for
5119  *     compatibility recursively;
5120  *   - everything else shouldn't be ever a target of relocation.
5121  * These rules are not set in stone and probably will be adjusted as we get
5122  * more experience with using BPF CO-RE relocations.
5123  */
5124 static int bpf_core_fields_are_compat(const struct btf *local_btf,
5125                                       __u32 local_id,
5126                                       const struct btf *targ_btf,
5127                                       __u32 targ_id)
5128 {
5129         const struct btf_type *local_type, *targ_type;
5130
5131 recur:
5132         local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
5133         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
5134         if (!local_type || !targ_type)
5135                 return -EINVAL;
5136
5137         if (btf_is_composite(local_type) && btf_is_composite(targ_type))
5138                 return 1;
5139         if (btf_kind(local_type) != btf_kind(targ_type))
5140                 return 0;
5141
5142         switch (btf_kind(local_type)) {
5143         case BTF_KIND_PTR:
5144                 return 1;
5145         case BTF_KIND_FWD:
5146         case BTF_KIND_ENUM: {
5147                 const char *local_name, *targ_name;
5148                 size_t local_len, targ_len;
5149
5150                 local_name = btf__name_by_offset(local_btf,
5151                                                  local_type->name_off);
5152                 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
5153                 local_len = bpf_core_essential_name_len(local_name);
5154                 targ_len = bpf_core_essential_name_len(targ_name);
5155                 /* one of them is anonymous or both w/ same flavor-less names */
5156                 return local_len == 0 || targ_len == 0 ||
5157                        (local_len == targ_len &&
5158                         strncmp(local_name, targ_name, local_len) == 0);
5159         }
5160         case BTF_KIND_INT:
5161                 /* just reject deprecated bitfield-like integers; all other
5162                  * integers are by default compatible between each other
5163                  */
5164                 return btf_int_offset(local_type) == 0 &&
5165                        btf_int_offset(targ_type) == 0;
5166         case BTF_KIND_ARRAY:
5167                 local_id = btf_array(local_type)->type;
5168                 targ_id = btf_array(targ_type)->type;
5169                 goto recur;
5170         default:
5171                 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
5172                         btf_kind(local_type), local_id, targ_id);
5173                 return 0;
5174         }
5175 }
5176
5177 /*
5178  * Given single high-level named field accessor in local type, find
5179  * corresponding high-level accessor for a target type. Along the way,
5180  * maintain low-level spec for target as well. Also keep updating target
5181  * bit offset.
5182  *
5183  * Searching is performed through recursive exhaustive enumeration of all
5184  * fields of a struct/union. If there are any anonymous (embedded)
5185  * structs/unions, they are recursively searched as well. If field with
5186  * desired name is found, check compatibility between local and target types,
5187  * before returning result.
5188  *
5189  * 1 is returned, if field is found.
5190  * 0 is returned if no compatible field is found.
5191  * <0 is returned on error.
5192  */
5193 static int bpf_core_match_member(const struct btf *local_btf,
5194                                  const struct bpf_core_accessor *local_acc,
5195                                  const struct btf *targ_btf,
5196                                  __u32 targ_id,
5197                                  struct bpf_core_spec *spec,
5198                                  __u32 *next_targ_id)
5199 {
5200         const struct btf_type *local_type, *targ_type;
5201         const struct btf_member *local_member, *m;
5202         const char *local_name, *targ_name;
5203         __u32 local_id;
5204         int i, n, found;
5205
5206         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
5207         if (!targ_type)
5208                 return -EINVAL;
5209         if (!btf_is_composite(targ_type))
5210                 return 0;
5211
5212         local_id = local_acc->type_id;
5213         local_type = btf__type_by_id(local_btf, local_id);
5214         local_member = btf_members(local_type) + local_acc->idx;
5215         local_name = btf__name_by_offset(local_btf, local_member->name_off);
5216
5217         n = btf_vlen(targ_type);
5218         m = btf_members(targ_type);
5219         for (i = 0; i < n; i++, m++) {
5220                 __u32 bit_offset;
5221
5222                 bit_offset = btf_member_bit_offset(targ_type, i);
5223
5224                 /* too deep struct/union/array nesting */
5225                 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
5226                         return -E2BIG;
5227
5228                 /* speculate this member will be the good one */
5229                 spec->bit_offset += bit_offset;
5230                 spec->raw_spec[spec->raw_len++] = i;
5231
5232                 targ_name = btf__name_by_offset(targ_btf, m->name_off);
5233                 if (str_is_empty(targ_name)) {
5234                         /* embedded struct/union, we need to go deeper */
5235                         found = bpf_core_match_member(local_btf, local_acc,
5236                                                       targ_btf, m->type,
5237                                                       spec, next_targ_id);
5238                         if (found) /* either found or error */
5239                                 return found;
5240                 } else if (strcmp(local_name, targ_name) == 0) {
5241                         /* matching named field */
5242                         struct bpf_core_accessor *targ_acc;
5243
5244                         targ_acc = &spec->spec[spec->len++];
5245                         targ_acc->type_id = targ_id;
5246                         targ_acc->idx = i;
5247                         targ_acc->name = targ_name;
5248
5249                         *next_targ_id = m->type;
5250                         found = bpf_core_fields_are_compat(local_btf,
5251                                                            local_member->type,
5252                                                            targ_btf, m->type);
5253                         if (!found)
5254                                 spec->len--; /* pop accessor */
5255                         return found;
5256                 }
5257                 /* member turned out not to be what we looked for */
5258                 spec->bit_offset -= bit_offset;
5259                 spec->raw_len--;
5260         }
5261
5262         return 0;
5263 }
5264
5265 /* Check local and target types for compatibility. This check is used for
5266  * type-based CO-RE relocations and follow slightly different rules than
5267  * field-based relocations. This function assumes that root types were already
5268  * checked for name match. Beyond that initial root-level name check, names
5269  * are completely ignored. Compatibility rules are as follows:
5270  *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5271  *     kind should match for local and target types (i.e., STRUCT is not
5272  *     compatible with UNION);
5273  *   - for ENUMs, the size is ignored;
5274  *   - for INT, size and signedness are ignored;
5275  *   - for ARRAY, dimensionality is ignored, element types are checked for
5276  *     compatibility recursively;
5277  *   - CONST/VOLATILE/RESTRICT modifiers are ignored;
5278  *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5279  *   - FUNC_PROTOs are compatible if they have compatible signature: same
5280  *     number of input args and compatible return and argument types.
5281  * These rules are not set in stone and probably will be adjusted as we get
5282  * more experience with using BPF CO-RE relocations.
5283  */
5284 static int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5285                                      const struct btf *targ_btf, __u32 targ_id)
5286 {
5287         const struct btf_type *local_type, *targ_type;
5288         int depth = 32; /* max recursion depth */
5289
5290         /* caller made sure that names match (ignoring flavor suffix) */
5291         local_type = btf__type_by_id(local_btf, local_id);
5292         targ_type = btf__type_by_id(targ_btf, targ_id);
5293         if (btf_kind(local_type) != btf_kind(targ_type))
5294                 return 0;
5295
5296 recur:
5297         depth--;
5298         if (depth < 0)
5299                 return -EINVAL;
5300
5301         local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
5302         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
5303         if (!local_type || !targ_type)
5304                 return -EINVAL;
5305
5306         if (btf_kind(local_type) != btf_kind(targ_type))
5307                 return 0;
5308
5309         switch (btf_kind(local_type)) {
5310         case BTF_KIND_UNKN:
5311         case BTF_KIND_STRUCT:
5312         case BTF_KIND_UNION:
5313         case BTF_KIND_ENUM:
5314         case BTF_KIND_FWD:
5315                 return 1;
5316         case BTF_KIND_INT:
5317                 /* just reject deprecated bitfield-like integers; all other
5318                  * integers are by default compatible between each other
5319                  */
5320                 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
5321         case BTF_KIND_PTR:
5322                 local_id = local_type->type;
5323                 targ_id = targ_type->type;
5324                 goto recur;
5325         case BTF_KIND_ARRAY:
5326                 local_id = btf_array(local_type)->type;
5327                 targ_id = btf_array(targ_type)->type;
5328                 goto recur;
5329         case BTF_KIND_FUNC_PROTO: {
5330                 struct btf_param *local_p = btf_params(local_type);
5331                 struct btf_param *targ_p = btf_params(targ_type);
5332                 __u16 local_vlen = btf_vlen(local_type);
5333                 __u16 targ_vlen = btf_vlen(targ_type);
5334                 int i, err;
5335
5336                 if (local_vlen != targ_vlen)
5337                         return 0;
5338
5339                 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
5340                         skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
5341                         skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
5342                         err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id);
5343                         if (err <= 0)
5344                                 return err;
5345                 }
5346
5347                 /* tail recurse for return type check */
5348                 skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
5349                 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
5350                 goto recur;
5351         }
5352         default:
5353                 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
5354                         btf_kind_str(local_type), local_id, targ_id);
5355                 return 0;
5356         }
5357 }
5358
5359 /*
5360  * Try to match local spec to a target type and, if successful, produce full
5361  * target spec (high-level, low-level + bit offset).
5362  */
5363 static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
5364                                const struct btf *targ_btf, __u32 targ_id,
5365                                struct bpf_core_spec *targ_spec)
5366 {
5367         const struct btf_type *targ_type;
5368         const struct bpf_core_accessor *local_acc;
5369         struct bpf_core_accessor *targ_acc;
5370         int i, sz, matched;
5371
5372         memset(targ_spec, 0, sizeof(*targ_spec));
5373         targ_spec->btf = targ_btf;
5374         targ_spec->root_type_id = targ_id;
5375         targ_spec->relo_kind = local_spec->relo_kind;
5376
5377         if (core_relo_is_type_based(local_spec->relo_kind)) {
5378                 return bpf_core_types_are_compat(local_spec->btf,
5379                                                  local_spec->root_type_id,
5380                                                  targ_btf, targ_id);
5381         }
5382
5383         local_acc = &local_spec->spec[0];
5384         targ_acc = &targ_spec->spec[0];
5385
5386         if (core_relo_is_enumval_based(local_spec->relo_kind)) {
5387                 size_t local_essent_len, targ_essent_len;
5388                 const struct btf_enum *e;
5389                 const char *targ_name;
5390
5391                 /* has to resolve to an enum */
5392                 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
5393                 if (!btf_is_enum(targ_type))
5394                         return 0;
5395
5396                 local_essent_len = bpf_core_essential_name_len(local_acc->name);
5397
5398                 for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) {
5399                         targ_name = btf__name_by_offset(targ_spec->btf, e->name_off);
5400                         targ_essent_len = bpf_core_essential_name_len(targ_name);
5401                         if (targ_essent_len != local_essent_len)
5402                                 continue;
5403                         if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {
5404                                 targ_acc->type_id = targ_id;
5405                                 targ_acc->idx = i;
5406                                 targ_acc->name = targ_name;
5407                                 targ_spec->len++;
5408                                 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5409                                 targ_spec->raw_len++;
5410                                 return 1;
5411                         }
5412                 }
5413                 return 0;
5414         }
5415
5416         if (!core_relo_is_field_based(local_spec->relo_kind))
5417                 return -EINVAL;
5418
5419         for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
5420                 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
5421                                                    &targ_id);
5422                 if (!targ_type)
5423                         return -EINVAL;
5424
5425                 if (local_acc->name) {
5426                         matched = bpf_core_match_member(local_spec->btf,
5427                                                         local_acc,
5428                                                         targ_btf, targ_id,
5429                                                         targ_spec, &targ_id);
5430                         if (matched <= 0)
5431                                 return matched;
5432                 } else {
5433                         /* for i=0, targ_id is already treated as array element
5434                          * type (because it's the original struct), for others
5435                          * we should find array element type first
5436                          */
5437                         if (i > 0) {
5438                                 const struct btf_array *a;
5439                                 bool flex;
5440
5441                                 if (!btf_is_array(targ_type))
5442                                         return 0;
5443
5444                                 a = btf_array(targ_type);
5445                                 flex = is_flex_arr(targ_btf, targ_acc - 1, a);
5446                                 if (!flex && local_acc->idx >= a->nelems)
5447                                         return 0;
5448                                 if (!skip_mods_and_typedefs(targ_btf, a->type,
5449                                                             &targ_id))
5450                                         return -EINVAL;
5451                         }
5452
5453                         /* too deep struct/union/array nesting */
5454                         if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
5455                                 return -E2BIG;
5456
5457                         targ_acc->type_id = targ_id;
5458                         targ_acc->idx = local_acc->idx;
5459                         targ_acc->name = NULL;
5460                         targ_spec->len++;
5461                         targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5462                         targ_spec->raw_len++;
5463
5464                         sz = btf__resolve_size(targ_btf, targ_id);
5465                         if (sz < 0)
5466                                 return sz;
5467                         targ_spec->bit_offset += local_acc->idx * sz * 8;
5468                 }
5469         }
5470
5471         return 1;
5472 }
5473
5474 static int bpf_core_calc_field_relo(const struct bpf_program *prog,
5475                                     const struct bpf_core_relo *relo,
5476                                     const struct bpf_core_spec *spec,
5477                                     __u32 *val, __u32 *field_sz, __u32 *type_id,
5478                                     bool *validate)
5479 {
5480         const struct bpf_core_accessor *acc;
5481         const struct btf_type *t;
5482         __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
5483         const struct btf_member *m;
5484         const struct btf_type *mt;
5485         bool bitfield;
5486         __s64 sz;
5487
5488         *field_sz = 0;
5489
5490         if (relo->kind == BPF_FIELD_EXISTS) {
5491                 *val = spec ? 1 : 0;
5492                 return 0;
5493         }
5494
5495         if (!spec)
5496                 return -EUCLEAN; /* request instruction poisoning */
5497
5498         acc = &spec->spec[spec->len - 1];
5499         t = btf__type_by_id(spec->btf, acc->type_id);
5500
5501         /* a[n] accessor needs special handling */
5502         if (!acc->name) {
5503                 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
5504                         *val = spec->bit_offset / 8;
5505                         /* remember field size for load/store mem size */
5506                         sz = btf__resolve_size(spec->btf, acc->type_id);
5507                         if (sz < 0)
5508                                 return -EINVAL;
5509                         *field_sz = sz;
5510                         *type_id = acc->type_id;
5511                 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
5512                         sz = btf__resolve_size(spec->btf, acc->type_id);
5513                         if (sz < 0)
5514                                 return -EINVAL;
5515                         *val = sz;
5516                 } else {
5517                         pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
5518                                 prog->name, relo->kind, relo->insn_off / 8);
5519                         return -EINVAL;
5520                 }
5521                 if (validate)
5522                         *validate = true;
5523                 return 0;
5524         }
5525
5526         m = btf_members(t) + acc->idx;
5527         mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
5528         bit_off = spec->bit_offset;
5529         bit_sz = btf_member_bitfield_size(t, acc->idx);
5530
5531         bitfield = bit_sz > 0;
5532         if (bitfield) {
5533                 byte_sz = mt->size;
5534                 byte_off = bit_off / 8 / byte_sz * byte_sz;
5535                 /* figure out smallest int size necessary for bitfield load */
5536                 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
5537                         if (byte_sz >= 8) {
5538                                 /* bitfield can't be read with 64-bit read */
5539                                 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
5540                                         prog->name, relo->kind, relo->insn_off / 8);
5541                                 return -E2BIG;
5542                         }
5543                         byte_sz *= 2;
5544                         byte_off = bit_off / 8 / byte_sz * byte_sz;
5545                 }
5546         } else {
5547                 sz = btf__resolve_size(spec->btf, field_type_id);
5548                 if (sz < 0)
5549                         return -EINVAL;
5550                 byte_sz = sz;
5551                 byte_off = spec->bit_offset / 8;
5552                 bit_sz = byte_sz * 8;
5553         }
5554
5555         /* for bitfields, all the relocatable aspects are ambiguous and we
5556          * might disagree with compiler, so turn off validation of expected
5557          * value, except for signedness
5558          */
5559         if (validate)
5560                 *validate = !bitfield;
5561
5562         switch (relo->kind) {
5563         case BPF_FIELD_BYTE_OFFSET:
5564                 *val = byte_off;
5565                 if (!bitfield) {
5566                         *field_sz = byte_sz;
5567                         *type_id = field_type_id;
5568                 }
5569                 break;
5570         case BPF_FIELD_BYTE_SIZE:
5571                 *val = byte_sz;
5572                 break;
5573         case BPF_FIELD_SIGNED:
5574                 /* enums will be assumed unsigned */
5575                 *val = btf_is_enum(mt) ||
5576                        (btf_int_encoding(mt) & BTF_INT_SIGNED);
5577                 if (validate)
5578                         *validate = true; /* signedness is never ambiguous */
5579                 break;
5580         case BPF_FIELD_LSHIFT_U64:
5581 #if __BYTE_ORDER == __LITTLE_ENDIAN
5582                 *val = 64 - (bit_off + bit_sz - byte_off  * 8);
5583 #else
5584                 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
5585 #endif
5586                 break;
5587         case BPF_FIELD_RSHIFT_U64:
5588                 *val = 64 - bit_sz;
5589                 if (validate)
5590                         *validate = true; /* right shift is never ambiguous */
5591                 break;
5592         case BPF_FIELD_EXISTS:
5593         default:
5594                 return -EOPNOTSUPP;
5595         }
5596
5597         return 0;
5598 }
5599
5600 static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
5601                                    const struct bpf_core_spec *spec,
5602                                    __u32 *val)
5603 {
5604         __s64 sz;
5605
5606         /* type-based relos return zero when target type is not found */
5607         if (!spec) {
5608                 *val = 0;
5609                 return 0;
5610         }
5611
5612         switch (relo->kind) {
5613         case BPF_TYPE_ID_TARGET:
5614                 *val = spec->root_type_id;
5615                 break;
5616         case BPF_TYPE_EXISTS:
5617                 *val = 1;
5618                 break;
5619         case BPF_TYPE_SIZE:
5620                 sz = btf__resolve_size(spec->btf, spec->root_type_id);
5621                 if (sz < 0)
5622                         return -EINVAL;
5623                 *val = sz;
5624                 break;
5625         case BPF_TYPE_ID_LOCAL:
5626         /* BPF_TYPE_ID_LOCAL is handled specially and shouldn't get here */
5627         default:
5628                 return -EOPNOTSUPP;
5629         }
5630
5631         return 0;
5632 }
5633
5634 static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
5635                                       const struct bpf_core_spec *spec,
5636                                       __u32 *val)
5637 {
5638         const struct btf_type *t;
5639         const struct btf_enum *e;
5640
5641         switch (relo->kind) {
5642         case BPF_ENUMVAL_EXISTS:
5643                 *val = spec ? 1 : 0;
5644                 break;
5645         case BPF_ENUMVAL_VALUE:
5646                 if (!spec)
5647                         return -EUCLEAN; /* request instruction poisoning */
5648                 t = btf__type_by_id(spec->btf, spec->spec[0].type_id);
5649                 e = btf_enum(t) + spec->spec[0].idx;
5650                 *val = e->val;
5651                 break;
5652         default:
5653                 return -EOPNOTSUPP;
5654         }
5655
5656         return 0;
5657 }
5658
5659 struct bpf_core_relo_res
5660 {
5661         /* expected value in the instruction, unless validate == false */
5662         __u32 orig_val;
5663         /* new value that needs to be patched up to */
5664         __u32 new_val;
5665         /* relocation unsuccessful, poison instruction, but don't fail load */
5666         bool poison;
5667         /* some relocations can't be validated against orig_val */
5668         bool validate;
5669         /* for field byte offset relocations or the forms:
5670          *     *(T *)(rX + <off>) = rY
5671          *     rX = *(T *)(rY + <off>),
5672          * we remember original and resolved field size to adjust direct
5673          * memory loads of pointers and integers; this is necessary for 32-bit
5674          * host kernel architectures, but also allows to automatically
5675          * relocate fields that were resized from, e.g., u32 to u64, etc.
5676          */
5677         bool fail_memsz_adjust;
5678         __u32 orig_sz;
5679         __u32 orig_type_id;
5680         __u32 new_sz;
5681         __u32 new_type_id;
5682 };
5683
5684 /* Calculate original and target relocation values, given local and target
5685  * specs and relocation kind. These values are calculated for each candidate.
5686  * If there are multiple candidates, resulting values should all be consistent
5687  * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity.
5688  * If instruction has to be poisoned, *poison will be set to true.
5689  */
5690 static int bpf_core_calc_relo(const struct bpf_program *prog,
5691                               const struct bpf_core_relo *relo,
5692                               int relo_idx,
5693                               const struct bpf_core_spec *local_spec,
5694                               const struct bpf_core_spec *targ_spec,
5695                               struct bpf_core_relo_res *res)
5696 {
5697         int err = -EOPNOTSUPP;
5698
5699         res->orig_val = 0;
5700         res->new_val = 0;
5701         res->poison = false;
5702         res->validate = true;
5703         res->fail_memsz_adjust = false;
5704         res->orig_sz = res->new_sz = 0;
5705         res->orig_type_id = res->new_type_id = 0;
5706
5707         if (core_relo_is_field_based(relo->kind)) {
5708                 err = bpf_core_calc_field_relo(prog, relo, local_spec,
5709                                                &res->orig_val, &res->orig_sz,
5710                                                &res->orig_type_id, &res->validate);
5711                 err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec,
5712                                                       &res->new_val, &res->new_sz,
5713                                                       &res->new_type_id, NULL);
5714                 if (err)
5715                         goto done;
5716                 /* Validate if it's safe to adjust load/store memory size.
5717                  * Adjustments are performed only if original and new memory
5718                  * sizes differ.
5719                  */
5720                 res->fail_memsz_adjust = false;
5721                 if (res->orig_sz != res->new_sz) {
5722                         const struct btf_type *orig_t, *new_t;
5723
5724                         orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id);
5725                         new_t = btf__type_by_id(targ_spec->btf, res->new_type_id);
5726
5727                         /* There are two use cases in which it's safe to
5728                          * adjust load/store's mem size:
5729                          *   - reading a 32-bit kernel pointer, while on BPF
5730                          *   size pointers are always 64-bit; in this case
5731                          *   it's safe to "downsize" instruction size due to
5732                          *   pointer being treated as unsigned integer with
5733                          *   zero-extended upper 32-bits;
5734                          *   - reading unsigned integers, again due to
5735                          *   zero-extension is preserving the value correctly.
5736                          *
5737                          * In all other cases it's incorrect to attempt to
5738                          * load/store field because read value will be
5739                          * incorrect, so we poison relocated instruction.
5740                          */
5741                         if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
5742                                 goto done;
5743                         if (btf_is_int(orig_t) && btf_is_int(new_t) &&
5744                             btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
5745                             btf_int_encoding(new_t) != BTF_INT_SIGNED)
5746                                 goto done;
5747
5748                         /* mark as invalid mem size adjustment, but this will
5749                          * only be checked for LDX/STX/ST insns
5750                          */
5751                         res->fail_memsz_adjust = true;
5752                 }
5753         } else if (core_relo_is_type_based(relo->kind)) {
5754                 err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
5755                 err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
5756         } else if (core_relo_is_enumval_based(relo->kind)) {
5757                 err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
5758                 err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
5759         }
5760
5761 done:
5762         if (err == -EUCLEAN) {
5763                 /* EUCLEAN is used to signal instruction poisoning request */
5764                 res->poison = true;
5765                 err = 0;
5766         } else if (err == -EOPNOTSUPP) {
5767                 /* EOPNOTSUPP means unknown/unsupported relocation */
5768                 pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",
5769                         prog->name, relo_idx, core_relo_kind_str(relo->kind),
5770                         relo->kind, relo->insn_off / 8);
5771         }
5772
5773         return err;
5774 }
5775
5776 /*
5777  * Turn instruction for which CO_RE relocation failed into invalid one with
5778  * distinct signature.
5779  */
5780 static void bpf_core_poison_insn(struct bpf_program *prog, int relo_idx,
5781                                  int insn_idx, struct bpf_insn *insn)
5782 {
5783         pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
5784                  prog->name, relo_idx, insn_idx);
5785         insn->code = BPF_JMP | BPF_CALL;
5786         insn->dst_reg = 0;
5787         insn->src_reg = 0;
5788         insn->off = 0;
5789         /* if this instruction is reachable (not a dead code),
5790          * verifier will complain with the following message:
5791          * invalid func unknown#195896080
5792          */
5793         insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
5794 }
5795
5796 static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
5797 {
5798         switch (BPF_SIZE(insn->code)) {
5799         case BPF_DW: return 8;
5800         case BPF_W: return 4;
5801         case BPF_H: return 2;
5802         case BPF_B: return 1;
5803         default: return -1;
5804         }
5805 }
5806
5807 static int insn_bytes_to_bpf_size(__u32 sz)
5808 {
5809         switch (sz) {
5810         case 8: return BPF_DW;
5811         case 4: return BPF_W;
5812         case 2: return BPF_H;
5813         case 1: return BPF_B;
5814         default: return -1;
5815         }
5816 }
5817
5818 /*
5819  * Patch relocatable BPF instruction.
5820  *
5821  * Patched value is determined by relocation kind and target specification.
5822  * For existence relocations target spec will be NULL if field/type is not found.
5823  * Expected insn->imm value is determined using relocation kind and local
5824  * spec, and is checked before patching instruction. If actual insn->imm value
5825  * is wrong, bail out with error.
5826  *
5827  * Currently supported classes of BPF instruction are:
5828  * 1. rX = <imm> (assignment with immediate operand);
5829  * 2. rX += <imm> (arithmetic operations with immediate operand);
5830  * 3. rX = <imm64> (load with 64-bit immediate value);
5831  * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64};
5832  * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
5833  * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
5834  */
5835 static int bpf_core_patch_insn(struct bpf_program *prog,
5836                                const struct bpf_core_relo *relo,
5837                                int relo_idx,
5838                                const struct bpf_core_relo_res *res)
5839 {
5840         __u32 orig_val, new_val;
5841         struct bpf_insn *insn;
5842         int insn_idx;
5843         __u8 class;
5844
5845         if (relo->insn_off % BPF_INSN_SZ)
5846                 return -EINVAL;
5847         insn_idx = relo->insn_off / BPF_INSN_SZ;
5848         /* adjust insn_idx from section frame of reference to the local
5849          * program's frame of reference; (sub-)program code is not yet
5850          * relocated, so it's enough to just subtract in-section offset
5851          */
5852         insn_idx = insn_idx - prog->sec_insn_off;
5853         insn = &prog->insns[insn_idx];
5854         class = BPF_CLASS(insn->code);
5855
5856         if (res->poison) {
5857 poison:
5858                 /* poison second part of ldimm64 to avoid confusing error from
5859                  * verifier about "unknown opcode 00"
5860                  */
5861                 if (is_ldimm64_insn(insn))
5862                         bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);
5863                 bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);
5864                 return 0;
5865         }
5866
5867         orig_val = res->orig_val;
5868         new_val = res->new_val;
5869
5870         switch (class) {
5871         case BPF_ALU:
5872         case BPF_ALU64:
5873                 if (BPF_SRC(insn->code) != BPF_K)
5874                         return -EINVAL;
5875                 if (res->validate && insn->imm != orig_val) {
5876                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
5877                                 prog->name, relo_idx,
5878                                 insn_idx, insn->imm, orig_val, new_val);
5879                         return -EINVAL;
5880                 }
5881                 orig_val = insn->imm;
5882                 insn->imm = new_val;
5883                 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
5884                          prog->name, relo_idx, insn_idx,
5885                          orig_val, new_val);
5886                 break;
5887         case BPF_LDX:
5888         case BPF_ST:
5889         case BPF_STX:
5890                 if (res->validate && insn->off != orig_val) {
5891                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n",
5892                                 prog->name, relo_idx, insn_idx, insn->off, orig_val, new_val);
5893                         return -EINVAL;
5894                 }
5895                 if (new_val > SHRT_MAX) {
5896                         pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
5897                                 prog->name, relo_idx, insn_idx, new_val);
5898                         return -ERANGE;
5899                 }
5900                 if (res->fail_memsz_adjust) {
5901                         pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
5902                                 "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
5903                                 prog->name, relo_idx, insn_idx);
5904                         goto poison;
5905                 }
5906
5907                 orig_val = insn->off;
5908                 insn->off = new_val;
5909                 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
5910                          prog->name, relo_idx, insn_idx, orig_val, new_val);
5911
5912                 if (res->new_sz != res->orig_sz) {
5913                         int insn_bytes_sz, insn_bpf_sz;
5914
5915                         insn_bytes_sz = insn_bpf_size_to_bytes(insn);
5916                         if (insn_bytes_sz != res->orig_sz) {
5917                                 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
5918                                         prog->name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
5919                                 return -EINVAL;
5920                         }
5921
5922                         insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
5923                         if (insn_bpf_sz < 0) {
5924                                 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
5925                                         prog->name, relo_idx, insn_idx, res->new_sz);
5926                                 return -EINVAL;
5927                         }
5928
5929                         insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code);
5930                         pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
5931                                  prog->name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
5932                 }
5933                 break;
5934         case BPF_LD: {
5935                 __u64 imm;
5936
5937                 if (!is_ldimm64_insn(insn) ||
5938                     insn[0].src_reg != 0 || insn[0].off != 0 ||
5939                     insn_idx + 1 >= prog->insns_cnt ||
5940                     insn[1].code != 0 || insn[1].dst_reg != 0 ||
5941                     insn[1].src_reg != 0 || insn[1].off != 0) {
5942                         pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",
5943                                 prog->name, relo_idx, insn_idx);
5944                         return -EINVAL;
5945                 }
5946
5947                 imm = insn[0].imm + ((__u64)insn[1].imm << 32);
5948                 if (res->validate && imm != orig_val) {
5949                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n",
5950                                 prog->name, relo_idx,
5951                                 insn_idx, (unsigned long long)imm,
5952                                 orig_val, new_val);
5953                         return -EINVAL;
5954                 }
5955
5956                 insn[0].imm = new_val;
5957                 insn[1].imm = 0; /* currently only 32-bit values are supported */
5958                 pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n",
5959                          prog->name, relo_idx, insn_idx,
5960                          (unsigned long long)imm, new_val);
5961                 break;
5962         }
5963         default:
5964                 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
5965                         prog->name, relo_idx, insn_idx, insn->code,
5966                         insn->src_reg, insn->dst_reg, insn->off, insn->imm);
5967                 return -EINVAL;
5968         }
5969
5970         return 0;
5971 }
5972
5973 /* Output spec definition in the format:
5974  * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
5975  * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
5976  */
5977 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
5978 {
5979         const struct btf_type *t;
5980         const struct btf_enum *e;
5981         const char *s;
5982         __u32 type_id;
5983         int i;
5984
5985         type_id = spec->root_type_id;
5986         t = btf__type_by_id(spec->btf, type_id);
5987         s = btf__name_by_offset(spec->btf, t->name_off);
5988
5989         libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
5990
5991         if (core_relo_is_type_based(spec->relo_kind))
5992                 return;
5993
5994         if (core_relo_is_enumval_based(spec->relo_kind)) {
5995                 t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
5996                 e = btf_enum(t) + spec->raw_spec[0];
5997                 s = btf__name_by_offset(spec->btf, e->name_off);
5998
5999                 libbpf_print(level, "::%s = %u", s, e->val);
6000                 return;
6001         }
6002
6003         if (core_relo_is_field_based(spec->relo_kind)) {
6004                 for (i = 0; i < spec->len; i++) {
6005                         if (spec->spec[i].name)
6006                                 libbpf_print(level, ".%s", spec->spec[i].name);
6007                         else if (i > 0 || spec->spec[i].idx > 0)
6008                                 libbpf_print(level, "[%u]", spec->spec[i].idx);
6009                 }
6010
6011                 libbpf_print(level, " (");
6012                 for (i = 0; i < spec->raw_len; i++)
6013                         libbpf_print(level, "%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
6014
6015                 if (spec->bit_offset % 8)
6016                         libbpf_print(level, " @ offset %u.%u)",
6017                                      spec->bit_offset / 8, spec->bit_offset % 8);
6018                 else
6019                         libbpf_print(level, " @ offset %u)", spec->bit_offset / 8);
6020                 return;
6021         }
6022 }
6023
6024 static size_t bpf_core_hash_fn(const void *key, void *ctx)
6025 {
6026         return (size_t)key;
6027 }
6028
6029 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
6030 {
6031         return k1 == k2;
6032 }
6033
6034 static void *u32_as_hash_key(__u32 x)
6035 {
6036         return (void *)(uintptr_t)x;
6037 }
6038
6039 /*
6040  * CO-RE relocate single instruction.
6041  *
6042  * The outline and important points of the algorithm:
6043  * 1. For given local type, find corresponding candidate target types.
6044  *    Candidate type is a type with the same "essential" name, ignoring
6045  *    everything after last triple underscore (___). E.g., `sample`,
6046  *    `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
6047  *    for each other. Names with triple underscore are referred to as
6048  *    "flavors" and are useful, among other things, to allow to
6049  *    specify/support incompatible variations of the same kernel struct, which
6050  *    might differ between different kernel versions and/or build
6051  *    configurations.
6052  *
6053  *    N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
6054  *    converter, when deduplicated BTF of a kernel still contains more than
6055  *    one different types with the same name. In that case, ___2, ___3, etc
6056  *    are appended starting from second name conflict. But start flavors are
6057  *    also useful to be defined "locally", in BPF program, to extract same
6058  *    data from incompatible changes between different kernel
6059  *    versions/configurations. For instance, to handle field renames between
6060  *    kernel versions, one can use two flavors of the struct name with the
6061  *    same common name and use conditional relocations to extract that field,
6062  *    depending on target kernel version.
6063  * 2. For each candidate type, try to match local specification to this
6064  *    candidate target type. Matching involves finding corresponding
6065  *    high-level spec accessors, meaning that all named fields should match,
6066  *    as well as all array accesses should be within the actual bounds. Also,
6067  *    types should be compatible (see bpf_core_fields_are_compat for details).
6068  * 3. It is supported and expected that there might be multiple flavors
6069  *    matching the spec. As long as all the specs resolve to the same set of
6070  *    offsets across all candidates, there is no error. If there is any
6071  *    ambiguity, CO-RE relocation will fail. This is necessary to accomodate
6072  *    imprefection of BTF deduplication, which can cause slight duplication of
6073  *    the same BTF type, if some directly or indirectly referenced (by
6074  *    pointer) type gets resolved to different actual types in different
6075  *    object files. If such situation occurs, deduplicated BTF will end up
6076  *    with two (or more) structurally identical types, which differ only in
6077  *    types they refer to through pointer. This should be OK in most cases and
6078  *    is not an error.
6079  * 4. Candidate types search is performed by linearly scanning through all
6080  *    types in target BTF. It is anticipated that this is overall more
6081  *    efficient memory-wise and not significantly worse (if not better)
6082  *    CPU-wise compared to prebuilding a map from all local type names to
6083  *    a list of candidate type names. It's also sped up by caching resolved
6084  *    list of matching candidates per each local "root" type ID, that has at
6085  *    least one bpf_core_relo associated with it. This list is shared
6086  *    between multiple relocations for the same type ID and is updated as some
6087  *    of the candidates are pruned due to structural incompatibility.
6088  */
6089 static int bpf_core_apply_relo(struct bpf_program *prog,
6090                                const struct bpf_core_relo *relo,
6091                                int relo_idx,
6092                                const struct btf *local_btf,
6093                                struct hashmap *cand_cache)
6094 {
6095         struct bpf_core_spec local_spec, cand_spec, targ_spec = {};
6096         const void *type_key = u32_as_hash_key(relo->type_id);
6097         struct bpf_core_relo_res cand_res, targ_res;
6098         const struct btf_type *local_type;
6099         const char *local_name;
6100         struct core_cand_list *cands = NULL;
6101         __u32 local_id;
6102         const char *spec_str;
6103         int i, j, err;
6104
6105         local_id = relo->type_id;
6106         local_type = btf__type_by_id(local_btf, local_id);
6107         if (!local_type)
6108                 return -EINVAL;
6109
6110         local_name = btf__name_by_offset(local_btf, local_type->name_off);
6111         if (!local_name)
6112                 return -EINVAL;
6113
6114         spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
6115         if (str_is_empty(spec_str))
6116                 return -EINVAL;
6117
6118         err = bpf_core_parse_spec(local_btf, local_id, spec_str, relo->kind, &local_spec);
6119         if (err) {
6120                 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
6121                         prog->name, relo_idx, local_id, btf_kind_str(local_type),
6122                         str_is_empty(local_name) ? "<anon>" : local_name,
6123                         spec_str, err);
6124                 return -EINVAL;
6125         }
6126
6127         pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog->name,
6128                  relo_idx, core_relo_kind_str(relo->kind), relo->kind);
6129         bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
6130         libbpf_print(LIBBPF_DEBUG, "\n");
6131
6132         /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
6133         if (relo->kind == BPF_TYPE_ID_LOCAL) {
6134                 targ_res.validate = true;
6135                 targ_res.poison = false;
6136                 targ_res.orig_val = local_spec.root_type_id;
6137                 targ_res.new_val = local_spec.root_type_id;
6138                 goto patch_insn;
6139         }
6140
6141         /* libbpf doesn't support candidate search for anonymous types */
6142         if (str_is_empty(spec_str)) {
6143                 pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
6144                         prog->name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
6145                 return -EOPNOTSUPP;
6146         }
6147
6148         if (!hashmap__find(cand_cache, type_key, (void **)&cands)) {
6149                 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6150                 if (IS_ERR(cands)) {
6151                         pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
6152                                 prog->name, relo_idx, local_id, btf_kind_str(local_type),
6153                                 local_name, PTR_ERR(cands));
6154                         return PTR_ERR(cands);
6155                 }
6156                 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
6157                 if (err) {
6158                         bpf_core_free_cands(cands);
6159                         return err;
6160                 }
6161         }
6162
6163         for (i = 0, j = 0; i < cands->len; i++) {
6164                 err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,
6165                                           cands->cands[i].id, &cand_spec);
6166                 if (err < 0) {
6167                         pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
6168                                 prog->name, relo_idx, i);
6169                         bpf_core_dump_spec(LIBBPF_WARN, &cand_spec);
6170                         libbpf_print(LIBBPF_WARN, ": %d\n", err);
6171                         return err;
6172                 }
6173
6174                 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog->name,
6175                          relo_idx, err == 0 ? "non-matching" : "matching", i);
6176                 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
6177                 libbpf_print(LIBBPF_DEBUG, "\n");
6178
6179                 if (err == 0)
6180                         continue;
6181
6182                 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, &cand_spec, &cand_res);
6183                 if (err)
6184                         return err;
6185
6186                 if (j == 0) {
6187                         targ_res = cand_res;
6188                         targ_spec = cand_spec;
6189                 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
6190                         /* if there are many field relo candidates, they
6191                          * should all resolve to the same bit offset
6192                          */
6193                         pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
6194                                 prog->name, relo_idx, cand_spec.bit_offset,
6195                                 targ_spec.bit_offset);
6196                         return -EINVAL;
6197                 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {
6198                         /* all candidates should result in the same relocation
6199                          * decision and value, otherwise it's dangerous to
6200                          * proceed due to ambiguity
6201                          */
6202                         pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n",
6203                                 prog->name, relo_idx,
6204                                 cand_res.poison ? "failure" : "success", cand_res.new_val,
6205                                 targ_res.poison ? "failure" : "success", targ_res.new_val);
6206                         return -EINVAL;
6207                 }
6208
6209                 cands->cands[j++] = cands->cands[i];
6210         }
6211
6212         /*
6213          * For BPF_FIELD_EXISTS relo or when used BPF program has field
6214          * existence checks or kernel version/config checks, it's expected
6215          * that we might not find any candidates. In this case, if field
6216          * wasn't found in any candidate, the list of candidates shouldn't
6217          * change at all, we'll just handle relocating appropriately,
6218          * depending on relo's kind.
6219          */
6220         if (j > 0)
6221                 cands->len = j;
6222
6223         /*
6224          * If no candidates were found, it might be both a programmer error,
6225          * as well as expected case, depending whether instruction w/
6226          * relocation is guarded in some way that makes it unreachable (dead
6227          * code) if relocation can't be resolved. This is handled in
6228          * bpf_core_patch_insn() uniformly by replacing that instruction with
6229          * BPF helper call insn (using invalid helper ID). If that instruction
6230          * is indeed unreachable, then it will be ignored and eliminated by
6231          * verifier. If it was an error, then verifier will complain and point
6232          * to a specific instruction number in its log.
6233          */
6234         if (j == 0) {
6235                 pr_debug("prog '%s': relo #%d: no matching targets found\n",
6236                          prog->name, relo_idx);
6237
6238                 /* calculate single target relo result explicitly */
6239                 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, NULL, &targ_res);
6240                 if (err)
6241                         return err;
6242         }
6243
6244 patch_insn:
6245         /* bpf_core_patch_insn() should know how to handle missing targ_spec */
6246         err = bpf_core_patch_insn(prog, relo, relo_idx, &targ_res);
6247         if (err) {
6248                 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
6249                         prog->name, relo_idx, relo->insn_off, err);
6250                 return -EINVAL;
6251         }
6252
6253         return 0;
6254 }
6255
6256 static int
6257 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
6258 {
6259         const struct btf_ext_info_sec *sec;
6260         const struct bpf_core_relo *rec;
6261         const struct btf_ext_info *seg;
6262         struct hashmap_entry *entry;
6263         struct hashmap *cand_cache = NULL;
6264         struct bpf_program *prog;
6265         const char *sec_name;
6266         int i, err = 0, insn_idx, sec_idx;
6267
6268         if (obj->btf_ext->core_relo_info.len == 0)
6269                 return 0;
6270
6271         if (targ_btf_path) {
6272                 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6273                 if (IS_ERR_OR_NULL(obj->btf_vmlinux_override)) {
6274                         err = PTR_ERR(obj->btf_vmlinux_override);
6275                         pr_warn("failed to parse target BTF: %d\n", err);
6276                         return err;
6277                 }
6278         }
6279
6280         cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
6281         if (IS_ERR(cand_cache)) {
6282                 err = PTR_ERR(cand_cache);
6283                 goto out;
6284         }
6285
6286         seg = &obj->btf_ext->core_relo_info;
6287         for_each_btf_ext_sec(seg, sec) {
6288                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6289                 if (str_is_empty(sec_name)) {
6290                         err = -EINVAL;
6291                         goto out;
6292                 }
6293                 /* bpf_object's ELF is gone by now so it's not easy to find
6294                  * section index by section name, but we can find *any*
6295                  * bpf_program within desired section name and use it's
6296                  * prog->sec_idx to do a proper search by section index and
6297                  * instruction offset
6298                  */
6299                 prog = NULL;
6300                 for (i = 0; i < obj->nr_programs; i++) {
6301                         prog = &obj->programs[i];
6302                         if (strcmp(prog->sec_name, sec_name) == 0)
6303                                 break;
6304                 }
6305                 if (!prog) {
6306                         pr_warn("sec '%s': failed to find a BPF program\n", sec_name);
6307                         return -ENOENT;
6308                 }
6309                 sec_idx = prog->sec_idx;
6310
6311                 pr_debug("sec '%s': found %d CO-RE relocations\n",
6312                          sec_name, sec->num_info);
6313
6314                 for_each_btf_ext_rec(seg, sec, i, rec) {
6315                         insn_idx = rec->insn_off / BPF_INSN_SZ;
6316                         prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
6317                         if (!prog) {
6318                                 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n",
6319                                         sec_name, insn_idx, i);
6320                                 err = -EINVAL;
6321                                 goto out;
6322                         }
6323                         /* no need to apply CO-RE relocation if the program is
6324                          * not going to be loaded
6325                          */
6326                         if (!prog->load)
6327                                 continue;
6328
6329                         err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache);
6330                         if (err) {
6331                                 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
6332                                         prog->name, i, err);
6333                                 goto out;
6334                         }
6335                 }
6336         }
6337
6338 out:
6339         /* obj->btf_vmlinux and module BTFs are freed after object load */
6340         btf__free(obj->btf_vmlinux_override);
6341         obj->btf_vmlinux_override = NULL;
6342
6343         if (!IS_ERR_OR_NULL(cand_cache)) {
6344                 hashmap__for_each_entry(cand_cache, entry, i) {
6345                         bpf_core_free_cands(entry->value);
6346                 }
6347                 hashmap__free(cand_cache);
6348         }
6349         return err;
6350 }
6351
6352 /* Relocate data references within program code:
6353  *  - map references;
6354  *  - global variable references;
6355  *  - extern references.
6356  */
6357 static int
6358 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
6359 {
6360         int i;
6361
6362         for (i = 0; i < prog->nr_reloc; i++) {
6363                 struct reloc_desc *relo = &prog->reloc_desc[i];
6364                 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6365                 struct extern_desc *ext;
6366
6367                 switch (relo->type) {
6368                 case RELO_LD64:
6369                         insn[0].src_reg = BPF_PSEUDO_MAP_FD;
6370                         insn[0].imm = obj->maps[relo->map_idx].fd;
6371                         break;
6372                 case RELO_DATA:
6373                         insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6374                         insn[1].imm = insn[0].imm + relo->sym_off;
6375                         insn[0].imm = obj->maps[relo->map_idx].fd;
6376                         break;
6377                 case RELO_EXTERN_VAR:
6378                         ext = &obj->externs[relo->sym_off];
6379                         if (ext->type == EXT_KCFG) {
6380                                 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6381                                 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6382                                 insn[1].imm = ext->kcfg.data_off;
6383                         } else /* EXT_KSYM */ {
6384                                 if (ext->ksym.type_id) { /* typed ksyms */
6385                                         insn[0].src_reg = BPF_PSEUDO_BTF_ID;
6386                                         insn[0].imm = ext->ksym.kernel_btf_id;
6387                                         insn[1].imm = ext->ksym.kernel_btf_obj_fd;
6388                                 } else { /* typeless ksyms */
6389                                         insn[0].imm = (__u32)ext->ksym.addr;
6390                                         insn[1].imm = ext->ksym.addr >> 32;
6391                                 }
6392                         }
6393                         break;
6394                 case RELO_EXTERN_FUNC:
6395                         ext = &obj->externs[relo->sym_off];
6396                         insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
6397                         insn[0].imm = ext->ksym.kernel_btf_id;
6398                         break;
6399                 case RELO_SUBPROG_ADDR:
6400                         insn[0].src_reg = BPF_PSEUDO_FUNC;
6401                         /* will be handled as a follow up pass */
6402                         break;
6403                 case RELO_CALL:
6404                         /* will be handled as a follow up pass */
6405                         break;
6406                 default:
6407                         pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6408                                 prog->name, i, relo->type);
6409                         return -EINVAL;
6410                 }
6411         }
6412
6413         return 0;
6414 }
6415
6416 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
6417                                     const struct bpf_program *prog,
6418                                     const struct btf_ext_info *ext_info,
6419                                     void **prog_info, __u32 *prog_rec_cnt,
6420                                     __u32 *prog_rec_sz)
6421 {
6422         void *copy_start = NULL, *copy_end = NULL;
6423         void *rec, *rec_end, *new_prog_info;
6424         const struct btf_ext_info_sec *sec;
6425         size_t old_sz, new_sz;
6426         const char *sec_name;
6427         int i, off_adj;
6428
6429         for_each_btf_ext_sec(ext_info, sec) {
6430                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6431                 if (!sec_name)
6432                         return -EINVAL;
6433                 if (strcmp(sec_name, prog->sec_name) != 0)
6434                         continue;
6435
6436                 for_each_btf_ext_rec(ext_info, sec, i, rec) {
6437                         __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
6438
6439                         if (insn_off < prog->sec_insn_off)
6440                                 continue;
6441                         if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6442                                 break;
6443
6444                         if (!copy_start)
6445                                 copy_start = rec;
6446                         copy_end = rec + ext_info->rec_size;
6447                 }
6448
6449                 if (!copy_start)
6450                         return -ENOENT;
6451
6452                 /* append func/line info of a given (sub-)program to the main
6453                  * program func/line info
6454                  */
6455                 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6456                 new_sz = old_sz + (copy_end - copy_start);
6457                 new_prog_info = realloc(*prog_info, new_sz);
6458                 if (!new_prog_info)
6459                         return -ENOMEM;
6460                 *prog_info = new_prog_info;
6461                 *prog_rec_cnt = new_sz / ext_info->rec_size;
6462                 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6463
6464                 /* Kernel instruction offsets are in units of 8-byte
6465                  * instructions, while .BTF.ext instruction offsets generated
6466                  * by Clang are in units of bytes. So convert Clang offsets
6467                  * into kernel offsets and adjust offset according to program
6468                  * relocated position.
6469                  */
6470                 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6471                 rec = new_prog_info + old_sz;
6472                 rec_end = new_prog_info + new_sz;
6473                 for (; rec < rec_end; rec += ext_info->rec_size) {
6474                         __u32 *insn_off = rec;
6475
6476                         *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
6477                 }
6478                 *prog_rec_sz = ext_info->rec_size;
6479                 return 0;
6480         }
6481
6482         return -ENOENT;
6483 }
6484
6485 static int
6486 reloc_prog_func_and_line_info(const struct bpf_object *obj,
6487                               struct bpf_program *main_prog,
6488                               const struct bpf_program *prog)
6489 {
6490         int err;
6491
6492         /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
6493          * supprot func/line info
6494          */
6495         if (!obj->btf_ext || !kernel_supports(FEAT_BTF_FUNC))
6496                 return 0;
6497
6498         /* only attempt func info relocation if main program's func_info
6499          * relocation was successful
6500          */
6501         if (main_prog != prog && !main_prog->func_info)
6502                 goto line_info;
6503
6504         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6505                                        &main_prog->func_info,
6506                                        &main_prog->func_info_cnt,
6507                                        &main_prog->func_info_rec_size);
6508         if (err) {
6509                 if (err != -ENOENT) {
6510                         pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6511                                 prog->name, err);
6512                         return err;
6513                 }
6514                 if (main_prog->func_info) {
6515                         /*
6516                          * Some info has already been found but has problem
6517                          * in the last btf_ext reloc. Must have to error out.
6518                          */
6519                         pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6520                         return err;
6521                 }
6522                 /* Have problem loading the very first info. Ignore the rest. */
6523                 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6524                         prog->name);
6525         }
6526
6527 line_info:
6528         /* don't relocate line info if main program's relocation failed */
6529         if (main_prog != prog && !main_prog->line_info)
6530                 return 0;
6531
6532         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6533                                        &main_prog->line_info,
6534                                        &main_prog->line_info_cnt,
6535                                        &main_prog->line_info_rec_size);
6536         if (err) {
6537                 if (err != -ENOENT) {
6538                         pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6539                                 prog->name, err);
6540                         return err;
6541                 }
6542                 if (main_prog->line_info) {
6543                         /*
6544                          * Some info has already been found but has problem
6545                          * in the last btf_ext reloc. Must have to error out.
6546                          */
6547                         pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6548                         return err;
6549                 }
6550                 /* Have problem loading the very first info. Ignore the rest. */
6551                 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6552                         prog->name);
6553         }
6554         return 0;
6555 }
6556
6557 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6558 {
6559         size_t insn_idx = *(const size_t *)key;
6560         const struct reloc_desc *relo = elem;
6561
6562         if (insn_idx == relo->insn_idx)
6563                 return 0;
6564         return insn_idx < relo->insn_idx ? -1 : 1;
6565 }
6566
6567 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6568 {
6569         return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6570                        sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6571 }
6572
6573 static int
6574 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6575                        struct bpf_program *prog)
6576 {
6577         size_t sub_insn_idx, insn_idx, new_cnt;
6578         struct bpf_program *subprog;
6579         struct bpf_insn *insns, *insn;
6580         struct reloc_desc *relo;
6581         int err;
6582
6583         err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6584         if (err)
6585                 return err;
6586
6587         for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6588                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6589                 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
6590                         continue;
6591
6592                 relo = find_prog_insn_relo(prog, insn_idx);
6593                 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
6594                         pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6595                                 prog->name, insn_idx, relo->type);
6596                         return -LIBBPF_ERRNO__RELOC;
6597                 }
6598                 if (relo) {
6599                         /* sub-program instruction index is a combination of
6600                          * an offset of a symbol pointed to by relocation and
6601                          * call instruction's imm field; for global functions,
6602                          * call always has imm = -1, but for static functions
6603                          * relocation is against STT_SECTION and insn->imm
6604                          * points to a start of a static function
6605                          *
6606                          * for subprog addr relocation, the relo->sym_off + insn->imm is
6607                          * the byte offset in the corresponding section.
6608                          */
6609                         if (relo->type == RELO_CALL)
6610                                 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6611                         else
6612                                 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
6613                 } else if (insn_is_pseudo_func(insn)) {
6614                         /*
6615                          * RELO_SUBPROG_ADDR relo is always emitted even if both
6616                          * functions are in the same section, so it shouldn't reach here.
6617                          */
6618                         pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6619                                 prog->name, insn_idx);
6620                         return -LIBBPF_ERRNO__RELOC;
6621                 } else {
6622                         /* if subprogram call is to a static function within
6623                          * the same ELF section, there won't be any relocation
6624                          * emitted, but it also means there is no additional
6625                          * offset necessary, insns->imm is relative to
6626                          * instruction's original position within the section
6627                          */
6628                         sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6629                 }
6630
6631                 /* we enforce that sub-programs should be in .text section */
6632                 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6633                 if (!subprog) {
6634                         pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6635                                 prog->name);
6636                         return -LIBBPF_ERRNO__RELOC;
6637                 }
6638
6639                 /* if it's the first call instruction calling into this
6640                  * subprogram (meaning this subprog hasn't been processed
6641                  * yet) within the context of current main program:
6642                  *   - append it at the end of main program's instructions blog;
6643                  *   - process is recursively, while current program is put on hold;
6644                  *   - if that subprogram calls some other not yet processes
6645                  *   subprogram, same thing will happen recursively until
6646                  *   there are no more unprocesses subprograms left to append
6647                  *   and relocate.
6648                  */
6649                 if (subprog->sub_insn_off == 0) {
6650                         subprog->sub_insn_off = main_prog->insns_cnt;
6651
6652                         new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6653                         insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6654                         if (!insns) {
6655                                 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6656                                 return -ENOMEM;
6657                         }
6658                         main_prog->insns = insns;
6659                         main_prog->insns_cnt = new_cnt;
6660
6661                         memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6662                                subprog->insns_cnt * sizeof(*insns));
6663
6664                         pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6665                                  main_prog->name, subprog->insns_cnt, subprog->name);
6666
6667                         err = bpf_object__reloc_code(obj, main_prog, subprog);
6668                         if (err)
6669                                 return err;
6670                 }
6671
6672                 /* main_prog->insns memory could have been re-allocated, so
6673                  * calculate pointer again
6674                  */
6675                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6676                 /* calculate correct instruction position within current main
6677                  * prog; each main prog can have a different set of
6678                  * subprograms appended (potentially in different order as
6679                  * well), so position of any subprog can be different for
6680                  * different main programs */
6681                 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6682
6683                 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6684                          prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6685         }
6686
6687         return 0;
6688 }
6689
6690 /*
6691  * Relocate sub-program calls.
6692  *
6693  * Algorithm operates as follows. Each entry-point BPF program (referred to as
6694  * main prog) is processed separately. For each subprog (non-entry functions,
6695  * that can be called from either entry progs or other subprogs) gets their
6696  * sub_insn_off reset to zero. This serves as indicator that this subprogram
6697  * hasn't been yet appended and relocated within current main prog. Once its
6698  * relocated, sub_insn_off will point at the position within current main prog
6699  * where given subprog was appended. This will further be used to relocate all
6700  * the call instructions jumping into this subprog.
6701  *
6702  * We start with main program and process all call instructions. If the call
6703  * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6704  * is zero), subprog instructions are appended at the end of main program's
6705  * instruction array. Then main program is "put on hold" while we recursively
6706  * process newly appended subprogram. If that subprogram calls into another
6707  * subprogram that hasn't been appended, new subprogram is appended again to
6708  * the *main* prog's instructions (subprog's instructions are always left
6709  * untouched, as they need to be in unmodified state for subsequent main progs
6710  * and subprog instructions are always sent only as part of a main prog) and
6711  * the process continues recursively. Once all the subprogs called from a main
6712  * prog or any of its subprogs are appended (and relocated), all their
6713  * positions within finalized instructions array are known, so it's easy to
6714  * rewrite call instructions with correct relative offsets, corresponding to
6715  * desired target subprog.
6716  *
6717  * Its important to realize that some subprogs might not be called from some
6718  * main prog and any of its called/used subprogs. Those will keep their
6719  * subprog->sub_insn_off as zero at all times and won't be appended to current
6720  * main prog and won't be relocated within the context of current main prog.
6721  * They might still be used from other main progs later.
6722  *
6723  * Visually this process can be shown as below. Suppose we have two main
6724  * programs mainA and mainB and BPF object contains three subprogs: subA,
6725  * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6726  * subC both call subB:
6727  *
6728  *        +--------+ +-------+
6729  *        |        v v       |
6730  *     +--+---+ +--+-+-+ +---+--+
6731  *     | subA | | subB | | subC |
6732  *     +--+---+ +------+ +---+--+
6733  *        ^                  ^
6734  *        |                  |
6735  *    +---+-------+   +------+----+
6736  *    |   mainA   |   |   mainB   |
6737  *    +-----------+   +-----------+
6738  *
6739  * We'll start relocating mainA, will find subA, append it and start
6740  * processing sub A recursively:
6741  *
6742  *    +-----------+------+
6743  *    |   mainA   | subA |
6744  *    +-----------+------+
6745  *
6746  * At this point we notice that subB is used from subA, so we append it and
6747  * relocate (there are no further subcalls from subB):
6748  *
6749  *    +-----------+------+------+
6750  *    |   mainA   | subA | subB |
6751  *    +-----------+------+------+
6752  *
6753  * At this point, we relocate subA calls, then go one level up and finish with
6754  * relocatin mainA calls. mainA is done.
6755  *
6756  * For mainB process is similar but results in different order. We start with
6757  * mainB and skip subA and subB, as mainB never calls them (at least
6758  * directly), but we see subC is needed, so we append and start processing it:
6759  *
6760  *    +-----------+------+
6761  *    |   mainB   | subC |
6762  *    +-----------+------+
6763  * Now we see subC needs subB, so we go back to it, append and relocate it:
6764  *
6765  *    +-----------+------+------+
6766  *    |   mainB   | subC | subB |
6767  *    +-----------+------+------+
6768  *
6769  * At this point we unwind recursion, relocate calls in subC, then in mainB.
6770  */
6771 static int
6772 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6773 {
6774         struct bpf_program *subprog;
6775         int i, err;
6776
6777         /* mark all subprogs as not relocated (yet) within the context of
6778          * current main program
6779          */
6780         for (i = 0; i < obj->nr_programs; i++) {
6781                 subprog = &obj->programs[i];
6782                 if (!prog_is_subprog(obj, subprog))
6783                         continue;
6784
6785                 subprog->sub_insn_off = 0;
6786         }
6787
6788         err = bpf_object__reloc_code(obj, prog, prog);
6789         if (err)
6790                 return err;
6791
6792
6793         return 0;
6794 }
6795
6796 static int
6797 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6798 {
6799         struct bpf_program *prog;
6800         size_t i;
6801         int err;
6802
6803         if (obj->btf_ext) {
6804                 err = bpf_object__relocate_core(obj, targ_btf_path);
6805                 if (err) {
6806                         pr_warn("failed to perform CO-RE relocations: %d\n",
6807                                 err);
6808                         return err;
6809                 }
6810         }
6811         /* relocate data references first for all programs and sub-programs,
6812          * as they don't change relative to code locations, so subsequent
6813          * subprogram processing won't need to re-calculate any of them
6814          */
6815         for (i = 0; i < obj->nr_programs; i++) {
6816                 prog = &obj->programs[i];
6817                 err = bpf_object__relocate_data(obj, prog);
6818                 if (err) {
6819                         pr_warn("prog '%s': failed to relocate data references: %d\n",
6820                                 prog->name, err);
6821                         return err;
6822                 }
6823         }
6824         /* now relocate subprogram calls and append used subprograms to main
6825          * programs; each copy of subprogram code needs to be relocated
6826          * differently for each main program, because its code location might
6827          * have changed
6828          */
6829         for (i = 0; i < obj->nr_programs; i++) {
6830                 prog = &obj->programs[i];
6831                 /* sub-program's sub-calls are relocated within the context of
6832                  * its main program only
6833                  */
6834                 if (prog_is_subprog(obj, prog))
6835                         continue;
6836
6837                 err = bpf_object__relocate_calls(obj, prog);
6838                 if (err) {
6839                         pr_warn("prog '%s': failed to relocate calls: %d\n",
6840                                 prog->name, err);
6841                         return err;
6842                 }
6843         }
6844         /* free up relocation descriptors */
6845         for (i = 0; i < obj->nr_programs; i++) {
6846                 prog = &obj->programs[i];
6847                 zfree(&prog->reloc_desc);
6848                 prog->nr_reloc = 0;
6849         }
6850         return 0;
6851 }
6852
6853 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6854                                             GElf_Shdr *shdr, Elf_Data *data);
6855
6856 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6857                                          GElf_Shdr *shdr, Elf_Data *data)
6858 {
6859         const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6860         int i, j, nrels, new_sz;
6861         const struct btf_var_secinfo *vi = NULL;
6862         const struct btf_type *sec, *var, *def;
6863         struct bpf_map *map = NULL, *targ_map;
6864         const struct btf_member *member;
6865         const char *name, *mname;
6866         Elf_Data *symbols;
6867         unsigned int moff;
6868         GElf_Sym sym;
6869         GElf_Rel rel;
6870         void *tmp;
6871
6872         if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6873                 return -EINVAL;
6874         sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6875         if (!sec)
6876                 return -EINVAL;
6877
6878         symbols = obj->efile.symbols;
6879         nrels = shdr->sh_size / shdr->sh_entsize;
6880         for (i = 0; i < nrels; i++) {
6881                 if (!gelf_getrel(data, i, &rel)) {
6882                         pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6883                         return -LIBBPF_ERRNO__FORMAT;
6884                 }
6885                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
6886                         pr_warn(".maps relo #%d: symbol %zx not found\n",
6887                                 i, (size_t)GELF_R_SYM(rel.r_info));
6888                         return -LIBBPF_ERRNO__FORMAT;
6889                 }
6890                 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
6891                 if (sym.st_shndx != obj->efile.btf_maps_shndx) {
6892                         pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6893                                 i, name);
6894                         return -LIBBPF_ERRNO__RELOC;
6895                 }
6896
6897                 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n",
6898                          i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value,
6899                          (size_t)rel.r_offset, sym.st_name, name);
6900
6901                 for (j = 0; j < obj->nr_maps; j++) {
6902                         map = &obj->maps[j];
6903                         if (map->sec_idx != obj->efile.btf_maps_shndx)
6904                                 continue;
6905
6906                         vi = btf_var_secinfos(sec) + map->btf_var_idx;
6907                         if (vi->offset <= rel.r_offset &&
6908                             rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6909                                 break;
6910                 }
6911                 if (j == obj->nr_maps) {
6912                         pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n",
6913                                 i, name, (size_t)rel.r_offset);
6914                         return -EINVAL;
6915                 }
6916
6917                 if (!bpf_map_type__is_map_in_map(map->def.type))
6918                         return -EINVAL;
6919                 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6920                     map->def.key_size != sizeof(int)) {
6921                         pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6922                                 i, map->name, sizeof(int));
6923                         return -EINVAL;
6924                 }
6925
6926                 targ_map = bpf_object__find_map_by_name(obj, name);
6927                 if (!targ_map)
6928                         return -ESRCH;
6929
6930                 var = btf__type_by_id(obj->btf, vi->type);
6931                 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6932                 if (btf_vlen(def) == 0)
6933                         return -EINVAL;
6934                 member = btf_members(def) + btf_vlen(def) - 1;
6935                 mname = btf__name_by_offset(obj->btf, member->name_off);
6936                 if (strcmp(mname, "values"))
6937                         return -EINVAL;
6938
6939                 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6940                 if (rel.r_offset - vi->offset < moff)
6941                         return -EINVAL;
6942
6943                 moff = rel.r_offset - vi->offset - moff;
6944                 /* here we use BPF pointer size, which is always 64 bit, as we
6945                  * are parsing ELF that was built for BPF target
6946                  */
6947                 if (moff % bpf_ptr_sz)
6948                         return -EINVAL;
6949                 moff /= bpf_ptr_sz;
6950                 if (moff >= map->init_slots_sz) {
6951                         new_sz = moff + 1;
6952                         tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6953                         if (!tmp)
6954                                 return -ENOMEM;
6955                         map->init_slots = tmp;
6956                         memset(map->init_slots + map->init_slots_sz, 0,
6957                                (new_sz - map->init_slots_sz) * host_ptr_sz);
6958                         map->init_slots_sz = new_sz;
6959                 }
6960                 map->init_slots[moff] = targ_map;
6961
6962                 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n",
6963                          i, map->name, moff, name);
6964         }
6965
6966         return 0;
6967 }
6968
6969 static int cmp_relocs(const void *_a, const void *_b)
6970 {
6971         const struct reloc_desc *a = _a;
6972         const struct reloc_desc *b = _b;
6973
6974         if (a->insn_idx != b->insn_idx)
6975                 return a->insn_idx < b->insn_idx ? -1 : 1;
6976
6977         /* no two relocations should have the same insn_idx, but ... */
6978         if (a->type != b->type)
6979                 return a->type < b->type ? -1 : 1;
6980
6981         return 0;
6982 }
6983
6984 static int bpf_object__collect_relos(struct bpf_object *obj)
6985 {
6986         int i, err;
6987
6988         for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
6989                 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
6990                 Elf_Data *data = obj->efile.reloc_sects[i].data;
6991                 int idx = shdr->sh_info;
6992
6993                 if (shdr->sh_type != SHT_REL) {
6994                         pr_warn("internal error at %d\n", __LINE__);
6995                         return -LIBBPF_ERRNO__INTERNAL;
6996                 }
6997
6998                 if (idx == obj->efile.st_ops_shndx)
6999                         err = bpf_object__collect_st_ops_relos(obj, shdr, data);
7000                 else if (idx == obj->efile.btf_maps_shndx)
7001                         err = bpf_object__collect_map_relos(obj, shdr, data);
7002                 else
7003                         err = bpf_object__collect_prog_relos(obj, shdr, data);
7004                 if (err)
7005                         return err;
7006         }
7007
7008         for (i = 0; i < obj->nr_programs; i++) {
7009                 struct bpf_program *p = &obj->programs[i];
7010
7011                 if (!p->nr_reloc)
7012                         continue;
7013
7014                 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
7015         }
7016         return 0;
7017 }
7018
7019 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
7020 {
7021         if (BPF_CLASS(insn->code) == BPF_JMP &&
7022             BPF_OP(insn->code) == BPF_CALL &&
7023             BPF_SRC(insn->code) == BPF_K &&
7024             insn->src_reg == 0 &&
7025             insn->dst_reg == 0) {
7026                     *func_id = insn->imm;
7027                     return true;
7028         }
7029         return false;
7030 }
7031
7032 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
7033 {
7034         struct bpf_insn *insn = prog->insns;
7035         enum bpf_func_id func_id;
7036         int i;
7037
7038         for (i = 0; i < prog->insns_cnt; i++, insn++) {
7039                 if (!insn_is_helper_call(insn, &func_id))
7040                         continue;
7041
7042                 /* on kernels that don't yet support
7043                  * bpf_probe_read_{kernel,user}[_str] helpers, fall back
7044                  * to bpf_probe_read() which works well for old kernels
7045                  */
7046                 switch (func_id) {
7047                 case BPF_FUNC_probe_read_kernel:
7048                 case BPF_FUNC_probe_read_user:
7049                         if (!kernel_supports(FEAT_PROBE_READ_KERN))
7050                                 insn->imm = BPF_FUNC_probe_read;
7051                         break;
7052                 case BPF_FUNC_probe_read_kernel_str:
7053                 case BPF_FUNC_probe_read_user_str:
7054                         if (!kernel_supports(FEAT_PROBE_READ_KERN))
7055                                 insn->imm = BPF_FUNC_probe_read_str;
7056                         break;
7057                 default:
7058                         break;
7059                 }
7060         }
7061         return 0;
7062 }
7063
7064 static int
7065 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
7066              char *license, __u32 kern_version, int *pfd)
7067 {
7068         struct bpf_prog_load_params load_attr = {};
7069         char *cp, errmsg[STRERR_BUFSIZE];
7070         size_t log_buf_size = 0;
7071         char *log_buf = NULL;
7072         int btf_fd, ret;
7073
7074         if (prog->type == BPF_PROG_TYPE_UNSPEC) {
7075                 /*
7076                  * The program type must be set.  Most likely we couldn't find a proper
7077                  * section definition at load time, and thus we didn't infer the type.
7078                  */
7079                 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
7080                         prog->name, prog->sec_name);
7081                 return -EINVAL;
7082         }
7083
7084         if (!insns || !insns_cnt)
7085                 return -EINVAL;
7086
7087         load_attr.prog_type = prog->type;
7088         /* old kernels might not support specifying expected_attach_type */
7089         if (!kernel_supports(FEAT_EXP_ATTACH_TYPE) && prog->sec_def &&
7090             prog->sec_def->is_exp_attach_type_optional)
7091                 load_attr.expected_attach_type = 0;
7092         else
7093                 load_attr.expected_attach_type = prog->expected_attach_type;
7094         if (kernel_supports(FEAT_PROG_NAME))
7095                 load_attr.name = prog->name;
7096         load_attr.insns = insns;
7097         load_attr.insn_cnt = insns_cnt;
7098         load_attr.license = license;
7099         load_attr.attach_btf_id = prog->attach_btf_id;
7100         if (prog->attach_prog_fd)
7101                 load_attr.attach_prog_fd = prog->attach_prog_fd;
7102         else
7103                 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7104         load_attr.attach_btf_id = prog->attach_btf_id;
7105         load_attr.kern_version = kern_version;
7106         load_attr.prog_ifindex = prog->prog_ifindex;
7107
7108         /* specify func_info/line_info only if kernel supports them */
7109         btf_fd = bpf_object__btf_fd(prog->obj);
7110         if (btf_fd >= 0 && kernel_supports(FEAT_BTF_FUNC)) {
7111                 load_attr.prog_btf_fd = btf_fd;
7112                 load_attr.func_info = prog->func_info;
7113                 load_attr.func_info_rec_size = prog->func_info_rec_size;
7114                 load_attr.func_info_cnt = prog->func_info_cnt;
7115                 load_attr.line_info = prog->line_info;
7116                 load_attr.line_info_rec_size = prog->line_info_rec_size;
7117                 load_attr.line_info_cnt = prog->line_info_cnt;
7118         }
7119         load_attr.log_level = prog->log_level;
7120         load_attr.prog_flags = prog->prog_flags;
7121
7122 retry_load:
7123         if (log_buf_size) {
7124                 log_buf = malloc(log_buf_size);
7125                 if (!log_buf)
7126                         return -ENOMEM;
7127
7128                 *log_buf = 0;
7129         }
7130
7131         load_attr.log_buf = log_buf;
7132         load_attr.log_buf_sz = log_buf_size;
7133         ret = libbpf__bpf_prog_load(&load_attr);
7134
7135         if (ret >= 0) {
7136                 if (log_buf && load_attr.log_level)
7137                         pr_debug("verifier log:\n%s", log_buf);
7138
7139                 if (prog->obj->rodata_map_idx >= 0 &&
7140                     kernel_supports(FEAT_PROG_BIND_MAP)) {
7141                         struct bpf_map *rodata_map =
7142                                 &prog->obj->maps[prog->obj->rodata_map_idx];
7143
7144                         if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) {
7145                                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7146                                 pr_warn("prog '%s': failed to bind .rodata map: %s\n",
7147                                         prog->name, cp);
7148                                 /* Don't fail hard if can't bind rodata. */
7149                         }
7150                 }
7151
7152                 *pfd = ret;
7153                 ret = 0;
7154                 goto out;
7155         }
7156
7157         if (!log_buf || errno == ENOSPC) {
7158                 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
7159                                    log_buf_size << 1);
7160
7161                 free(log_buf);
7162                 goto retry_load;
7163         }
7164         ret = errno ? -errno : -LIBBPF_ERRNO__LOAD;
7165         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7166         pr_warn("load bpf program failed: %s\n", cp);
7167         pr_perm_msg(ret);
7168
7169         if (log_buf && log_buf[0] != '\0') {
7170                 ret = -LIBBPF_ERRNO__VERIFY;
7171                 pr_warn("-- BEGIN DUMP LOG ---\n");
7172                 pr_warn("\n%s\n", log_buf);
7173                 pr_warn("-- END LOG --\n");
7174         } else if (load_attr.insn_cnt >= BPF_MAXINSNS) {
7175                 pr_warn("Program too large (%zu insns), at most %d insns\n",
7176                         load_attr.insn_cnt, BPF_MAXINSNS);
7177                 ret = -LIBBPF_ERRNO__PROG2BIG;
7178         } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
7179                 /* Wrong program type? */
7180                 int fd;
7181
7182                 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
7183                 load_attr.expected_attach_type = 0;
7184                 load_attr.log_buf = NULL;
7185                 load_attr.log_buf_sz = 0;
7186                 fd = libbpf__bpf_prog_load(&load_attr);
7187                 if (fd >= 0) {
7188                         close(fd);
7189                         ret = -LIBBPF_ERRNO__PROGTYPE;
7190                         goto out;
7191                 }
7192         }
7193
7194 out:
7195         free(log_buf);
7196         return ret;
7197 }
7198
7199 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id);
7200
7201 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
7202 {
7203         int err = 0, fd, i;
7204
7205         if (prog->obj->loaded) {
7206                 pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
7207                 return -EINVAL;
7208         }
7209
7210         if ((prog->type == BPF_PROG_TYPE_TRACING ||
7211              prog->type == BPF_PROG_TYPE_LSM ||
7212              prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
7213                 int btf_obj_fd = 0, btf_type_id = 0;
7214
7215                 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id);
7216                 if (err)
7217                         return err;
7218
7219                 prog->attach_btf_obj_fd = btf_obj_fd;
7220                 prog->attach_btf_id = btf_type_id;
7221         }
7222
7223         if (prog->instances.nr < 0 || !prog->instances.fds) {
7224                 if (prog->preprocessor) {
7225                         pr_warn("Internal error: can't load program '%s'\n",
7226                                 prog->name);
7227                         return -LIBBPF_ERRNO__INTERNAL;
7228                 }
7229
7230                 prog->instances.fds = malloc(sizeof(int));
7231                 if (!prog->instances.fds) {
7232                         pr_warn("Not enough memory for BPF fds\n");
7233                         return -ENOMEM;
7234                 }
7235                 prog->instances.nr = 1;
7236                 prog->instances.fds[0] = -1;
7237         }
7238
7239         if (!prog->preprocessor) {
7240                 if (prog->instances.nr != 1) {
7241                         pr_warn("prog '%s': inconsistent nr(%d) != 1\n",
7242                                 prog->name, prog->instances.nr);
7243                 }
7244                 err = load_program(prog, prog->insns, prog->insns_cnt,
7245                                    license, kern_ver, &fd);
7246                 if (!err)
7247                         prog->instances.fds[0] = fd;
7248                 goto out;
7249         }
7250
7251         for (i = 0; i < prog->instances.nr; i++) {
7252                 struct bpf_prog_prep_result result;
7253                 bpf_program_prep_t preprocessor = prog->preprocessor;
7254
7255                 memset(&result, 0, sizeof(result));
7256                 err = preprocessor(prog, i, prog->insns,
7257                                    prog->insns_cnt, &result);
7258                 if (err) {
7259                         pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
7260                                 i, prog->name);
7261                         goto out;
7262                 }
7263
7264                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
7265                         pr_debug("Skip loading the %dth instance of program '%s'\n",
7266                                  i, prog->name);
7267                         prog->instances.fds[i] = -1;
7268                         if (result.pfd)
7269                                 *result.pfd = -1;
7270                         continue;
7271                 }
7272
7273                 err = load_program(prog, result.new_insn_ptr,
7274                                    result.new_insn_cnt, license, kern_ver, &fd);
7275                 if (err) {
7276                         pr_warn("Loading the %dth instance of program '%s' failed\n",
7277                                 i, prog->name);
7278                         goto out;
7279                 }
7280
7281                 if (result.pfd)
7282                         *result.pfd = fd;
7283                 prog->instances.fds[i] = fd;
7284         }
7285 out:
7286         if (err)
7287                 pr_warn("failed to load program '%s'\n", prog->name);
7288         zfree(&prog->insns);
7289         prog->insns_cnt = 0;
7290         return err;
7291 }
7292
7293 static int
7294 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7295 {
7296         struct bpf_program *prog;
7297         size_t i;
7298         int err;
7299
7300         for (i = 0; i < obj->nr_programs; i++) {
7301                 prog = &obj->programs[i];
7302                 err = bpf_object__sanitize_prog(obj, prog);
7303                 if (err)
7304                         return err;
7305         }
7306
7307         for (i = 0; i < obj->nr_programs; i++) {
7308                 prog = &obj->programs[i];
7309                 if (prog_is_subprog(obj, prog))
7310                         continue;
7311                 if (!prog->load) {
7312                         pr_debug("prog '%s': skipped loading\n", prog->name);
7313                         continue;
7314                 }
7315                 prog->log_level |= log_level;
7316                 err = bpf_program__load(prog, obj->license, obj->kern_version);
7317                 if (err)
7318                         return err;
7319         }
7320         return 0;
7321 }
7322
7323 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7324
7325 static struct bpf_object *
7326 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7327                    const struct bpf_object_open_opts *opts)
7328 {
7329         const char *obj_name, *kconfig;
7330         struct bpf_program *prog;
7331         struct bpf_object *obj;
7332         char tmp_name[64];
7333         int err;
7334
7335         if (elf_version(EV_CURRENT) == EV_NONE) {
7336                 pr_warn("failed to init libelf for %s\n",
7337                         path ? : "(mem buf)");
7338                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7339         }
7340
7341         if (!OPTS_VALID(opts, bpf_object_open_opts))
7342                 return ERR_PTR(-EINVAL);
7343
7344         obj_name = OPTS_GET(opts, object_name, NULL);
7345         if (obj_buf) {
7346                 if (!obj_name) {
7347                         snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7348                                  (unsigned long)obj_buf,
7349                                  (unsigned long)obj_buf_sz);
7350                         obj_name = tmp_name;
7351                 }
7352                 path = obj_name;
7353                 pr_debug("loading object '%s' from buffer\n", obj_name);
7354         }
7355
7356         obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7357         if (IS_ERR(obj))
7358                 return obj;
7359
7360         kconfig = OPTS_GET(opts, kconfig, NULL);
7361         if (kconfig) {
7362                 obj->kconfig = strdup(kconfig);
7363                 if (!obj->kconfig)
7364                         return ERR_PTR(-ENOMEM);
7365         }
7366
7367         err = bpf_object__elf_init(obj);
7368         err = err ? : bpf_object__check_endianness(obj);
7369         err = err ? : bpf_object__elf_collect(obj);
7370         err = err ? : bpf_object__collect_externs(obj);
7371         err = err ? : bpf_object__finalize_btf(obj);
7372         err = err ? : bpf_object__init_maps(obj, opts);
7373         err = err ? : bpf_object__collect_relos(obj);
7374         if (err)
7375                 goto out;
7376         bpf_object__elf_finish(obj);
7377
7378         bpf_object__for_each_program(prog, obj) {
7379                 prog->sec_def = find_sec_def(prog->sec_name);
7380                 if (!prog->sec_def) {
7381                         /* couldn't guess, but user might manually specify */
7382                         pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7383                                 prog->name, prog->sec_name);
7384                         continue;
7385                 }
7386
7387                 if (prog->sec_def->is_sleepable)
7388                         prog->prog_flags |= BPF_F_SLEEPABLE;
7389                 bpf_program__set_type(prog, prog->sec_def->prog_type);
7390                 bpf_program__set_expected_attach_type(prog,
7391                                 prog->sec_def->expected_attach_type);
7392
7393                 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
7394                     prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
7395                         prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
7396         }
7397
7398         return obj;
7399 out:
7400         bpf_object__close(obj);
7401         return ERR_PTR(err);
7402 }
7403
7404 static struct bpf_object *
7405 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
7406 {
7407         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7408                 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
7409         );
7410
7411         /* param validation */
7412         if (!attr->file)
7413                 return NULL;
7414
7415         pr_debug("loading %s\n", attr->file);
7416         return __bpf_object__open(attr->file, NULL, 0, &opts);
7417 }
7418
7419 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
7420 {
7421         return __bpf_object__open_xattr(attr, 0);
7422 }
7423
7424 struct bpf_object *bpf_object__open(const char *path)
7425 {
7426         struct bpf_object_open_attr attr = {
7427                 .file           = path,
7428                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
7429         };
7430
7431         return bpf_object__open_xattr(&attr);
7432 }
7433
7434 struct bpf_object *
7435 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7436 {
7437         if (!path)
7438                 return ERR_PTR(-EINVAL);
7439
7440         pr_debug("loading %s\n", path);
7441
7442         return __bpf_object__open(path, NULL, 0, opts);
7443 }
7444
7445 struct bpf_object *
7446 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7447                      const struct bpf_object_open_opts *opts)
7448 {
7449         if (!obj_buf || obj_buf_sz == 0)
7450                 return ERR_PTR(-EINVAL);
7451
7452         return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
7453 }
7454
7455 struct bpf_object *
7456 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
7457                         const char *name)
7458 {
7459         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7460                 .object_name = name,
7461                 /* wrong default, but backwards-compatible */
7462                 .relaxed_maps = true,
7463         );
7464
7465         /* returning NULL is wrong, but backwards-compatible */
7466         if (!obj_buf || obj_buf_sz == 0)
7467                 return NULL;
7468
7469         return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
7470 }
7471
7472 int bpf_object__unload(struct bpf_object *obj)
7473 {
7474         size_t i;
7475
7476         if (!obj)
7477                 return -EINVAL;
7478
7479         for (i = 0; i < obj->nr_maps; i++) {
7480                 zclose(obj->maps[i].fd);
7481                 if (obj->maps[i].st_ops)
7482                         zfree(&obj->maps[i].st_ops->kern_vdata);
7483         }
7484
7485         for (i = 0; i < obj->nr_programs; i++)
7486                 bpf_program__unload(&obj->programs[i]);
7487
7488         return 0;
7489 }
7490
7491 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7492 {
7493         struct bpf_map *m;
7494
7495         bpf_object__for_each_map(m, obj) {
7496                 if (!bpf_map__is_internal(m))
7497                         continue;
7498                 if (!kernel_supports(FEAT_GLOBAL_DATA)) {
7499                         pr_warn("kernel doesn't support global data\n");
7500                         return -ENOTSUP;
7501                 }
7502                 if (!kernel_supports(FEAT_ARRAY_MMAP))
7503                         m->def.map_flags ^= BPF_F_MMAPABLE;
7504         }
7505
7506         return 0;
7507 }
7508
7509 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7510 {
7511         char sym_type, sym_name[500];
7512         unsigned long long sym_addr;
7513         const struct btf_type *t;
7514         struct extern_desc *ext;
7515         int ret, err = 0;
7516         FILE *f;
7517
7518         f = fopen("/proc/kallsyms", "r");
7519         if (!f) {
7520                 err = -errno;
7521                 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7522                 return err;
7523         }
7524
7525         while (true) {
7526                 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7527                              &sym_addr, &sym_type, sym_name);
7528                 if (ret == EOF && feof(f))
7529                         break;
7530                 if (ret != 3) {
7531                         pr_warn("failed to read kallsyms entry: %d\n", ret);
7532                         err = -EINVAL;
7533                         goto out;
7534                 }
7535
7536                 ext = find_extern_by_name(obj, sym_name);
7537                 if (!ext || ext->type != EXT_KSYM)
7538                         continue;
7539
7540                 t = btf__type_by_id(obj->btf, ext->btf_id);
7541                 if (!btf_is_var(t))
7542                         continue;
7543
7544                 if (ext->is_set && ext->ksym.addr != sym_addr) {
7545                         pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
7546                                 sym_name, ext->ksym.addr, sym_addr);
7547                         err = -EINVAL;
7548                         goto out;
7549                 }
7550                 if (!ext->is_set) {
7551                         ext->is_set = true;
7552                         ext->ksym.addr = sym_addr;
7553                         pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr);
7554                 }
7555         }
7556
7557 out:
7558         fclose(f);
7559         return err;
7560 }
7561
7562 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
7563                             __u16 kind, struct btf **res_btf,
7564                             int *res_btf_fd)
7565 {
7566         int i, id, btf_fd, err;
7567         struct btf *btf;
7568
7569         btf = obj->btf_vmlinux;
7570         btf_fd = 0;
7571         id = btf__find_by_name_kind(btf, ksym_name, kind);
7572
7573         if (id == -ENOENT) {
7574                 err = load_module_btfs(obj);
7575                 if (err)
7576                         return err;
7577
7578                 for (i = 0; i < obj->btf_module_cnt; i++) {
7579                         btf = obj->btf_modules[i].btf;
7580                         /* we assume module BTF FD is always >0 */
7581                         btf_fd = obj->btf_modules[i].fd;
7582                         id = btf__find_by_name_kind(btf, ksym_name, kind);
7583                         if (id != -ENOENT)
7584                                 break;
7585                 }
7586         }
7587         if (id <= 0) {
7588                 pr_warn("extern (%s ksym) '%s': failed to find BTF ID in kernel BTF(s).\n",
7589                         __btf_kind_str(kind), ksym_name);
7590                 return -ESRCH;
7591         }
7592
7593         *res_btf = btf;
7594         *res_btf_fd = btf_fd;
7595         return id;
7596 }
7597
7598 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
7599                                                struct extern_desc *ext)
7600 {
7601         const struct btf_type *targ_var, *targ_type;
7602         __u32 targ_type_id, local_type_id;
7603         const char *targ_var_name;
7604         int id, btf_fd = 0, err;
7605         struct btf *btf = NULL;
7606
7607         id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd);
7608         if (id < 0)
7609                 return id;
7610
7611         /* find local type_id */
7612         local_type_id = ext->ksym.type_id;
7613
7614         /* find target type_id */
7615         targ_var = btf__type_by_id(btf, id);
7616         targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
7617         targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
7618
7619         err = bpf_core_types_are_compat(obj->btf, local_type_id,
7620                                         btf, targ_type_id);
7621         if (err <= 0) {
7622                 const struct btf_type *local_type;
7623                 const char *targ_name, *local_name;
7624
7625                 local_type = btf__type_by_id(obj->btf, local_type_id);
7626                 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7627                 targ_name = btf__name_by_offset(btf, targ_type->name_off);
7628
7629                 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7630                         ext->name, local_type_id,
7631                         btf_kind_str(local_type), local_name, targ_type_id,
7632                         btf_kind_str(targ_type), targ_name);
7633                 return -EINVAL;
7634         }
7635
7636         ext->is_set = true;
7637         ext->ksym.kernel_btf_obj_fd = btf_fd;
7638         ext->ksym.kernel_btf_id = id;
7639         pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
7640                  ext->name, id, btf_kind_str(targ_var), targ_var_name);
7641
7642         return 0;
7643 }
7644
7645 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
7646                                                 struct extern_desc *ext)
7647 {
7648         int local_func_proto_id, kfunc_proto_id, kfunc_id;
7649         const struct btf_type *kern_func;
7650         struct btf *kern_btf = NULL;
7651         int ret, kern_btf_fd = 0;
7652
7653         local_func_proto_id = ext->ksym.type_id;
7654
7655         kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC,
7656                                     &kern_btf, &kern_btf_fd);
7657         if (kfunc_id < 0) {
7658                 pr_warn("extern (func ksym) '%s': not found in kernel BTF\n",
7659                         ext->name);
7660                 return kfunc_id;
7661         }
7662
7663         if (kern_btf != obj->btf_vmlinux) {
7664                 pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n",
7665                         ext->name);
7666                 return -ENOTSUP;
7667         }
7668
7669         kern_func = btf__type_by_id(kern_btf, kfunc_id);
7670         kfunc_proto_id = kern_func->type;
7671
7672         ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
7673                                         kern_btf, kfunc_proto_id);
7674         if (ret <= 0) {
7675                 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
7676                         ext->name, local_func_proto_id, kfunc_proto_id);
7677                 return -EINVAL;
7678         }
7679
7680         ext->is_set = true;
7681         ext->ksym.kernel_btf_obj_fd = kern_btf_fd;
7682         ext->ksym.kernel_btf_id = kfunc_id;
7683         pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
7684                  ext->name, kfunc_id);
7685
7686         return 0;
7687 }
7688
7689 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7690 {
7691         const struct btf_type *t;
7692         struct extern_desc *ext;
7693         int i, err;
7694
7695         for (i = 0; i < obj->nr_extern; i++) {
7696                 ext = &obj->externs[i];
7697                 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7698                         continue;
7699
7700                 t = btf__type_by_id(obj->btf, ext->btf_id);
7701                 if (btf_is_var(t))
7702                         err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
7703                 else
7704                         err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
7705                 if (err)
7706                         return err;
7707         }
7708         return 0;
7709 }
7710
7711 static int bpf_object__resolve_externs(struct bpf_object *obj,
7712                                        const char *extra_kconfig)
7713 {
7714         bool need_config = false, need_kallsyms = false;
7715         bool need_vmlinux_btf = false;
7716         struct extern_desc *ext;
7717         void *kcfg_data = NULL;
7718         int err, i;
7719
7720         if (obj->nr_extern == 0)
7721                 return 0;
7722
7723         if (obj->kconfig_map_idx >= 0)
7724                 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7725
7726         for (i = 0; i < obj->nr_extern; i++) {
7727                 ext = &obj->externs[i];
7728
7729                 if (ext->type == EXT_KCFG &&
7730                     strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7731                         void *ext_val = kcfg_data + ext->kcfg.data_off;
7732                         __u32 kver = get_kernel_version();
7733
7734                         if (!kver) {
7735                                 pr_warn("failed to get kernel version\n");
7736                                 return -EINVAL;
7737                         }
7738                         err = set_kcfg_value_num(ext, ext_val, kver);
7739                         if (err)
7740                                 return err;
7741                         pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver);
7742                 } else if (ext->type == EXT_KCFG &&
7743                            strncmp(ext->name, "CONFIG_", 7) == 0) {
7744                         need_config = true;
7745                 } else if (ext->type == EXT_KSYM) {
7746                         if (ext->ksym.type_id)
7747                                 need_vmlinux_btf = true;
7748                         else
7749                                 need_kallsyms = true;
7750                 } else {
7751                         pr_warn("unrecognized extern '%s'\n", ext->name);
7752                         return -EINVAL;
7753                 }
7754         }
7755         if (need_config && extra_kconfig) {
7756                 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7757                 if (err)
7758                         return -EINVAL;
7759                 need_config = false;
7760                 for (i = 0; i < obj->nr_extern; i++) {
7761                         ext = &obj->externs[i];
7762                         if (ext->type == EXT_KCFG && !ext->is_set) {
7763                                 need_config = true;
7764                                 break;
7765                         }
7766                 }
7767         }
7768         if (need_config) {
7769                 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7770                 if (err)
7771                         return -EINVAL;
7772         }
7773         if (need_kallsyms) {
7774                 err = bpf_object__read_kallsyms_file(obj);
7775                 if (err)
7776                         return -EINVAL;
7777         }
7778         if (need_vmlinux_btf) {
7779                 err = bpf_object__resolve_ksyms_btf_id(obj);
7780                 if (err)
7781                         return -EINVAL;
7782         }
7783         for (i = 0; i < obj->nr_extern; i++) {
7784                 ext = &obj->externs[i];
7785
7786                 if (!ext->is_set && !ext->is_weak) {
7787                         pr_warn("extern %s (strong) not resolved\n", ext->name);
7788                         return -ESRCH;
7789                 } else if (!ext->is_set) {
7790                         pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
7791                                  ext->name);
7792                 }
7793         }
7794
7795         return 0;
7796 }
7797
7798 int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
7799 {
7800         struct bpf_object *obj;
7801         int err, i;
7802
7803         if (!attr)
7804                 return -EINVAL;
7805         obj = attr->obj;
7806         if (!obj)
7807                 return -EINVAL;
7808
7809         if (obj->loaded) {
7810                 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7811                 return -EINVAL;
7812         }
7813
7814         err = bpf_object__probe_loading(obj);
7815         err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7816         err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7817         err = err ? : bpf_object__sanitize_and_load_btf(obj);
7818         err = err ? : bpf_object__sanitize_maps(obj);
7819         err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7820         err = err ? : bpf_object__create_maps(obj);
7821         err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
7822         err = err ? : bpf_object__load_progs(obj, attr->log_level);
7823
7824         /* clean up module BTFs */
7825         for (i = 0; i < obj->btf_module_cnt; i++) {
7826                 close(obj->btf_modules[i].fd);
7827                 btf__free(obj->btf_modules[i].btf);
7828                 free(obj->btf_modules[i].name);
7829         }
7830         free(obj->btf_modules);
7831
7832         /* clean up vmlinux BTF */
7833         btf__free(obj->btf_vmlinux);
7834         obj->btf_vmlinux = NULL;
7835
7836         obj->loaded = true; /* doesn't matter if successfully or not */
7837
7838         if (err)
7839                 goto out;
7840
7841         return 0;
7842 out:
7843         /* unpin any maps that were auto-pinned during load */
7844         for (i = 0; i < obj->nr_maps; i++)
7845                 if (obj->maps[i].pinned && !obj->maps[i].reused)
7846                         bpf_map__unpin(&obj->maps[i], NULL);
7847
7848         bpf_object__unload(obj);
7849         pr_warn("failed to load object '%s'\n", obj->path);
7850         return err;
7851 }
7852
7853 int bpf_object__load(struct bpf_object *obj)
7854 {
7855         struct bpf_object_load_attr attr = {
7856                 .obj = obj,
7857         };
7858
7859         return bpf_object__load_xattr(&attr);
7860 }
7861
7862 static int make_parent_dir(const char *path)
7863 {
7864         char *cp, errmsg[STRERR_BUFSIZE];
7865         char *dname, *dir;
7866         int err = 0;
7867
7868         dname = strdup(path);
7869         if (dname == NULL)
7870                 return -ENOMEM;
7871
7872         dir = dirname(dname);
7873         if (mkdir(dir, 0700) && errno != EEXIST)
7874                 err = -errno;
7875
7876         free(dname);
7877         if (err) {
7878                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7879                 pr_warn("failed to mkdir %s: %s\n", path, cp);
7880         }
7881         return err;
7882 }
7883
7884 static int check_path(const char *path)
7885 {
7886         char *cp, errmsg[STRERR_BUFSIZE];
7887         struct statfs st_fs;
7888         char *dname, *dir;
7889         int err = 0;
7890
7891         if (path == NULL)
7892                 return -EINVAL;
7893
7894         dname = strdup(path);
7895         if (dname == NULL)
7896                 return -ENOMEM;
7897
7898         dir = dirname(dname);
7899         if (statfs(dir, &st_fs)) {
7900                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7901                 pr_warn("failed to statfs %s: %s\n", dir, cp);
7902                 err = -errno;
7903         }
7904         free(dname);
7905
7906         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7907                 pr_warn("specified path %s is not on BPF FS\n", path);
7908                 err = -EINVAL;
7909         }
7910
7911         return err;
7912 }
7913
7914 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
7915                               int instance)
7916 {
7917         char *cp, errmsg[STRERR_BUFSIZE];
7918         int err;
7919
7920         err = make_parent_dir(path);
7921         if (err)
7922                 return err;
7923
7924         err = check_path(path);
7925         if (err)
7926                 return err;
7927
7928         if (prog == NULL) {
7929                 pr_warn("invalid program pointer\n");
7930                 return -EINVAL;
7931         }
7932
7933         if (instance < 0 || instance >= prog->instances.nr) {
7934                 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7935                         instance, prog->name, prog->instances.nr);
7936                 return -EINVAL;
7937         }
7938
7939         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
7940                 err = -errno;
7941                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7942                 pr_warn("failed to pin program: %s\n", cp);
7943                 return err;
7944         }
7945         pr_debug("pinned program '%s'\n", path);
7946
7947         return 0;
7948 }
7949
7950 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
7951                                 int instance)
7952 {
7953         int err;
7954
7955         err = check_path(path);
7956         if (err)
7957                 return err;
7958
7959         if (prog == NULL) {
7960                 pr_warn("invalid program pointer\n");
7961                 return -EINVAL;
7962         }
7963
7964         if (instance < 0 || instance >= prog->instances.nr) {
7965                 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7966                         instance, prog->name, prog->instances.nr);
7967                 return -EINVAL;
7968         }
7969
7970         err = unlink(path);
7971         if (err != 0)
7972                 return -errno;
7973         pr_debug("unpinned program '%s'\n", path);
7974
7975         return 0;
7976 }
7977
7978 int bpf_program__pin(struct bpf_program *prog, const char *path)
7979 {
7980         int i, err;
7981
7982         err = make_parent_dir(path);
7983         if (err)
7984                 return err;
7985
7986         err = check_path(path);
7987         if (err)
7988                 return err;
7989
7990         if (prog == NULL) {
7991                 pr_warn("invalid program pointer\n");
7992                 return -EINVAL;
7993         }
7994
7995         if (prog->instances.nr <= 0) {
7996                 pr_warn("no instances of prog %s to pin\n", prog->name);
7997                 return -EINVAL;
7998         }
7999
8000         if (prog->instances.nr == 1) {
8001                 /* don't create subdirs when pinning single instance */
8002                 return bpf_program__pin_instance(prog, path, 0);
8003         }
8004
8005         for (i = 0; i < prog->instances.nr; i++) {
8006                 char buf[PATH_MAX];
8007                 int len;
8008
8009                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
8010                 if (len < 0) {
8011                         err = -EINVAL;
8012                         goto err_unpin;
8013                 } else if (len >= PATH_MAX) {
8014                         err = -ENAMETOOLONG;
8015                         goto err_unpin;
8016                 }
8017
8018                 err = bpf_program__pin_instance(prog, buf, i);
8019                 if (err)
8020                         goto err_unpin;
8021         }
8022
8023         return 0;
8024
8025 err_unpin:
8026         for (i = i - 1; i >= 0; i--) {
8027                 char buf[PATH_MAX];
8028                 int len;
8029
8030                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
8031                 if (len < 0)
8032                         continue;
8033                 else if (len >= PATH_MAX)
8034                         continue;
8035
8036                 bpf_program__unpin_instance(prog, buf, i);
8037         }
8038
8039         rmdir(path);
8040
8041         return err;
8042 }
8043
8044 int bpf_program__unpin(struct bpf_program *prog, const char *path)
8045 {
8046         int i, err;
8047
8048         err = check_path(path);
8049         if (err)
8050                 return err;
8051
8052         if (prog == NULL) {
8053                 pr_warn("invalid program pointer\n");
8054                 return -EINVAL;
8055         }
8056
8057         if (prog->instances.nr <= 0) {
8058                 pr_warn("no instances of prog %s to pin\n", prog->name);
8059                 return -EINVAL;
8060         }
8061
8062         if (prog->instances.nr == 1) {
8063                 /* don't create subdirs when pinning single instance */
8064                 return bpf_program__unpin_instance(prog, path, 0);
8065         }
8066
8067         for (i = 0; i < prog->instances.nr; i++) {
8068                 char buf[PATH_MAX];
8069                 int len;
8070
8071                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
8072                 if (len < 0)
8073                         return -EINVAL;
8074                 else if (len >= PATH_MAX)
8075                         return -ENAMETOOLONG;
8076
8077                 err = bpf_program__unpin_instance(prog, buf, i);
8078                 if (err)
8079                         return err;
8080         }
8081
8082         err = rmdir(path);
8083         if (err)
8084                 return -errno;
8085
8086         return 0;
8087 }
8088
8089 int bpf_map__pin(struct bpf_map *map, const char *path)
8090 {
8091         char *cp, errmsg[STRERR_BUFSIZE];
8092         int err;
8093
8094         if (map == NULL) {
8095                 pr_warn("invalid map pointer\n");
8096                 return -EINVAL;
8097         }
8098
8099         if (map->pin_path) {
8100                 if (path && strcmp(path, map->pin_path)) {
8101                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
8102                                 bpf_map__name(map), map->pin_path, path);
8103                         return -EINVAL;
8104                 } else if (map->pinned) {
8105                         pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
8106                                  bpf_map__name(map), map->pin_path);
8107                         return 0;
8108                 }
8109         } else {
8110                 if (!path) {
8111                         pr_warn("missing a path to pin map '%s' at\n",
8112                                 bpf_map__name(map));
8113                         return -EINVAL;
8114                 } else if (map->pinned) {
8115                         pr_warn("map '%s' already pinned\n", bpf_map__name(map));
8116                         return -EEXIST;
8117                 }
8118
8119                 map->pin_path = strdup(path);
8120                 if (!map->pin_path) {
8121                         err = -errno;
8122                         goto out_err;
8123                 }
8124         }
8125
8126         err = make_parent_dir(map->pin_path);
8127         if (err)
8128                 return err;
8129
8130         err = check_path(map->pin_path);
8131         if (err)
8132                 return err;
8133
8134         if (bpf_obj_pin(map->fd, map->pin_path)) {
8135                 err = -errno;
8136                 goto out_err;
8137         }
8138
8139         map->pinned = true;
8140         pr_debug("pinned map '%s'\n", map->pin_path);
8141
8142         return 0;
8143
8144 out_err:
8145         cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
8146         pr_warn("failed to pin map: %s\n", cp);
8147         return err;
8148 }
8149
8150 int bpf_map__unpin(struct bpf_map *map, const char *path)
8151 {
8152         int err;
8153
8154         if (map == NULL) {
8155                 pr_warn("invalid map pointer\n");
8156                 return -EINVAL;
8157         }
8158
8159         if (map->pin_path) {
8160                 if (path && strcmp(path, map->pin_path)) {
8161                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
8162                                 bpf_map__name(map), map->pin_path, path);
8163                         return -EINVAL;
8164                 }
8165                 path = map->pin_path;
8166         } else if (!path) {
8167                 pr_warn("no path to unpin map '%s' from\n",
8168                         bpf_map__name(map));
8169                 return -EINVAL;
8170         }
8171
8172         err = check_path(path);
8173         if (err)
8174                 return err;
8175
8176         err = unlink(path);
8177         if (err != 0)
8178                 return -errno;
8179
8180         map->pinned = false;
8181         pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
8182
8183         return 0;
8184 }
8185
8186 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
8187 {
8188         char *new = NULL;
8189
8190         if (path) {
8191                 new = strdup(path);
8192                 if (!new)
8193                         return -errno;
8194         }
8195
8196         free(map->pin_path);
8197         map->pin_path = new;
8198         return 0;
8199 }
8200
8201 const char *bpf_map__get_pin_path(const struct bpf_map *map)
8202 {
8203         return map->pin_path;
8204 }
8205
8206 bool bpf_map__is_pinned(const struct bpf_map *map)
8207 {
8208         return map->pinned;
8209 }
8210
8211 static void sanitize_pin_path(char *s)
8212 {
8213         /* bpffs disallows periods in path names */
8214         while (*s) {
8215                 if (*s == '.')
8216                         *s = '_';
8217                 s++;
8218         }
8219 }
8220
8221 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
8222 {
8223         struct bpf_map *map;
8224         int err;
8225
8226         if (!obj)
8227                 return -ENOENT;
8228
8229         if (!obj->loaded) {
8230                 pr_warn("object not yet loaded; load it first\n");
8231                 return -ENOENT;
8232         }
8233
8234         bpf_object__for_each_map(map, obj) {
8235                 char *pin_path = NULL;
8236                 char buf[PATH_MAX];
8237
8238                 if (path) {
8239                         int len;
8240
8241                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
8242                                        bpf_map__name(map));
8243                         if (len < 0) {
8244                                 err = -EINVAL;
8245                                 goto err_unpin_maps;
8246                         } else if (len >= PATH_MAX) {
8247                                 err = -ENAMETOOLONG;
8248                                 goto err_unpin_maps;
8249                         }
8250                         sanitize_pin_path(buf);
8251                         pin_path = buf;
8252                 } else if (!map->pin_path) {
8253                         continue;
8254                 }
8255
8256                 err = bpf_map__pin(map, pin_path);
8257                 if (err)
8258                         goto err_unpin_maps;
8259         }
8260
8261         return 0;
8262
8263 err_unpin_maps:
8264         while ((map = bpf_map__prev(map, obj))) {
8265                 if (!map->pin_path)
8266                         continue;
8267
8268                 bpf_map__unpin(map, NULL);
8269         }
8270
8271         return err;
8272 }
8273
8274 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
8275 {
8276         struct bpf_map *map;
8277         int err;
8278
8279         if (!obj)
8280                 return -ENOENT;
8281
8282         bpf_object__for_each_map(map, obj) {
8283                 char *pin_path = NULL;
8284                 char buf[PATH_MAX];
8285
8286                 if (path) {
8287                         int len;
8288
8289                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
8290                                        bpf_map__name(map));
8291                         if (len < 0)
8292                                 return -EINVAL;
8293                         else if (len >= PATH_MAX)
8294                                 return -ENAMETOOLONG;
8295                         sanitize_pin_path(buf);
8296                         pin_path = buf;
8297                 } else if (!map->pin_path) {
8298                         continue;
8299                 }
8300
8301                 err = bpf_map__unpin(map, pin_path);
8302                 if (err)
8303                         return err;
8304         }
8305
8306         return 0;
8307 }
8308
8309 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
8310 {
8311         struct bpf_program *prog;
8312         int err;
8313
8314         if (!obj)
8315                 return -ENOENT;
8316
8317         if (!obj->loaded) {
8318                 pr_warn("object not yet loaded; load it first\n");
8319                 return -ENOENT;
8320         }
8321
8322         bpf_object__for_each_program(prog, obj) {
8323                 char buf[PATH_MAX];
8324                 int len;
8325
8326                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8327                                prog->pin_name);
8328                 if (len < 0) {
8329                         err = -EINVAL;
8330                         goto err_unpin_programs;
8331                 } else if (len >= PATH_MAX) {
8332                         err = -ENAMETOOLONG;
8333                         goto err_unpin_programs;
8334                 }
8335
8336                 err = bpf_program__pin(prog, buf);
8337                 if (err)
8338                         goto err_unpin_programs;
8339         }
8340
8341         return 0;
8342
8343 err_unpin_programs:
8344         while ((prog = bpf_program__prev(prog, obj))) {
8345                 char buf[PATH_MAX];
8346                 int len;
8347
8348                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8349                                prog->pin_name);
8350                 if (len < 0)
8351                         continue;
8352                 else if (len >= PATH_MAX)
8353                         continue;
8354
8355                 bpf_program__unpin(prog, buf);
8356         }
8357
8358         return err;
8359 }
8360
8361 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8362 {
8363         struct bpf_program *prog;
8364         int err;
8365
8366         if (!obj)
8367                 return -ENOENT;
8368
8369         bpf_object__for_each_program(prog, obj) {
8370                 char buf[PATH_MAX];
8371                 int len;
8372
8373                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8374                                prog->pin_name);
8375                 if (len < 0)
8376                         return -EINVAL;
8377                 else if (len >= PATH_MAX)
8378                         return -ENAMETOOLONG;
8379
8380                 err = bpf_program__unpin(prog, buf);
8381                 if (err)
8382                         return err;
8383         }
8384
8385         return 0;
8386 }
8387
8388 int bpf_object__pin(struct bpf_object *obj, const char *path)
8389 {
8390         int err;
8391
8392         err = bpf_object__pin_maps(obj, path);
8393         if (err)
8394                 return err;
8395
8396         err = bpf_object__pin_programs(obj, path);
8397         if (err) {
8398                 bpf_object__unpin_maps(obj, path);
8399                 return err;
8400         }
8401
8402         return 0;
8403 }
8404
8405 static void bpf_map__destroy(struct bpf_map *map)
8406 {
8407         if (map->clear_priv)
8408                 map->clear_priv(map, map->priv);
8409         map->priv = NULL;
8410         map->clear_priv = NULL;
8411
8412         if (map->inner_map) {
8413                 bpf_map__destroy(map->inner_map);
8414                 zfree(&map->inner_map);
8415         }
8416
8417         zfree(&map->init_slots);
8418         map->init_slots_sz = 0;
8419
8420         if (map->mmaped) {
8421                 munmap(map->mmaped, bpf_map_mmap_sz(map));
8422                 map->mmaped = NULL;
8423         }
8424
8425         if (map->st_ops) {
8426                 zfree(&map->st_ops->data);
8427                 zfree(&map->st_ops->progs);
8428                 zfree(&map->st_ops->kern_func_off);
8429                 zfree(&map->st_ops);
8430         }
8431
8432         zfree(&map->name);
8433         zfree(&map->pin_path);
8434
8435         if (map->fd >= 0)
8436                 zclose(map->fd);
8437 }
8438
8439 void bpf_object__close(struct bpf_object *obj)
8440 {
8441         size_t i;
8442
8443         if (IS_ERR_OR_NULL(obj))
8444                 return;
8445
8446         if (obj->clear_priv)
8447                 obj->clear_priv(obj, obj->priv);
8448
8449         bpf_object__elf_finish(obj);
8450         bpf_object__unload(obj);
8451         btf__free(obj->btf);
8452         btf_ext__free(obj->btf_ext);
8453
8454         for (i = 0; i < obj->nr_maps; i++)
8455                 bpf_map__destroy(&obj->maps[i]);
8456
8457         zfree(&obj->kconfig);
8458         zfree(&obj->externs);
8459         obj->nr_extern = 0;
8460
8461         zfree(&obj->maps);
8462         obj->nr_maps = 0;
8463
8464         if (obj->programs && obj->nr_programs) {
8465                 for (i = 0; i < obj->nr_programs; i++)
8466                         bpf_program__exit(&obj->programs[i]);
8467         }
8468         zfree(&obj->programs);
8469
8470         list_del(&obj->list);
8471         free(obj);
8472 }
8473
8474 struct bpf_object *
8475 bpf_object__next(struct bpf_object *prev)
8476 {
8477         struct bpf_object *next;
8478
8479         if (!prev)
8480                 next = list_first_entry(&bpf_objects_list,
8481                                         struct bpf_object,
8482                                         list);
8483         else
8484                 next = list_next_entry(prev, list);
8485
8486         /* Empty list is noticed here so don't need checking on entry. */
8487         if (&next->list == &bpf_objects_list)
8488                 return NULL;
8489
8490         return next;
8491 }
8492
8493 const char *bpf_object__name(const struct bpf_object *obj)
8494 {
8495         return obj ? obj->name : ERR_PTR(-EINVAL);
8496 }
8497
8498 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8499 {
8500         return obj ? obj->kern_version : 0;
8501 }
8502
8503 struct btf *bpf_object__btf(const struct bpf_object *obj)
8504 {
8505         return obj ? obj->btf : NULL;
8506 }
8507
8508 int bpf_object__btf_fd(const struct bpf_object *obj)
8509 {
8510         return obj->btf ? btf__fd(obj->btf) : -1;
8511 }
8512
8513 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
8514 {
8515         if (obj->loaded)
8516                 return -EINVAL;
8517
8518         obj->kern_version = kern_version;
8519
8520         return 0;
8521 }
8522
8523 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
8524                          bpf_object_clear_priv_t clear_priv)
8525 {
8526         if (obj->priv && obj->clear_priv)
8527                 obj->clear_priv(obj, obj->priv);
8528
8529         obj->priv = priv;
8530         obj->clear_priv = clear_priv;
8531         return 0;
8532 }
8533
8534 void *bpf_object__priv(const struct bpf_object *obj)
8535 {
8536         return obj ? obj->priv : ERR_PTR(-EINVAL);
8537 }
8538
8539 static struct bpf_program *
8540 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8541                     bool forward)
8542 {
8543         size_t nr_programs = obj->nr_programs;
8544         ssize_t idx;
8545
8546         if (!nr_programs)
8547                 return NULL;
8548
8549         if (!p)
8550                 /* Iter from the beginning */
8551                 return forward ? &obj->programs[0] :
8552                         &obj->programs[nr_programs - 1];
8553
8554         if (p->obj != obj) {
8555                 pr_warn("error: program handler doesn't match object\n");
8556                 return NULL;
8557         }
8558
8559         idx = (p - obj->programs) + (forward ? 1 : -1);
8560         if (idx >= obj->nr_programs || idx < 0)
8561                 return NULL;
8562         return &obj->programs[idx];
8563 }
8564
8565 struct bpf_program *
8566 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
8567 {
8568         struct bpf_program *prog = prev;
8569
8570         do {
8571                 prog = __bpf_program__iter(prog, obj, true);
8572         } while (prog && prog_is_subprog(obj, prog));
8573
8574         return prog;
8575 }
8576
8577 struct bpf_program *
8578 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
8579 {
8580         struct bpf_program *prog = next;
8581
8582         do {
8583                 prog = __bpf_program__iter(prog, obj, false);
8584         } while (prog && prog_is_subprog(obj, prog));
8585
8586         return prog;
8587 }
8588
8589 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
8590                           bpf_program_clear_priv_t clear_priv)
8591 {
8592         if (prog->priv && prog->clear_priv)
8593                 prog->clear_priv(prog, prog->priv);
8594
8595         prog->priv = priv;
8596         prog->clear_priv = clear_priv;
8597         return 0;
8598 }
8599
8600 void *bpf_program__priv(const struct bpf_program *prog)
8601 {
8602         return prog ? prog->priv : ERR_PTR(-EINVAL);
8603 }
8604
8605 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8606 {
8607         prog->prog_ifindex = ifindex;
8608 }
8609
8610 const char *bpf_program__name(const struct bpf_program *prog)
8611 {
8612         return prog->name;
8613 }
8614
8615 const char *bpf_program__section_name(const struct bpf_program *prog)
8616 {
8617         return prog->sec_name;
8618 }
8619
8620 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
8621 {
8622         const char *title;
8623
8624         title = prog->sec_name;
8625         if (needs_copy) {
8626                 title = strdup(title);
8627                 if (!title) {
8628                         pr_warn("failed to strdup program title\n");
8629                         return ERR_PTR(-ENOMEM);
8630                 }
8631         }
8632
8633         return title;
8634 }
8635
8636 bool bpf_program__autoload(const struct bpf_program *prog)
8637 {
8638         return prog->load;
8639 }
8640
8641 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8642 {
8643         if (prog->obj->loaded)
8644                 return -EINVAL;
8645
8646         prog->load = autoload;
8647         return 0;
8648 }
8649
8650 int bpf_program__fd(const struct bpf_program *prog)
8651 {
8652         return bpf_program__nth_fd(prog, 0);
8653 }
8654
8655 size_t bpf_program__size(const struct bpf_program *prog)
8656 {
8657         return prog->insns_cnt * BPF_INSN_SZ;
8658 }
8659
8660 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
8661                           bpf_program_prep_t prep)
8662 {
8663         int *instances_fds;
8664
8665         if (nr_instances <= 0 || !prep)
8666                 return -EINVAL;
8667
8668         if (prog->instances.nr > 0 || prog->instances.fds) {
8669                 pr_warn("Can't set pre-processor after loading\n");
8670                 return -EINVAL;
8671         }
8672
8673         instances_fds = malloc(sizeof(int) * nr_instances);
8674         if (!instances_fds) {
8675                 pr_warn("alloc memory failed for fds\n");
8676                 return -ENOMEM;
8677         }
8678
8679         /* fill all fd with -1 */
8680         memset(instances_fds, -1, sizeof(int) * nr_instances);
8681
8682         prog->instances.nr = nr_instances;
8683         prog->instances.fds = instances_fds;
8684         prog->preprocessor = prep;
8685         return 0;
8686 }
8687
8688 int bpf_program__nth_fd(const struct bpf_program *prog, int n)
8689 {
8690         int fd;
8691
8692         if (!prog)
8693                 return -EINVAL;
8694
8695         if (n >= prog->instances.nr || n < 0) {
8696                 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
8697                         n, prog->name, prog->instances.nr);
8698                 return -EINVAL;
8699         }
8700
8701         fd = prog->instances.fds[n];
8702         if (fd < 0) {
8703                 pr_warn("%dth instance of program '%s' is invalid\n",
8704                         n, prog->name);
8705                 return -ENOENT;
8706         }
8707
8708         return fd;
8709 }
8710
8711 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog)
8712 {
8713         return prog->type;
8714 }
8715
8716 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8717 {
8718         prog->type = type;
8719 }
8720
8721 static bool bpf_program__is_type(const struct bpf_program *prog,
8722                                  enum bpf_prog_type type)
8723 {
8724         return prog ? (prog->type == type) : false;
8725 }
8726
8727 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                           \
8728 int bpf_program__set_##NAME(struct bpf_program *prog)           \
8729 {                                                               \
8730         if (!prog)                                              \
8731                 return -EINVAL;                                 \
8732         bpf_program__set_type(prog, TYPE);                      \
8733         return 0;                                               \
8734 }                                                               \
8735                                                                 \
8736 bool bpf_program__is_##NAME(const struct bpf_program *prog)     \
8737 {                                                               \
8738         return bpf_program__is_type(prog, TYPE);                \
8739 }                                                               \
8740
8741 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
8742 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
8743 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
8744 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
8745 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
8746 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
8747 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
8748 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
8749 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
8750 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
8751 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
8752 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
8753 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
8754
8755 enum bpf_attach_type
8756 bpf_program__get_expected_attach_type(const struct bpf_program *prog)
8757 {
8758         return prog->expected_attach_type;
8759 }
8760
8761 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
8762                                            enum bpf_attach_type type)
8763 {
8764         prog->expected_attach_type = type;
8765 }
8766
8767 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional,           \
8768                           attachable, attach_btf)                           \
8769         {                                                                   \
8770                 .sec = string,                                              \
8771                 .len = sizeof(string) - 1,                                  \
8772                 .prog_type = ptype,                                         \
8773                 .expected_attach_type = eatype,                             \
8774                 .is_exp_attach_type_optional = eatype_optional,             \
8775                 .is_attachable = attachable,                                \
8776                 .is_attach_btf = attach_btf,                                \
8777         }
8778
8779 /* Programs that can NOT be attached. */
8780 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
8781
8782 /* Programs that can be attached. */
8783 #define BPF_APROG_SEC(string, ptype, atype) \
8784         BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0)
8785
8786 /* Programs that must specify expected attach type at load time. */
8787 #define BPF_EAPROG_SEC(string, ptype, eatype) \
8788         BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0)
8789
8790 /* Programs that use BTF to identify attach point */
8791 #define BPF_PROG_BTF(string, ptype, eatype) \
8792         BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1)
8793
8794 /* Programs that can be attached but attach type can't be identified by section
8795  * name. Kept for backward compatibility.
8796  */
8797 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
8798
8799 #define SEC_DEF(sec_pfx, ptype, ...) {                                      \
8800         .sec = sec_pfx,                                                     \
8801         .len = sizeof(sec_pfx) - 1,                                         \
8802         .prog_type = BPF_PROG_TYPE_##ptype,                                 \
8803         __VA_ARGS__                                                         \
8804 }
8805
8806 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
8807                                       struct bpf_program *prog);
8808 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
8809                                   struct bpf_program *prog);
8810 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
8811                                       struct bpf_program *prog);
8812 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
8813                                      struct bpf_program *prog);
8814 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
8815                                    struct bpf_program *prog);
8816 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
8817                                     struct bpf_program *prog);
8818
8819 static const struct bpf_sec_def section_defs[] = {
8820         BPF_PROG_SEC("socket",                  BPF_PROG_TYPE_SOCKET_FILTER),
8821         BPF_PROG_SEC("sk_reuseport",            BPF_PROG_TYPE_SK_REUSEPORT),
8822         SEC_DEF("kprobe/", KPROBE,
8823                 .attach_fn = attach_kprobe),
8824         BPF_PROG_SEC("uprobe/",                 BPF_PROG_TYPE_KPROBE),
8825         SEC_DEF("kretprobe/", KPROBE,
8826                 .attach_fn = attach_kprobe),
8827         BPF_PROG_SEC("uretprobe/",              BPF_PROG_TYPE_KPROBE),
8828         BPF_PROG_SEC("classifier",              BPF_PROG_TYPE_SCHED_CLS),
8829         BPF_PROG_SEC("action",                  BPF_PROG_TYPE_SCHED_ACT),
8830         SEC_DEF("tracepoint/", TRACEPOINT,
8831                 .attach_fn = attach_tp),
8832         SEC_DEF("tp/", TRACEPOINT,
8833                 .attach_fn = attach_tp),
8834         SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
8835                 .attach_fn = attach_raw_tp),
8836         SEC_DEF("raw_tp/", RAW_TRACEPOINT,
8837                 .attach_fn = attach_raw_tp),
8838         SEC_DEF("tp_btf/", TRACING,
8839                 .expected_attach_type = BPF_TRACE_RAW_TP,
8840                 .is_attach_btf = true,
8841                 .attach_fn = attach_trace),
8842         SEC_DEF("fentry/", TRACING,
8843                 .expected_attach_type = BPF_TRACE_FENTRY,
8844                 .is_attach_btf = true,
8845                 .attach_fn = attach_trace),
8846         SEC_DEF("fmod_ret/", TRACING,
8847                 .expected_attach_type = BPF_MODIFY_RETURN,
8848                 .is_attach_btf = true,
8849                 .attach_fn = attach_trace),
8850         SEC_DEF("fexit/", TRACING,
8851                 .expected_attach_type = BPF_TRACE_FEXIT,
8852                 .is_attach_btf = true,
8853                 .attach_fn = attach_trace),
8854         SEC_DEF("fentry.s/", TRACING,
8855                 .expected_attach_type = BPF_TRACE_FENTRY,
8856                 .is_attach_btf = true,
8857                 .is_sleepable = true,
8858                 .attach_fn = attach_trace),
8859         SEC_DEF("fmod_ret.s/", TRACING,
8860                 .expected_attach_type = BPF_MODIFY_RETURN,
8861                 .is_attach_btf = true,
8862                 .is_sleepable = true,
8863                 .attach_fn = attach_trace),
8864         SEC_DEF("fexit.s/", TRACING,
8865                 .expected_attach_type = BPF_TRACE_FEXIT,
8866                 .is_attach_btf = true,
8867                 .is_sleepable = true,
8868                 .attach_fn = attach_trace),
8869         SEC_DEF("freplace/", EXT,
8870                 .is_attach_btf = true,
8871                 .attach_fn = attach_trace),
8872         SEC_DEF("lsm/", LSM,
8873                 .is_attach_btf = true,
8874                 .expected_attach_type = BPF_LSM_MAC,
8875                 .attach_fn = attach_lsm),
8876         SEC_DEF("lsm.s/", LSM,
8877                 .is_attach_btf = true,
8878                 .is_sleepable = true,
8879                 .expected_attach_type = BPF_LSM_MAC,
8880                 .attach_fn = attach_lsm),
8881         SEC_DEF("iter/", TRACING,
8882                 .expected_attach_type = BPF_TRACE_ITER,
8883                 .is_attach_btf = true,
8884                 .attach_fn = attach_iter),
8885         BPF_EAPROG_SEC("xdp_devmap/",           BPF_PROG_TYPE_XDP,
8886                                                 BPF_XDP_DEVMAP),
8887         BPF_EAPROG_SEC("xdp_cpumap/",           BPF_PROG_TYPE_XDP,
8888                                                 BPF_XDP_CPUMAP),
8889         BPF_APROG_SEC("xdp",                    BPF_PROG_TYPE_XDP,
8890                                                 BPF_XDP),
8891         BPF_PROG_SEC("perf_event",              BPF_PROG_TYPE_PERF_EVENT),
8892         BPF_PROG_SEC("lwt_in",                  BPF_PROG_TYPE_LWT_IN),
8893         BPF_PROG_SEC("lwt_out",                 BPF_PROG_TYPE_LWT_OUT),
8894         BPF_PROG_SEC("lwt_xmit",                BPF_PROG_TYPE_LWT_XMIT),
8895         BPF_PROG_SEC("lwt_seg6local",           BPF_PROG_TYPE_LWT_SEG6LOCAL),
8896         BPF_APROG_SEC("cgroup_skb/ingress",     BPF_PROG_TYPE_CGROUP_SKB,
8897                                                 BPF_CGROUP_INET_INGRESS),
8898         BPF_APROG_SEC("cgroup_skb/egress",      BPF_PROG_TYPE_CGROUP_SKB,
8899                                                 BPF_CGROUP_INET_EGRESS),
8900         BPF_APROG_COMPAT("cgroup/skb",          BPF_PROG_TYPE_CGROUP_SKB),
8901         BPF_EAPROG_SEC("cgroup/sock_create",    BPF_PROG_TYPE_CGROUP_SOCK,
8902                                                 BPF_CGROUP_INET_SOCK_CREATE),
8903         BPF_EAPROG_SEC("cgroup/sock_release",   BPF_PROG_TYPE_CGROUP_SOCK,
8904                                                 BPF_CGROUP_INET_SOCK_RELEASE),
8905         BPF_APROG_SEC("cgroup/sock",            BPF_PROG_TYPE_CGROUP_SOCK,
8906                                                 BPF_CGROUP_INET_SOCK_CREATE),
8907         BPF_EAPROG_SEC("cgroup/post_bind4",     BPF_PROG_TYPE_CGROUP_SOCK,
8908                                                 BPF_CGROUP_INET4_POST_BIND),
8909         BPF_EAPROG_SEC("cgroup/post_bind6",     BPF_PROG_TYPE_CGROUP_SOCK,
8910                                                 BPF_CGROUP_INET6_POST_BIND),
8911         BPF_APROG_SEC("cgroup/dev",             BPF_PROG_TYPE_CGROUP_DEVICE,
8912                                                 BPF_CGROUP_DEVICE),
8913         BPF_APROG_SEC("sockops",                BPF_PROG_TYPE_SOCK_OPS,
8914                                                 BPF_CGROUP_SOCK_OPS),
8915         BPF_APROG_SEC("sk_skb/stream_parser",   BPF_PROG_TYPE_SK_SKB,
8916                                                 BPF_SK_SKB_STREAM_PARSER),
8917         BPF_APROG_SEC("sk_skb/stream_verdict",  BPF_PROG_TYPE_SK_SKB,
8918                                                 BPF_SK_SKB_STREAM_VERDICT),
8919         BPF_APROG_COMPAT("sk_skb",              BPF_PROG_TYPE_SK_SKB),
8920         BPF_APROG_SEC("sk_msg",                 BPF_PROG_TYPE_SK_MSG,
8921                                                 BPF_SK_MSG_VERDICT),
8922         BPF_APROG_SEC("lirc_mode2",             BPF_PROG_TYPE_LIRC_MODE2,
8923                                                 BPF_LIRC_MODE2),
8924         BPF_APROG_SEC("flow_dissector",         BPF_PROG_TYPE_FLOW_DISSECTOR,
8925                                                 BPF_FLOW_DISSECTOR),
8926         BPF_EAPROG_SEC("cgroup/bind4",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8927                                                 BPF_CGROUP_INET4_BIND),
8928         BPF_EAPROG_SEC("cgroup/bind6",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8929                                                 BPF_CGROUP_INET6_BIND),
8930         BPF_EAPROG_SEC("cgroup/connect4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8931                                                 BPF_CGROUP_INET4_CONNECT),
8932         BPF_EAPROG_SEC("cgroup/connect6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8933                                                 BPF_CGROUP_INET6_CONNECT),
8934         BPF_EAPROG_SEC("cgroup/sendmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8935                                                 BPF_CGROUP_UDP4_SENDMSG),
8936         BPF_EAPROG_SEC("cgroup/sendmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8937                                                 BPF_CGROUP_UDP6_SENDMSG),
8938         BPF_EAPROG_SEC("cgroup/recvmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8939                                                 BPF_CGROUP_UDP4_RECVMSG),
8940         BPF_EAPROG_SEC("cgroup/recvmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8941                                                 BPF_CGROUP_UDP6_RECVMSG),
8942         BPF_EAPROG_SEC("cgroup/getpeername4",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8943                                                 BPF_CGROUP_INET4_GETPEERNAME),
8944         BPF_EAPROG_SEC("cgroup/getpeername6",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8945                                                 BPF_CGROUP_INET6_GETPEERNAME),
8946         BPF_EAPROG_SEC("cgroup/getsockname4",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8947                                                 BPF_CGROUP_INET4_GETSOCKNAME),
8948         BPF_EAPROG_SEC("cgroup/getsockname6",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8949                                                 BPF_CGROUP_INET6_GETSOCKNAME),
8950         BPF_EAPROG_SEC("cgroup/sysctl",         BPF_PROG_TYPE_CGROUP_SYSCTL,
8951                                                 BPF_CGROUP_SYSCTL),
8952         BPF_EAPROG_SEC("cgroup/getsockopt",     BPF_PROG_TYPE_CGROUP_SOCKOPT,
8953                                                 BPF_CGROUP_GETSOCKOPT),
8954         BPF_EAPROG_SEC("cgroup/setsockopt",     BPF_PROG_TYPE_CGROUP_SOCKOPT,
8955                                                 BPF_CGROUP_SETSOCKOPT),
8956         BPF_PROG_SEC("struct_ops",              BPF_PROG_TYPE_STRUCT_OPS),
8957         BPF_EAPROG_SEC("sk_lookup/",            BPF_PROG_TYPE_SK_LOOKUP,
8958                                                 BPF_SK_LOOKUP),
8959 };
8960
8961 #undef BPF_PROG_SEC_IMPL
8962 #undef BPF_PROG_SEC
8963 #undef BPF_APROG_SEC
8964 #undef BPF_EAPROG_SEC
8965 #undef BPF_APROG_COMPAT
8966 #undef SEC_DEF
8967
8968 #define MAX_TYPE_NAME_SIZE 32
8969
8970 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8971 {
8972         int i, n = ARRAY_SIZE(section_defs);
8973
8974         for (i = 0; i < n; i++) {
8975                 if (strncmp(sec_name,
8976                             section_defs[i].sec, section_defs[i].len))
8977                         continue;
8978                 return &section_defs[i];
8979         }
8980         return NULL;
8981 }
8982
8983 static char *libbpf_get_type_names(bool attach_type)
8984 {
8985         int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8986         char *buf;
8987
8988         buf = malloc(len);
8989         if (!buf)
8990                 return NULL;
8991
8992         buf[0] = '\0';
8993         /* Forge string buf with all available names */
8994         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8995                 if (attach_type && !section_defs[i].is_attachable)
8996                         continue;
8997
8998                 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8999                         free(buf);
9000                         return NULL;
9001                 }
9002                 strcat(buf, " ");
9003                 strcat(buf, section_defs[i].sec);
9004         }
9005
9006         return buf;
9007 }
9008
9009 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
9010                              enum bpf_attach_type *expected_attach_type)
9011 {
9012         const struct bpf_sec_def *sec_def;
9013         char *type_names;
9014
9015         if (!name)
9016                 return -EINVAL;
9017
9018         sec_def = find_sec_def(name);
9019         if (sec_def) {
9020                 *prog_type = sec_def->prog_type;
9021                 *expected_attach_type = sec_def->expected_attach_type;
9022                 return 0;
9023         }
9024
9025         pr_debug("failed to guess program type from ELF section '%s'\n", name);
9026         type_names = libbpf_get_type_names(false);
9027         if (type_names != NULL) {
9028                 pr_debug("supported section(type) names are:%s\n", type_names);
9029                 free(type_names);
9030         }
9031
9032         return -ESRCH;
9033 }
9034
9035 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
9036                                                      size_t offset)
9037 {
9038         struct bpf_map *map;
9039         size_t i;
9040
9041         for (i = 0; i < obj->nr_maps; i++) {
9042                 map = &obj->maps[i];
9043                 if (!bpf_map__is_struct_ops(map))
9044                         continue;
9045                 if (map->sec_offset <= offset &&
9046                     offset - map->sec_offset < map->def.value_size)
9047                         return map;
9048         }
9049
9050         return NULL;
9051 }
9052
9053 /* Collect the reloc from ELF and populate the st_ops->progs[] */
9054 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
9055                                             GElf_Shdr *shdr, Elf_Data *data)
9056 {
9057         const struct btf_member *member;
9058         struct bpf_struct_ops *st_ops;
9059         struct bpf_program *prog;
9060         unsigned int shdr_idx;
9061         const struct btf *btf;
9062         struct bpf_map *map;
9063         Elf_Data *symbols;
9064         unsigned int moff, insn_idx;
9065         const char *name;
9066         __u32 member_idx;
9067         GElf_Sym sym;
9068         GElf_Rel rel;
9069         int i, nrels;
9070
9071         symbols = obj->efile.symbols;
9072         btf = obj->btf;
9073         nrels = shdr->sh_size / shdr->sh_entsize;
9074         for (i = 0; i < nrels; i++) {
9075                 if (!gelf_getrel(data, i, &rel)) {
9076                         pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
9077                         return -LIBBPF_ERRNO__FORMAT;
9078                 }
9079
9080                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
9081                         pr_warn("struct_ops reloc: symbol %zx not found\n",
9082                                 (size_t)GELF_R_SYM(rel.r_info));
9083                         return -LIBBPF_ERRNO__FORMAT;
9084                 }
9085
9086                 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
9087                 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
9088                 if (!map) {
9089                         pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
9090                                 (size_t)rel.r_offset);
9091                         return -EINVAL;
9092                 }
9093
9094                 moff = rel.r_offset - map->sec_offset;
9095                 shdr_idx = sym.st_shndx;
9096                 st_ops = map->st_ops;
9097                 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
9098                          map->name,
9099                          (long long)(rel.r_info >> 32),
9100                          (long long)sym.st_value,
9101                          shdr_idx, (size_t)rel.r_offset,
9102                          map->sec_offset, sym.st_name, name);
9103
9104                 if (shdr_idx >= SHN_LORESERVE) {
9105                         pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
9106                                 map->name, (size_t)rel.r_offset, shdr_idx);
9107                         return -LIBBPF_ERRNO__RELOC;
9108                 }
9109                 if (sym.st_value % BPF_INSN_SZ) {
9110                         pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
9111                                 map->name, (unsigned long long)sym.st_value);
9112                         return -LIBBPF_ERRNO__FORMAT;
9113                 }
9114                 insn_idx = sym.st_value / BPF_INSN_SZ;
9115
9116                 member = find_member_by_offset(st_ops->type, moff * 8);
9117                 if (!member) {
9118                         pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
9119                                 map->name, moff);
9120                         return -EINVAL;
9121                 }
9122                 member_idx = member - btf_members(st_ops->type);
9123                 name = btf__name_by_offset(btf, member->name_off);
9124
9125                 if (!resolve_func_ptr(btf, member->type, NULL)) {
9126                         pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
9127                                 map->name, name);
9128                         return -EINVAL;
9129                 }
9130
9131                 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
9132                 if (!prog) {
9133                         pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
9134                                 map->name, shdr_idx, name);
9135                         return -EINVAL;
9136                 }
9137
9138                 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
9139                         const struct bpf_sec_def *sec_def;
9140
9141                         sec_def = find_sec_def(prog->sec_name);
9142                         if (sec_def &&
9143                             sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
9144                                 /* for pr_warn */
9145                                 prog->type = sec_def->prog_type;
9146                                 goto invalid_prog;
9147                         }
9148
9149                         prog->type = BPF_PROG_TYPE_STRUCT_OPS;
9150                         prog->attach_btf_id = st_ops->type_id;
9151                         prog->expected_attach_type = member_idx;
9152                 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
9153                            prog->attach_btf_id != st_ops->type_id ||
9154                            prog->expected_attach_type != member_idx) {
9155                         goto invalid_prog;
9156                 }
9157                 st_ops->progs[member_idx] = prog;
9158         }
9159
9160         return 0;
9161
9162 invalid_prog:
9163         pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
9164                 map->name, prog->name, prog->sec_name, prog->type,
9165                 prog->attach_btf_id, prog->expected_attach_type, name);
9166         return -EINVAL;
9167 }
9168
9169 #define BTF_TRACE_PREFIX "btf_trace_"
9170 #define BTF_LSM_PREFIX "bpf_lsm_"
9171 #define BTF_ITER_PREFIX "bpf_iter_"
9172 #define BTF_MAX_NAME_SIZE 128
9173
9174 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
9175                                    const char *name, __u32 kind)
9176 {
9177         char btf_type_name[BTF_MAX_NAME_SIZE];
9178         int ret;
9179
9180         ret = snprintf(btf_type_name, sizeof(btf_type_name),
9181                        "%s%s", prefix, name);
9182         /* snprintf returns the number of characters written excluding the
9183          * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
9184          * indicates truncation.
9185          */
9186         if (ret < 0 || ret >= sizeof(btf_type_name))
9187                 return -ENAMETOOLONG;
9188         return btf__find_by_name_kind(btf, btf_type_name, kind);
9189 }
9190
9191 static inline int find_attach_btf_id(struct btf *btf, const char *name,
9192                                      enum bpf_attach_type attach_type)
9193 {
9194         int err;
9195
9196         if (attach_type == BPF_TRACE_RAW_TP)
9197                 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name,
9198                                               BTF_KIND_TYPEDEF);
9199         else if (attach_type == BPF_LSM_MAC)
9200                 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name,
9201                                               BTF_KIND_FUNC);
9202         else if (attach_type == BPF_TRACE_ITER)
9203                 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name,
9204                                               BTF_KIND_FUNC);
9205         else
9206                 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
9207
9208         return err;
9209 }
9210
9211 int libbpf_find_vmlinux_btf_id(const char *name,
9212                                enum bpf_attach_type attach_type)
9213 {
9214         struct btf *btf;
9215         int err;
9216
9217         btf = libbpf_find_kernel_btf();
9218         if (IS_ERR(btf)) {
9219                 pr_warn("vmlinux BTF is not found\n");
9220                 return -EINVAL;
9221         }
9222
9223         err = find_attach_btf_id(btf, name, attach_type);
9224         if (err <= 0)
9225                 pr_warn("%s is not found in vmlinux BTF\n", name);
9226
9227         btf__free(btf);
9228         return err;
9229 }
9230
9231 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
9232 {
9233         struct bpf_prog_info_linear *info_linear;
9234         struct bpf_prog_info *info;
9235         struct btf *btf = NULL;
9236         int err = -EINVAL;
9237
9238         info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
9239         if (IS_ERR_OR_NULL(info_linear)) {
9240                 pr_warn("failed get_prog_info_linear for FD %d\n",
9241                         attach_prog_fd);
9242                 return -EINVAL;
9243         }
9244         info = &info_linear->info;
9245         if (!info->btf_id) {
9246                 pr_warn("The target program doesn't have BTF\n");
9247                 goto out;
9248         }
9249         if (btf__get_from_id(info->btf_id, &btf)) {
9250                 pr_warn("Failed to get BTF of the program\n");
9251                 goto out;
9252         }
9253         err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
9254         btf__free(btf);
9255         if (err <= 0) {
9256                 pr_warn("%s is not found in prog's BTF\n", name);
9257                 goto out;
9258         }
9259 out:
9260         free(info_linear);
9261         return err;
9262 }
9263
9264 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
9265                               enum bpf_attach_type attach_type,
9266                               int *btf_obj_fd, int *btf_type_id)
9267 {
9268         int ret, i;
9269
9270         ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9271         if (ret > 0) {
9272                 *btf_obj_fd = 0; /* vmlinux BTF */
9273                 *btf_type_id = ret;
9274                 return 0;
9275         }
9276         if (ret != -ENOENT)
9277                 return ret;
9278
9279         ret = load_module_btfs(obj);
9280         if (ret)
9281                 return ret;
9282
9283         for (i = 0; i < obj->btf_module_cnt; i++) {
9284                 const struct module_btf *mod = &obj->btf_modules[i];
9285
9286                 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
9287                 if (ret > 0) {
9288                         *btf_obj_fd = mod->fd;
9289                         *btf_type_id = ret;
9290                         return 0;
9291                 }
9292                 if (ret == -ENOENT)
9293                         continue;
9294
9295                 return ret;
9296         }
9297
9298         return -ESRCH;
9299 }
9300
9301 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id)
9302 {
9303         enum bpf_attach_type attach_type = prog->expected_attach_type;
9304         __u32 attach_prog_fd = prog->attach_prog_fd;
9305         const char *name = prog->sec_name, *attach_name;
9306         const struct bpf_sec_def *sec = NULL;
9307         int i, err;
9308
9309         if (!name)
9310                 return -EINVAL;
9311
9312         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
9313                 if (!section_defs[i].is_attach_btf)
9314                         continue;
9315                 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
9316                         continue;
9317
9318                 sec = &section_defs[i];
9319                 break;
9320         }
9321
9322         if (!sec) {
9323                 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);
9324                 return -ESRCH;
9325         }
9326         attach_name = name + sec->len;
9327
9328         /* BPF program's BTF ID */
9329         if (attach_prog_fd) {
9330                 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
9331                 if (err < 0) {
9332                         pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
9333                                  attach_prog_fd, attach_name, err);
9334                         return err;
9335                 }
9336                 *btf_obj_fd = 0;
9337                 *btf_type_id = err;
9338                 return 0;
9339         }
9340
9341         /* kernel/module BTF ID */
9342         err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9343         if (err) {
9344                 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
9345                 return err;
9346         }
9347         return 0;
9348 }
9349
9350 int libbpf_attach_type_by_name(const char *name,
9351                                enum bpf_attach_type *attach_type)
9352 {
9353         char *type_names;
9354         int i;
9355
9356         if (!name)
9357                 return -EINVAL;
9358
9359         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
9360                 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
9361                         continue;
9362                 if (!section_defs[i].is_attachable)
9363                         return -EINVAL;
9364                 *attach_type = section_defs[i].expected_attach_type;
9365                 return 0;
9366         }
9367         pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9368         type_names = libbpf_get_type_names(true);
9369         if (type_names != NULL) {
9370                 pr_debug("attachable section(type) names are:%s\n", type_names);
9371                 free(type_names);
9372         }
9373
9374         return -EINVAL;
9375 }
9376
9377 int bpf_map__fd(const struct bpf_map *map)
9378 {
9379         return map ? map->fd : -EINVAL;
9380 }
9381
9382 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
9383 {
9384         return map ? &map->def : ERR_PTR(-EINVAL);
9385 }
9386
9387 const char *bpf_map__name(const struct bpf_map *map)
9388 {
9389         return map ? map->name : NULL;
9390 }
9391
9392 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9393 {
9394         return map->def.type;
9395 }
9396
9397 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9398 {
9399         if (map->fd >= 0)
9400                 return -EBUSY;
9401         map->def.type = type;
9402         return 0;
9403 }
9404
9405 __u32 bpf_map__map_flags(const struct bpf_map *map)
9406 {
9407         return map->def.map_flags;
9408 }
9409
9410 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9411 {
9412         if (map->fd >= 0)
9413                 return -EBUSY;
9414         map->def.map_flags = flags;
9415         return 0;
9416 }
9417
9418 __u32 bpf_map__numa_node(const struct bpf_map *map)
9419 {
9420         return map->numa_node;
9421 }
9422
9423 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9424 {
9425         if (map->fd >= 0)
9426                 return -EBUSY;
9427         map->numa_node = numa_node;
9428         return 0;
9429 }
9430
9431 __u32 bpf_map__key_size(const struct bpf_map *map)
9432 {
9433         return map->def.key_size;
9434 }
9435
9436 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9437 {
9438         if (map->fd >= 0)
9439                 return -EBUSY;
9440         map->def.key_size = size;
9441         return 0;
9442 }
9443
9444 __u32 bpf_map__value_size(const struct bpf_map *map)
9445 {
9446         return map->def.value_size;
9447 }
9448
9449 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9450 {
9451         if (map->fd >= 0)
9452                 return -EBUSY;
9453         map->def.value_size = size;
9454         return 0;
9455 }
9456
9457 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9458 {
9459         return map ? map->btf_key_type_id : 0;
9460 }
9461
9462 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9463 {
9464         return map ? map->btf_value_type_id : 0;
9465 }
9466
9467 int bpf_map__set_priv(struct bpf_map *map, void *priv,
9468                      bpf_map_clear_priv_t clear_priv)
9469 {
9470         if (!map)
9471                 return -EINVAL;
9472
9473         if (map->priv) {
9474                 if (map->clear_priv)
9475                         map->clear_priv(map, map->priv);
9476         }
9477
9478         map->priv = priv;
9479         map->clear_priv = clear_priv;
9480         return 0;
9481 }
9482
9483 void *bpf_map__priv(const struct bpf_map *map)
9484 {
9485         return map ? map->priv : ERR_PTR(-EINVAL);
9486 }
9487
9488 int bpf_map__set_initial_value(struct bpf_map *map,
9489                                const void *data, size_t size)
9490 {
9491         if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9492             size != map->def.value_size || map->fd >= 0)
9493                 return -EINVAL;
9494
9495         memcpy(map->mmaped, data, size);
9496         return 0;
9497 }
9498
9499 bool bpf_map__is_offload_neutral(const struct bpf_map *map)
9500 {
9501         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
9502 }
9503
9504 bool bpf_map__is_internal(const struct bpf_map *map)
9505 {
9506         return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9507 }
9508
9509 __u32 bpf_map__ifindex(const struct bpf_map *map)
9510 {
9511         return map->map_ifindex;
9512 }
9513
9514 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9515 {
9516         if (map->fd >= 0)
9517                 return -EBUSY;
9518         map->map_ifindex = ifindex;
9519         return 0;
9520 }
9521
9522 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9523 {
9524         if (!bpf_map_type__is_map_in_map(map->def.type)) {
9525                 pr_warn("error: unsupported map type\n");
9526                 return -EINVAL;
9527         }
9528         if (map->inner_map_fd != -1) {
9529                 pr_warn("error: inner_map_fd already specified\n");
9530                 return -EINVAL;
9531         }
9532         zfree(&map->inner_map);
9533         map->inner_map_fd = fd;
9534         return 0;
9535 }
9536
9537 static struct bpf_map *
9538 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9539 {
9540         ssize_t idx;
9541         struct bpf_map *s, *e;
9542
9543         if (!obj || !obj->maps)
9544                 return NULL;
9545
9546         s = obj->maps;
9547         e = obj->maps + obj->nr_maps;
9548
9549         if ((m < s) || (m >= e)) {
9550                 pr_warn("error in %s: map handler doesn't belong to object\n",
9551                          __func__);
9552                 return NULL;
9553         }
9554
9555         idx = (m - obj->maps) + i;
9556         if (idx >= obj->nr_maps || idx < 0)
9557                 return NULL;
9558         return &obj->maps[idx];
9559 }
9560
9561 struct bpf_map *
9562 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
9563 {
9564         if (prev == NULL)
9565                 return obj->maps;
9566
9567         return __bpf_map__iter(prev, obj, 1);
9568 }
9569
9570 struct bpf_map *
9571 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
9572 {
9573         if (next == NULL) {
9574                 if (!obj->nr_maps)
9575                         return NULL;
9576                 return obj->maps + obj->nr_maps - 1;
9577         }
9578
9579         return __bpf_map__iter(next, obj, -1);
9580 }
9581
9582 struct bpf_map *
9583 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9584 {
9585         struct bpf_map *pos;
9586
9587         bpf_object__for_each_map(pos, obj) {
9588                 if (pos->name && !strcmp(pos->name, name))
9589                         return pos;
9590         }
9591         return NULL;
9592 }
9593
9594 int
9595 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9596 {
9597         return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9598 }
9599
9600 struct bpf_map *
9601 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
9602 {
9603         return ERR_PTR(-ENOTSUP);
9604 }
9605
9606 long libbpf_get_error(const void *ptr)
9607 {
9608         return PTR_ERR_OR_ZERO(ptr);
9609 }
9610
9611 int bpf_prog_load(const char *file, enum bpf_prog_type type,
9612                   struct bpf_object **pobj, int *prog_fd)
9613 {
9614         struct bpf_prog_load_attr attr;
9615
9616         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
9617         attr.file = file;
9618         attr.prog_type = type;
9619         attr.expected_attach_type = 0;
9620
9621         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
9622 }
9623
9624 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
9625                         struct bpf_object **pobj, int *prog_fd)
9626 {
9627         struct bpf_object_open_attr open_attr = {};
9628         struct bpf_program *prog, *first_prog = NULL;
9629         struct bpf_object *obj;
9630         struct bpf_map *map;
9631         int err;
9632
9633         if (!attr)
9634                 return -EINVAL;
9635         if (!attr->file)
9636                 return -EINVAL;
9637
9638         open_attr.file = attr->file;
9639         open_attr.prog_type = attr->prog_type;
9640
9641         obj = bpf_object__open_xattr(&open_attr);
9642         if (IS_ERR_OR_NULL(obj))
9643                 return -ENOENT;
9644
9645         bpf_object__for_each_program(prog, obj) {
9646                 enum bpf_attach_type attach_type = attr->expected_attach_type;
9647                 /*
9648                  * to preserve backwards compatibility, bpf_prog_load treats
9649                  * attr->prog_type, if specified, as an override to whatever
9650                  * bpf_object__open guessed
9651                  */
9652                 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
9653                         bpf_program__set_type(prog, attr->prog_type);
9654                         bpf_program__set_expected_attach_type(prog,
9655                                                               attach_type);
9656                 }
9657                 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
9658                         /*
9659                          * we haven't guessed from section name and user
9660                          * didn't provide a fallback type, too bad...
9661                          */
9662                         bpf_object__close(obj);
9663                         return -EINVAL;
9664                 }
9665
9666                 prog->prog_ifindex = attr->ifindex;
9667                 prog->log_level = attr->log_level;
9668                 prog->prog_flags |= attr->prog_flags;
9669                 if (!first_prog)
9670                         first_prog = prog;
9671         }
9672
9673         bpf_object__for_each_map(map, obj) {
9674                 if (!bpf_map__is_offload_neutral(map))
9675                         map->map_ifindex = attr->ifindex;
9676         }
9677
9678         if (!first_prog) {
9679                 pr_warn("object file doesn't contain bpf program\n");
9680                 bpf_object__close(obj);
9681                 return -ENOENT;
9682         }
9683
9684         err = bpf_object__load(obj);
9685         if (err) {
9686                 bpf_object__close(obj);
9687                 return err;
9688         }
9689
9690         *pobj = obj;
9691         *prog_fd = bpf_program__fd(first_prog);
9692         return 0;
9693 }
9694
9695 struct bpf_link {
9696         int (*detach)(struct bpf_link *link);
9697         int (*destroy)(struct bpf_link *link);
9698         char *pin_path;         /* NULL, if not pinned */
9699         int fd;                 /* hook FD, -1 if not applicable */
9700         bool disconnected;
9701 };
9702
9703 /* Replace link's underlying BPF program with the new one */
9704 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9705 {
9706         return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9707 }
9708
9709 /* Release "ownership" of underlying BPF resource (typically, BPF program
9710  * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
9711  * link, when destructed through bpf_link__destroy() call won't attempt to
9712  * detach/unregisted that BPF resource. This is useful in situations where,
9713  * say, attached BPF program has to outlive userspace program that attached it
9714  * in the system. Depending on type of BPF program, though, there might be
9715  * additional steps (like pinning BPF program in BPF FS) necessary to ensure
9716  * exit of userspace program doesn't trigger automatic detachment and clean up
9717  * inside the kernel.
9718  */
9719 void bpf_link__disconnect(struct bpf_link *link)
9720 {
9721         link->disconnected = true;
9722 }
9723
9724 int bpf_link__destroy(struct bpf_link *link)
9725 {
9726         int err = 0;
9727
9728         if (IS_ERR_OR_NULL(link))
9729                 return 0;
9730
9731         if (!link->disconnected && link->detach)
9732                 err = link->detach(link);
9733         if (link->destroy)
9734                 link->destroy(link);
9735         if (link->pin_path)
9736                 free(link->pin_path);
9737         free(link);
9738
9739         return err;
9740 }
9741
9742 int bpf_link__fd(const struct bpf_link *link)
9743 {
9744         return link->fd;
9745 }
9746
9747 const char *bpf_link__pin_path(const struct bpf_link *link)
9748 {
9749         return link->pin_path;
9750 }
9751
9752 static int bpf_link__detach_fd(struct bpf_link *link)
9753 {
9754         return close(link->fd);
9755 }
9756
9757 struct bpf_link *bpf_link__open(const char *path)
9758 {
9759         struct bpf_link *link;
9760         int fd;
9761
9762         fd = bpf_obj_get(path);
9763         if (fd < 0) {
9764                 fd = -errno;
9765                 pr_warn("failed to open link at %s: %d\n", path, fd);
9766                 return ERR_PTR(fd);
9767         }
9768
9769         link = calloc(1, sizeof(*link));
9770         if (!link) {
9771                 close(fd);
9772                 return ERR_PTR(-ENOMEM);
9773         }
9774         link->detach = &bpf_link__detach_fd;
9775         link->fd = fd;
9776
9777         link->pin_path = strdup(path);
9778         if (!link->pin_path) {
9779                 bpf_link__destroy(link);
9780                 return ERR_PTR(-ENOMEM);
9781         }
9782
9783         return link;
9784 }
9785
9786 int bpf_link__detach(struct bpf_link *link)
9787 {
9788         return bpf_link_detach(link->fd) ? -errno : 0;
9789 }
9790
9791 int bpf_link__pin(struct bpf_link *link, const char *path)
9792 {
9793         int err;
9794
9795         if (link->pin_path)
9796                 return -EBUSY;
9797         err = make_parent_dir(path);
9798         if (err)
9799                 return err;
9800         err = check_path(path);
9801         if (err)
9802                 return err;
9803
9804         link->pin_path = strdup(path);
9805         if (!link->pin_path)
9806                 return -ENOMEM;
9807
9808         if (bpf_obj_pin(link->fd, link->pin_path)) {
9809                 err = -errno;
9810                 zfree(&link->pin_path);
9811                 return err;
9812         }
9813
9814         pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9815         return 0;
9816 }
9817
9818 int bpf_link__unpin(struct bpf_link *link)
9819 {
9820         int err;
9821
9822         if (!link->pin_path)
9823                 return -EINVAL;
9824
9825         err = unlink(link->pin_path);
9826         if (err != 0)
9827                 return -errno;
9828
9829         pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9830         zfree(&link->pin_path);
9831         return 0;
9832 }
9833
9834 static int bpf_link__detach_perf_event(struct bpf_link *link)
9835 {
9836         int err;
9837
9838         err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0);
9839         if (err)
9840                 err = -errno;
9841
9842         close(link->fd);
9843         return err;
9844 }
9845
9846 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
9847                                                 int pfd)
9848 {
9849         char errmsg[STRERR_BUFSIZE];
9850         struct bpf_link *link;
9851         int prog_fd, err;
9852
9853         if (pfd < 0) {
9854                 pr_warn("prog '%s': invalid perf event FD %d\n",
9855                         prog->name, pfd);
9856                 return ERR_PTR(-EINVAL);
9857         }
9858         prog_fd = bpf_program__fd(prog);
9859         if (prog_fd < 0) {
9860                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9861                         prog->name);
9862                 return ERR_PTR(-EINVAL);
9863         }
9864
9865         link = calloc(1, sizeof(*link));
9866         if (!link)
9867                 return ERR_PTR(-ENOMEM);
9868         link->detach = &bpf_link__detach_perf_event;
9869         link->fd = pfd;
9870
9871         if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9872                 err = -errno;
9873                 free(link);
9874                 pr_warn("prog '%s': failed to attach to pfd %d: %s\n",
9875                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9876                 if (err == -EPROTO)
9877                         pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9878                                 prog->name, pfd);
9879                 return ERR_PTR(err);
9880         }
9881         if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9882                 err = -errno;
9883                 free(link);
9884                 pr_warn("prog '%s': failed to enable pfd %d: %s\n",
9885                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9886                 return ERR_PTR(err);
9887         }
9888         return link;
9889 }
9890
9891 /*
9892  * this function is expected to parse integer in the range of [0, 2^31-1] from
9893  * given file using scanf format string fmt. If actual parsed value is
9894  * negative, the result might be indistinguishable from error
9895  */
9896 static int parse_uint_from_file(const char *file, const char *fmt)
9897 {
9898         char buf[STRERR_BUFSIZE];
9899         int err, ret;
9900         FILE *f;
9901
9902         f = fopen(file, "r");
9903         if (!f) {
9904                 err = -errno;
9905                 pr_debug("failed to open '%s': %s\n", file,
9906                          libbpf_strerror_r(err, buf, sizeof(buf)));
9907                 return err;
9908         }
9909         err = fscanf(f, fmt, &ret);
9910         if (err != 1) {
9911                 err = err == EOF ? -EIO : -errno;
9912                 pr_debug("failed to parse '%s': %s\n", file,
9913                         libbpf_strerror_r(err, buf, sizeof(buf)));
9914                 fclose(f);
9915                 return err;
9916         }
9917         fclose(f);
9918         return ret;
9919 }
9920
9921 static int determine_kprobe_perf_type(void)
9922 {
9923         const char *file = "/sys/bus/event_source/devices/kprobe/type";
9924
9925         return parse_uint_from_file(file, "%d\n");
9926 }
9927
9928 static int determine_uprobe_perf_type(void)
9929 {
9930         const char *file = "/sys/bus/event_source/devices/uprobe/type";
9931
9932         return parse_uint_from_file(file, "%d\n");
9933 }
9934
9935 static int determine_kprobe_retprobe_bit(void)
9936 {
9937         const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9938
9939         return parse_uint_from_file(file, "config:%d\n");
9940 }
9941
9942 static int determine_uprobe_retprobe_bit(void)
9943 {
9944         const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9945
9946         return parse_uint_from_file(file, "config:%d\n");
9947 }
9948
9949 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9950                                  uint64_t offset, int pid)
9951 {
9952         struct perf_event_attr attr = {};
9953         char errmsg[STRERR_BUFSIZE];
9954         int type, pfd, err;
9955
9956         type = uprobe ? determine_uprobe_perf_type()
9957                       : determine_kprobe_perf_type();
9958         if (type < 0) {
9959                 pr_warn("failed to determine %s perf type: %s\n",
9960                         uprobe ? "uprobe" : "kprobe",
9961                         libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9962                 return type;
9963         }
9964         if (retprobe) {
9965                 int bit = uprobe ? determine_uprobe_retprobe_bit()
9966                                  : determine_kprobe_retprobe_bit();
9967
9968                 if (bit < 0) {
9969                         pr_warn("failed to determine %s retprobe bit: %s\n",
9970                                 uprobe ? "uprobe" : "kprobe",
9971                                 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9972                         return bit;
9973                 }
9974                 attr.config |= 1 << bit;
9975         }
9976         attr.size = sizeof(attr);
9977         attr.type = type;
9978         attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
9979         attr.config2 = offset;           /* kprobe_addr or probe_offset */
9980
9981         /* pid filter is meaningful only for uprobes */
9982         pfd = syscall(__NR_perf_event_open, &attr,
9983                       pid < 0 ? -1 : pid /* pid */,
9984                       pid == -1 ? 0 : -1 /* cpu */,
9985                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9986         if (pfd < 0) {
9987                 err = -errno;
9988                 pr_warn("%s perf_event_open() failed: %s\n",
9989                         uprobe ? "uprobe" : "kprobe",
9990                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9991                 return err;
9992         }
9993         return pfd;
9994 }
9995
9996 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
9997                                             bool retprobe,
9998                                             const char *func_name)
9999 {
10000         char errmsg[STRERR_BUFSIZE];
10001         struct bpf_link *link;
10002         int pfd, err;
10003
10004         pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,
10005                                     0 /* offset */, -1 /* pid */);
10006         if (pfd < 0) {
10007                 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n",
10008                         prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
10009                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
10010                 return ERR_PTR(pfd);
10011         }
10012         link = bpf_program__attach_perf_event(prog, pfd);
10013         if (IS_ERR(link)) {
10014                 close(pfd);
10015                 err = PTR_ERR(link);
10016                 pr_warn("prog '%s': failed to attach to %s '%s': %s\n",
10017                         prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
10018                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10019                 return link;
10020         }
10021         return link;
10022 }
10023
10024 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
10025                                       struct bpf_program *prog)
10026 {
10027         const char *func_name;
10028         bool retprobe;
10029
10030         func_name = prog->sec_name + sec->len;
10031         retprobe = strcmp(sec->sec, "kretprobe/") == 0;
10032
10033         return bpf_program__attach_kprobe(prog, retprobe, func_name);
10034 }
10035
10036 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
10037                                             bool retprobe, pid_t pid,
10038                                             const char *binary_path,
10039                                             size_t func_offset)
10040 {
10041         char errmsg[STRERR_BUFSIZE];
10042         struct bpf_link *link;
10043         int pfd, err;
10044
10045         pfd = perf_event_open_probe(true /* uprobe */, retprobe,
10046                                     binary_path, func_offset, pid);
10047         if (pfd < 0) {
10048                 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
10049                         prog->name, retprobe ? "uretprobe" : "uprobe",
10050                         binary_path, func_offset,
10051                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
10052                 return ERR_PTR(pfd);
10053         }
10054         link = bpf_program__attach_perf_event(prog, pfd);
10055         if (IS_ERR(link)) {
10056                 close(pfd);
10057                 err = PTR_ERR(link);
10058                 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
10059                         prog->name, retprobe ? "uretprobe" : "uprobe",
10060                         binary_path, func_offset,
10061                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10062                 return link;
10063         }
10064         return link;
10065 }
10066
10067 static int determine_tracepoint_id(const char *tp_category,
10068                                    const char *tp_name)
10069 {
10070         char file[PATH_MAX];
10071         int ret;
10072
10073         ret = snprintf(file, sizeof(file),
10074                        "/sys/kernel/debug/tracing/events/%s/%s/id",
10075                        tp_category, tp_name);
10076         if (ret < 0)
10077                 return -errno;
10078         if (ret >= sizeof(file)) {
10079                 pr_debug("tracepoint %s/%s path is too long\n",
10080                          tp_category, tp_name);
10081                 return -E2BIG;
10082         }
10083         return parse_uint_from_file(file, "%d\n");
10084 }
10085
10086 static int perf_event_open_tracepoint(const char *tp_category,
10087                                       const char *tp_name)
10088 {
10089         struct perf_event_attr attr = {};
10090         char errmsg[STRERR_BUFSIZE];
10091         int tp_id, pfd, err;
10092
10093         tp_id = determine_tracepoint_id(tp_category, tp_name);
10094         if (tp_id < 0) {
10095                 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
10096                         tp_category, tp_name,
10097                         libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
10098                 return tp_id;
10099         }
10100
10101         attr.type = PERF_TYPE_TRACEPOINT;
10102         attr.size = sizeof(attr);
10103         attr.config = tp_id;
10104
10105         pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
10106                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
10107         if (pfd < 0) {
10108                 err = -errno;
10109                 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
10110                         tp_category, tp_name,
10111                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10112                 return err;
10113         }
10114         return pfd;
10115 }
10116
10117 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
10118                                                 const char *tp_category,
10119                                                 const char *tp_name)
10120 {
10121         char errmsg[STRERR_BUFSIZE];
10122         struct bpf_link *link;
10123         int pfd, err;
10124
10125         pfd = perf_event_open_tracepoint(tp_category, tp_name);
10126         if (pfd < 0) {
10127                 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
10128                         prog->name, tp_category, tp_name,
10129                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
10130                 return ERR_PTR(pfd);
10131         }
10132         link = bpf_program__attach_perf_event(prog, pfd);
10133         if (IS_ERR(link)) {
10134                 close(pfd);
10135                 err = PTR_ERR(link);
10136                 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
10137                         prog->name, tp_category, tp_name,
10138                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10139                 return link;
10140         }
10141         return link;
10142 }
10143
10144 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
10145                                   struct bpf_program *prog)
10146 {
10147         char *sec_name, *tp_cat, *tp_name;
10148         struct bpf_link *link;
10149
10150         sec_name = strdup(prog->sec_name);
10151         if (!sec_name)
10152                 return ERR_PTR(-ENOMEM);
10153
10154         /* extract "tp/<category>/<name>" */
10155         tp_cat = sec_name + sec->len;
10156         tp_name = strchr(tp_cat, '/');
10157         if (!tp_name) {
10158                 link = ERR_PTR(-EINVAL);
10159                 goto out;
10160         }
10161         *tp_name = '\0';
10162         tp_name++;
10163
10164         link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
10165 out:
10166         free(sec_name);
10167         return link;
10168 }
10169
10170 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
10171                                                     const char *tp_name)
10172 {
10173         char errmsg[STRERR_BUFSIZE];
10174         struct bpf_link *link;
10175         int prog_fd, pfd;
10176
10177         prog_fd = bpf_program__fd(prog);
10178         if (prog_fd < 0) {
10179                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
10180                 return ERR_PTR(-EINVAL);
10181         }
10182
10183         link = calloc(1, sizeof(*link));
10184         if (!link)
10185                 return ERR_PTR(-ENOMEM);
10186         link->detach = &bpf_link__detach_fd;
10187
10188         pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
10189         if (pfd < 0) {
10190                 pfd = -errno;
10191                 free(link);
10192                 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
10193                         prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
10194                 return ERR_PTR(pfd);
10195         }
10196         link->fd = pfd;
10197         return link;
10198 }
10199
10200 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
10201                                       struct bpf_program *prog)
10202 {
10203         const char *tp_name = prog->sec_name + sec->len;
10204
10205         return bpf_program__attach_raw_tracepoint(prog, tp_name);
10206 }
10207
10208 /* Common logic for all BPF program types that attach to a btf_id */
10209 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
10210 {
10211         char errmsg[STRERR_BUFSIZE];
10212         struct bpf_link *link;
10213         int prog_fd, pfd;
10214
10215         prog_fd = bpf_program__fd(prog);
10216         if (prog_fd < 0) {
10217                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
10218                 return ERR_PTR(-EINVAL);
10219         }
10220
10221         link = calloc(1, sizeof(*link));
10222         if (!link)
10223                 return ERR_PTR(-ENOMEM);
10224         link->detach = &bpf_link__detach_fd;
10225
10226         pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
10227         if (pfd < 0) {
10228                 pfd = -errno;
10229                 free(link);
10230                 pr_warn("prog '%s': failed to attach: %s\n",
10231                         prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
10232                 return ERR_PTR(pfd);
10233         }
10234         link->fd = pfd;
10235         return (struct bpf_link *)link;
10236 }
10237
10238 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
10239 {
10240         return bpf_program__attach_btf_id(prog);
10241 }
10242
10243 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
10244 {
10245         return bpf_program__attach_btf_id(prog);
10246 }
10247
10248 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
10249                                      struct bpf_program *prog)
10250 {
10251         return bpf_program__attach_trace(prog);
10252 }
10253
10254 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
10255                                    struct bpf_program *prog)
10256 {
10257         return bpf_program__attach_lsm(prog);
10258 }
10259
10260 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
10261                                     struct bpf_program *prog)
10262 {
10263         return bpf_program__attach_iter(prog, NULL);
10264 }
10265
10266 static struct bpf_link *
10267 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id,
10268                        const char *target_name)
10269 {
10270         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
10271                             .target_btf_id = btf_id);
10272         enum bpf_attach_type attach_type;
10273         char errmsg[STRERR_BUFSIZE];
10274         struct bpf_link *link;
10275         int prog_fd, link_fd;
10276
10277         prog_fd = bpf_program__fd(prog);
10278         if (prog_fd < 0) {
10279                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
10280                 return ERR_PTR(-EINVAL);
10281         }
10282
10283         link = calloc(1, sizeof(*link));
10284         if (!link)
10285                 return ERR_PTR(-ENOMEM);
10286         link->detach = &bpf_link__detach_fd;
10287
10288         attach_type = bpf_program__get_expected_attach_type(prog);
10289         link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
10290         if (link_fd < 0) {
10291                 link_fd = -errno;
10292                 free(link);
10293                 pr_warn("prog '%s': failed to attach to %s: %s\n",
10294                         prog->name, target_name,
10295                         libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
10296                 return ERR_PTR(link_fd);
10297         }
10298         link->fd = link_fd;
10299         return link;
10300 }
10301
10302 struct bpf_link *
10303 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
10304 {
10305         return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
10306 }
10307
10308 struct bpf_link *
10309 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)
10310 {
10311         return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
10312 }
10313
10314 struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)
10315 {
10316         /* target_fd/target_ifindex use the same field in LINK_CREATE */
10317         return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
10318 }
10319
10320 struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog,
10321                                               int target_fd,
10322                                               const char *attach_func_name)
10323 {
10324         int btf_id;
10325
10326         if (!!target_fd != !!attach_func_name) {
10327                 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
10328                         prog->name);
10329                 return ERR_PTR(-EINVAL);
10330         }
10331
10332         if (prog->type != BPF_PROG_TYPE_EXT) {
10333                 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
10334                         prog->name);
10335                 return ERR_PTR(-EINVAL);
10336         }
10337
10338         if (target_fd) {
10339                 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
10340                 if (btf_id < 0)
10341                         return ERR_PTR(btf_id);
10342
10343                 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
10344         } else {
10345                 /* no target, so use raw_tracepoint_open for compatibility
10346                  * with old kernels
10347                  */
10348                 return bpf_program__attach_trace(prog);
10349         }
10350 }
10351
10352 struct bpf_link *
10353 bpf_program__attach_iter(struct bpf_program *prog,
10354                          const struct bpf_iter_attach_opts *opts)
10355 {
10356         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
10357         char errmsg[STRERR_BUFSIZE];
10358         struct bpf_link *link;
10359         int prog_fd, link_fd;
10360         __u32 target_fd = 0;
10361
10362         if (!OPTS_VALID(opts, bpf_iter_attach_opts))
10363                 return ERR_PTR(-EINVAL);
10364
10365         link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
10366         link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
10367
10368         prog_fd = bpf_program__fd(prog);
10369         if (prog_fd < 0) {
10370                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
10371                 return ERR_PTR(-EINVAL);
10372         }
10373
10374         link = calloc(1, sizeof(*link));
10375         if (!link)
10376                 return ERR_PTR(-ENOMEM);
10377         link->detach = &bpf_link__detach_fd;
10378
10379         link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
10380                                   &link_create_opts);
10381         if (link_fd < 0) {
10382                 link_fd = -errno;
10383                 free(link);
10384                 pr_warn("prog '%s': failed to attach to iterator: %s\n",
10385                         prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
10386                 return ERR_PTR(link_fd);
10387         }
10388         link->fd = link_fd;
10389         return link;
10390 }
10391
10392 struct bpf_link *bpf_program__attach(struct bpf_program *prog)
10393 {
10394         const struct bpf_sec_def *sec_def;
10395
10396         sec_def = find_sec_def(prog->sec_name);
10397         if (!sec_def || !sec_def->attach_fn)
10398                 return ERR_PTR(-ESRCH);
10399
10400         return sec_def->attach_fn(sec_def, prog);
10401 }
10402
10403 static int bpf_link__detach_struct_ops(struct bpf_link *link)
10404 {
10405         __u32 zero = 0;
10406
10407         if (bpf_map_delete_elem(link->fd, &zero))
10408                 return -errno;
10409
10410         return 0;
10411 }
10412
10413 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
10414 {
10415         struct bpf_struct_ops *st_ops;
10416         struct bpf_link *link;
10417         __u32 i, zero = 0;
10418         int err;
10419
10420         if (!bpf_map__is_struct_ops(map) || map->fd == -1)
10421                 return ERR_PTR(-EINVAL);
10422
10423         link = calloc(1, sizeof(*link));
10424         if (!link)
10425                 return ERR_PTR(-EINVAL);
10426
10427         st_ops = map->st_ops;
10428         for (i = 0; i < btf_vlen(st_ops->type); i++) {
10429                 struct bpf_program *prog = st_ops->progs[i];
10430                 void *kern_data;
10431                 int prog_fd;
10432
10433                 if (!prog)
10434                         continue;
10435
10436                 prog_fd = bpf_program__fd(prog);
10437                 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
10438                 *(unsigned long *)kern_data = prog_fd;
10439         }
10440
10441         err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
10442         if (err) {
10443                 err = -errno;
10444                 free(link);
10445                 return ERR_PTR(err);
10446         }
10447
10448         link->detach = bpf_link__detach_struct_ops;
10449         link->fd = map->fd;
10450
10451         return link;
10452 }
10453
10454 enum bpf_perf_event_ret
10455 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
10456                            void **copy_mem, size_t *copy_size,
10457                            bpf_perf_event_print_t fn, void *private_data)
10458 {
10459         struct perf_event_mmap_page *header = mmap_mem;
10460         __u64 data_head = ring_buffer_read_head(header);
10461         __u64 data_tail = header->data_tail;
10462         void *base = ((__u8 *)header) + page_size;
10463         int ret = LIBBPF_PERF_EVENT_CONT;
10464         struct perf_event_header *ehdr;
10465         size_t ehdr_size;
10466
10467         while (data_head != data_tail) {
10468                 ehdr = base + (data_tail & (mmap_size - 1));
10469                 ehdr_size = ehdr->size;
10470
10471                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
10472                         void *copy_start = ehdr;
10473                         size_t len_first = base + mmap_size - copy_start;
10474                         size_t len_secnd = ehdr_size - len_first;
10475
10476                         if (*copy_size < ehdr_size) {
10477                                 free(*copy_mem);
10478                                 *copy_mem = malloc(ehdr_size);
10479                                 if (!*copy_mem) {
10480                                         *copy_size = 0;
10481                                         ret = LIBBPF_PERF_EVENT_ERROR;
10482                                         break;
10483                                 }
10484                                 *copy_size = ehdr_size;
10485                         }
10486
10487                         memcpy(*copy_mem, copy_start, len_first);
10488                         memcpy(*copy_mem + len_first, base, len_secnd);
10489                         ehdr = *copy_mem;
10490                 }
10491
10492                 ret = fn(ehdr, private_data);
10493                 data_tail += ehdr_size;
10494                 if (ret != LIBBPF_PERF_EVENT_CONT)
10495                         break;
10496         }
10497
10498         ring_buffer_write_tail(header, data_tail);
10499         return ret;
10500 }
10501
10502 struct perf_buffer;
10503
10504 struct perf_buffer_params {
10505         struct perf_event_attr *attr;
10506         /* if event_cb is specified, it takes precendence */
10507         perf_buffer_event_fn event_cb;
10508         /* sample_cb and lost_cb are higher-level common-case callbacks */
10509         perf_buffer_sample_fn sample_cb;
10510         perf_buffer_lost_fn lost_cb;
10511         void *ctx;
10512         int cpu_cnt;
10513         int *cpus;
10514         int *map_keys;
10515 };
10516
10517 struct perf_cpu_buf {
10518         struct perf_buffer *pb;
10519         void *base; /* mmap()'ed memory */
10520         void *buf; /* for reconstructing segmented data */
10521         size_t buf_size;
10522         int fd;
10523         int cpu;
10524         int map_key;
10525 };
10526
10527 struct perf_buffer {
10528         perf_buffer_event_fn event_cb;
10529         perf_buffer_sample_fn sample_cb;
10530         perf_buffer_lost_fn lost_cb;
10531         void *ctx; /* passed into callbacks */
10532
10533         size_t page_size;
10534         size_t mmap_size;
10535         struct perf_cpu_buf **cpu_bufs;
10536         struct epoll_event *events;
10537         int cpu_cnt; /* number of allocated CPU buffers */
10538         int epoll_fd; /* perf event FD */
10539         int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
10540 };
10541
10542 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
10543                                       struct perf_cpu_buf *cpu_buf)
10544 {
10545         if (!cpu_buf)
10546                 return;
10547         if (cpu_buf->base &&
10548             munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
10549                 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
10550         if (cpu_buf->fd >= 0) {
10551                 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
10552                 close(cpu_buf->fd);
10553         }
10554         free(cpu_buf->buf);
10555         free(cpu_buf);
10556 }
10557
10558 void perf_buffer__free(struct perf_buffer *pb)
10559 {
10560         int i;
10561
10562         if (IS_ERR_OR_NULL(pb))
10563                 return;
10564         if (pb->cpu_bufs) {
10565                 for (i = 0; i < pb->cpu_cnt; i++) {
10566                         struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10567
10568                         if (!cpu_buf)
10569                                 continue;
10570
10571                         bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
10572                         perf_buffer__free_cpu_buf(pb, cpu_buf);
10573                 }
10574                 free(pb->cpu_bufs);
10575         }
10576         if (pb->epoll_fd >= 0)
10577                 close(pb->epoll_fd);
10578         free(pb->events);
10579         free(pb);
10580 }
10581
10582 static struct perf_cpu_buf *
10583 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
10584                           int cpu, int map_key)
10585 {
10586         struct perf_cpu_buf *cpu_buf;
10587         char msg[STRERR_BUFSIZE];
10588         int err;
10589
10590         cpu_buf = calloc(1, sizeof(*cpu_buf));
10591         if (!cpu_buf)
10592                 return ERR_PTR(-ENOMEM);
10593
10594         cpu_buf->pb = pb;
10595         cpu_buf->cpu = cpu;
10596         cpu_buf->map_key = map_key;
10597
10598         cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
10599                               -1, PERF_FLAG_FD_CLOEXEC);
10600         if (cpu_buf->fd < 0) {
10601                 err = -errno;
10602                 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
10603                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10604                 goto error;
10605         }
10606
10607         cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
10608                              PROT_READ | PROT_WRITE, MAP_SHARED,
10609                              cpu_buf->fd, 0);
10610         if (cpu_buf->base == MAP_FAILED) {
10611                 cpu_buf->base = NULL;
10612                 err = -errno;
10613                 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
10614                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10615                 goto error;
10616         }
10617
10618         if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
10619                 err = -errno;
10620                 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
10621                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10622                 goto error;
10623         }
10624
10625         return cpu_buf;
10626
10627 error:
10628         perf_buffer__free_cpu_buf(pb, cpu_buf);
10629         return (struct perf_cpu_buf *)ERR_PTR(err);
10630 }
10631
10632 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10633                                               struct perf_buffer_params *p);
10634
10635 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
10636                                      const struct perf_buffer_opts *opts)
10637 {
10638         struct perf_buffer_params p = {};
10639         struct perf_event_attr attr = { 0, };
10640
10641         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
10642         attr.type = PERF_TYPE_SOFTWARE;
10643         attr.sample_type = PERF_SAMPLE_RAW;
10644         attr.sample_period = 1;
10645         attr.wakeup_events = 1;
10646
10647         p.attr = &attr;
10648         p.sample_cb = opts ? opts->sample_cb : NULL;
10649         p.lost_cb = opts ? opts->lost_cb : NULL;
10650         p.ctx = opts ? opts->ctx : NULL;
10651
10652         return __perf_buffer__new(map_fd, page_cnt, &p);
10653 }
10654
10655 struct perf_buffer *
10656 perf_buffer__new_raw(int map_fd, size_t page_cnt,
10657                      const struct perf_buffer_raw_opts *opts)
10658 {
10659         struct perf_buffer_params p = {};
10660
10661         p.attr = opts->attr;
10662         p.event_cb = opts->event_cb;
10663         p.ctx = opts->ctx;
10664         p.cpu_cnt = opts->cpu_cnt;
10665         p.cpus = opts->cpus;
10666         p.map_keys = opts->map_keys;
10667
10668         return __perf_buffer__new(map_fd, page_cnt, &p);
10669 }
10670
10671 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10672                                               struct perf_buffer_params *p)
10673 {
10674         const char *online_cpus_file = "/sys/devices/system/cpu/online";
10675         struct bpf_map_info map;
10676         char msg[STRERR_BUFSIZE];
10677         struct perf_buffer *pb;
10678         bool *online = NULL;
10679         __u32 map_info_len;
10680         int err, i, j, n;
10681
10682         if (page_cnt & (page_cnt - 1)) {
10683                 pr_warn("page count should be power of two, but is %zu\n",
10684                         page_cnt);
10685                 return ERR_PTR(-EINVAL);
10686         }
10687
10688         /* best-effort sanity checks */
10689         memset(&map, 0, sizeof(map));
10690         map_info_len = sizeof(map);
10691         err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
10692         if (err) {
10693                 err = -errno;
10694                 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
10695                  * -EBADFD, -EFAULT, or -E2BIG on real error
10696                  */
10697                 if (err != -EINVAL) {
10698                         pr_warn("failed to get map info for map FD %d: %s\n",
10699                                 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
10700                         return ERR_PTR(err);
10701                 }
10702                 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
10703                          map_fd);
10704         } else {
10705                 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
10706                         pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
10707                                 map.name);
10708                         return ERR_PTR(-EINVAL);
10709                 }
10710         }
10711
10712         pb = calloc(1, sizeof(*pb));
10713         if (!pb)
10714                 return ERR_PTR(-ENOMEM);
10715
10716         pb->event_cb = p->event_cb;
10717         pb->sample_cb = p->sample_cb;
10718         pb->lost_cb = p->lost_cb;
10719         pb->ctx = p->ctx;
10720
10721         pb->page_size = getpagesize();
10722         pb->mmap_size = pb->page_size * page_cnt;
10723         pb->map_fd = map_fd;
10724
10725         pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
10726         if (pb->epoll_fd < 0) {
10727                 err = -errno;
10728                 pr_warn("failed to create epoll instance: %s\n",
10729                         libbpf_strerror_r(err, msg, sizeof(msg)));
10730                 goto error;
10731         }
10732
10733         if (p->cpu_cnt > 0) {
10734                 pb->cpu_cnt = p->cpu_cnt;
10735         } else {
10736                 pb->cpu_cnt = libbpf_num_possible_cpus();
10737                 if (pb->cpu_cnt < 0) {
10738                         err = pb->cpu_cnt;
10739                         goto error;
10740                 }
10741                 if (map.max_entries && map.max_entries < pb->cpu_cnt)
10742                         pb->cpu_cnt = map.max_entries;
10743         }
10744
10745         pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
10746         if (!pb->events) {
10747                 err = -ENOMEM;
10748                 pr_warn("failed to allocate events: out of memory\n");
10749                 goto error;
10750         }
10751         pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
10752         if (!pb->cpu_bufs) {
10753                 err = -ENOMEM;
10754                 pr_warn("failed to allocate buffers: out of memory\n");
10755                 goto error;
10756         }
10757
10758         err = parse_cpu_mask_file(online_cpus_file, &online, &n);
10759         if (err) {
10760                 pr_warn("failed to get online CPU mask: %d\n", err);
10761                 goto error;
10762         }
10763
10764         for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
10765                 struct perf_cpu_buf *cpu_buf;
10766                 int cpu, map_key;
10767
10768                 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
10769                 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
10770
10771                 /* in case user didn't explicitly requested particular CPUs to
10772                  * be attached to, skip offline/not present CPUs
10773                  */
10774                 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
10775                         continue;
10776
10777                 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
10778                 if (IS_ERR(cpu_buf)) {
10779                         err = PTR_ERR(cpu_buf);
10780                         goto error;
10781                 }
10782
10783                 pb->cpu_bufs[j] = cpu_buf;
10784
10785                 err = bpf_map_update_elem(pb->map_fd, &map_key,
10786                                           &cpu_buf->fd, 0);
10787                 if (err) {
10788                         err = -errno;
10789                         pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
10790                                 cpu, map_key, cpu_buf->fd,
10791                                 libbpf_strerror_r(err, msg, sizeof(msg)));
10792                         goto error;
10793                 }
10794
10795                 pb->events[j].events = EPOLLIN;
10796                 pb->events[j].data.ptr = cpu_buf;
10797                 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
10798                               &pb->events[j]) < 0) {
10799                         err = -errno;
10800                         pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
10801                                 cpu, cpu_buf->fd,
10802                                 libbpf_strerror_r(err, msg, sizeof(msg)));
10803                         goto error;
10804                 }
10805                 j++;
10806         }
10807         pb->cpu_cnt = j;
10808         free(online);
10809
10810         return pb;
10811
10812 error:
10813         free(online);
10814         if (pb)
10815                 perf_buffer__free(pb);
10816         return ERR_PTR(err);
10817 }
10818
10819 struct perf_sample_raw {
10820         struct perf_event_header header;
10821         uint32_t size;
10822         char data[];
10823 };
10824
10825 struct perf_sample_lost {
10826         struct perf_event_header header;
10827         uint64_t id;
10828         uint64_t lost;
10829         uint64_t sample_id;
10830 };
10831
10832 static enum bpf_perf_event_ret
10833 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
10834 {
10835         struct perf_cpu_buf *cpu_buf = ctx;
10836         struct perf_buffer *pb = cpu_buf->pb;
10837         void *data = e;
10838
10839         /* user wants full control over parsing perf event */
10840         if (pb->event_cb)
10841                 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
10842
10843         switch (e->type) {
10844         case PERF_RECORD_SAMPLE: {
10845                 struct perf_sample_raw *s = data;
10846
10847                 if (pb->sample_cb)
10848                         pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
10849                 break;
10850         }
10851         case PERF_RECORD_LOST: {
10852                 struct perf_sample_lost *s = data;
10853
10854                 if (pb->lost_cb)
10855                         pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
10856                 break;
10857         }
10858         default:
10859                 pr_warn("unknown perf sample type %d\n", e->type);
10860                 return LIBBPF_PERF_EVENT_ERROR;
10861         }
10862         return LIBBPF_PERF_EVENT_CONT;
10863 }
10864
10865 static int perf_buffer__process_records(struct perf_buffer *pb,
10866                                         struct perf_cpu_buf *cpu_buf)
10867 {
10868         enum bpf_perf_event_ret ret;
10869
10870         ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
10871                                          pb->page_size, &cpu_buf->buf,
10872                                          &cpu_buf->buf_size,
10873                                          perf_buffer__process_record, cpu_buf);
10874         if (ret != LIBBPF_PERF_EVENT_CONT)
10875                 return ret;
10876         return 0;
10877 }
10878
10879 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
10880 {
10881         return pb->epoll_fd;
10882 }
10883
10884 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
10885 {
10886         int i, cnt, err;
10887
10888         cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
10889         for (i = 0; i < cnt; i++) {
10890                 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
10891
10892                 err = perf_buffer__process_records(pb, cpu_buf);
10893                 if (err) {
10894                         pr_warn("error while processing records: %d\n", err);
10895                         return err;
10896                 }
10897         }
10898         return cnt < 0 ? -errno : cnt;
10899 }
10900
10901 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
10902  * manager.
10903  */
10904 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
10905 {
10906         return pb->cpu_cnt;
10907 }
10908
10909 /*
10910  * Return perf_event FD of a ring buffer in *buf_idx* slot of
10911  * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
10912  * select()/poll()/epoll() Linux syscalls.
10913  */
10914 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
10915 {
10916         struct perf_cpu_buf *cpu_buf;
10917
10918         if (buf_idx >= pb->cpu_cnt)
10919                 return -EINVAL;
10920
10921         cpu_buf = pb->cpu_bufs[buf_idx];
10922         if (!cpu_buf)
10923                 return -ENOENT;
10924
10925         return cpu_buf->fd;
10926 }
10927
10928 /*
10929  * Consume data from perf ring buffer corresponding to slot *buf_idx* in
10930  * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
10931  * consume, do nothing and return success.
10932  * Returns:
10933  *   - 0 on success;
10934  *   - <0 on failure.
10935  */
10936 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
10937 {
10938         struct perf_cpu_buf *cpu_buf;
10939
10940         if (buf_idx >= pb->cpu_cnt)
10941                 return -EINVAL;
10942
10943         cpu_buf = pb->cpu_bufs[buf_idx];
10944         if (!cpu_buf)
10945                 return -ENOENT;
10946
10947         return perf_buffer__process_records(pb, cpu_buf);
10948 }
10949
10950 int perf_buffer__consume(struct perf_buffer *pb)
10951 {
10952         int i, err;
10953
10954         for (i = 0; i < pb->cpu_cnt; i++) {
10955                 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10956
10957                 if (!cpu_buf)
10958                         continue;
10959
10960                 err = perf_buffer__process_records(pb, cpu_buf);
10961                 if (err) {
10962                         pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
10963                         return err;
10964                 }
10965         }
10966         return 0;
10967 }
10968
10969 struct bpf_prog_info_array_desc {
10970         int     array_offset;   /* e.g. offset of jited_prog_insns */
10971         int     count_offset;   /* e.g. offset of jited_prog_len */
10972         int     size_offset;    /* > 0: offset of rec size,
10973                                  * < 0: fix size of -size_offset
10974                                  */
10975 };
10976
10977 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
10978         [BPF_PROG_INFO_JITED_INSNS] = {
10979                 offsetof(struct bpf_prog_info, jited_prog_insns),
10980                 offsetof(struct bpf_prog_info, jited_prog_len),
10981                 -1,
10982         },
10983         [BPF_PROG_INFO_XLATED_INSNS] = {
10984                 offsetof(struct bpf_prog_info, xlated_prog_insns),
10985                 offsetof(struct bpf_prog_info, xlated_prog_len),
10986                 -1,
10987         },
10988         [BPF_PROG_INFO_MAP_IDS] = {
10989                 offsetof(struct bpf_prog_info, map_ids),
10990                 offsetof(struct bpf_prog_info, nr_map_ids),
10991                 -(int)sizeof(__u32),
10992         },
10993         [BPF_PROG_INFO_JITED_KSYMS] = {
10994                 offsetof(struct bpf_prog_info, jited_ksyms),
10995                 offsetof(struct bpf_prog_info, nr_jited_ksyms),
10996                 -(int)sizeof(__u64),
10997         },
10998         [BPF_PROG_INFO_JITED_FUNC_LENS] = {
10999                 offsetof(struct bpf_prog_info, jited_func_lens),
11000                 offsetof(struct bpf_prog_info, nr_jited_func_lens),
11001                 -(int)sizeof(__u32),
11002         },
11003         [BPF_PROG_INFO_FUNC_INFO] = {
11004                 offsetof(struct bpf_prog_info, func_info),
11005                 offsetof(struct bpf_prog_info, nr_func_info),
11006                 offsetof(struct bpf_prog_info, func_info_rec_size),
11007         },
11008         [BPF_PROG_INFO_LINE_INFO] = {
11009                 offsetof(struct bpf_prog_info, line_info),
11010                 offsetof(struct bpf_prog_info, nr_line_info),
11011                 offsetof(struct bpf_prog_info, line_info_rec_size),
11012         },
11013         [BPF_PROG_INFO_JITED_LINE_INFO] = {
11014                 offsetof(struct bpf_prog_info, jited_line_info),
11015                 offsetof(struct bpf_prog_info, nr_jited_line_info),
11016                 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
11017         },
11018         [BPF_PROG_INFO_PROG_TAGS] = {
11019                 offsetof(struct bpf_prog_info, prog_tags),
11020                 offsetof(struct bpf_prog_info, nr_prog_tags),
11021                 -(int)sizeof(__u8) * BPF_TAG_SIZE,
11022         },
11023
11024 };
11025
11026 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
11027                                            int offset)
11028 {
11029         __u32 *array = (__u32 *)info;
11030
11031         if (offset >= 0)
11032                 return array[offset / sizeof(__u32)];
11033         return -(int)offset;
11034 }
11035
11036 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
11037                                            int offset)
11038 {
11039         __u64 *array = (__u64 *)info;
11040
11041         if (offset >= 0)
11042                 return array[offset / sizeof(__u64)];
11043         return -(int)offset;
11044 }
11045
11046 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
11047                                          __u32 val)
11048 {
11049         __u32 *array = (__u32 *)info;
11050
11051         if (offset >= 0)
11052                 array[offset / sizeof(__u32)] = val;
11053 }
11054
11055 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
11056                                          __u64 val)
11057 {
11058         __u64 *array = (__u64 *)info;
11059
11060         if (offset >= 0)
11061                 array[offset / sizeof(__u64)] = val;
11062 }
11063
11064 struct bpf_prog_info_linear *
11065 bpf_program__get_prog_info_linear(int fd, __u64 arrays)
11066 {
11067         struct bpf_prog_info_linear *info_linear;
11068         struct bpf_prog_info info = {};
11069         __u32 info_len = sizeof(info);
11070         __u32 data_len = 0;
11071         int i, err;
11072         void *ptr;
11073
11074         if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
11075                 return ERR_PTR(-EINVAL);
11076
11077         /* step 1: get array dimensions */
11078         err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
11079         if (err) {
11080                 pr_debug("can't get prog info: %s", strerror(errno));
11081                 return ERR_PTR(-EFAULT);
11082         }
11083
11084         /* step 2: calculate total size of all arrays */
11085         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
11086                 bool include_array = (arrays & (1UL << i)) > 0;
11087                 struct bpf_prog_info_array_desc *desc;
11088                 __u32 count, size;
11089
11090                 desc = bpf_prog_info_array_desc + i;
11091
11092                 /* kernel is too old to support this field */
11093                 if (info_len < desc->array_offset + sizeof(__u32) ||
11094                     info_len < desc->count_offset + sizeof(__u32) ||
11095                     (desc->size_offset > 0 && info_len < desc->size_offset))
11096                         include_array = false;
11097
11098                 if (!include_array) {
11099                         arrays &= ~(1UL << i);  /* clear the bit */
11100                         continue;
11101                 }
11102
11103                 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
11104                 size  = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
11105
11106                 data_len += count * size;
11107         }
11108
11109         /* step 3: allocate continuous memory */
11110         data_len = roundup(data_len, sizeof(__u64));
11111         info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
11112         if (!info_linear)
11113                 return ERR_PTR(-ENOMEM);
11114
11115         /* step 4: fill data to info_linear->info */
11116         info_linear->arrays = arrays;
11117         memset(&info_linear->info, 0, sizeof(info));
11118         ptr = info_linear->data;
11119
11120         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
11121                 struct bpf_prog_info_array_desc *desc;
11122                 __u32 count, size;
11123
11124                 if ((arrays & (1UL << i)) == 0)
11125                         continue;
11126
11127                 desc  = bpf_prog_info_array_desc + i;
11128                 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
11129                 size  = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
11130                 bpf_prog_info_set_offset_u32(&info_linear->info,
11131                                              desc->count_offset, count);
11132                 bpf_prog_info_set_offset_u32(&info_linear->info,
11133                                              desc->size_offset, size);
11134                 bpf_prog_info_set_offset_u64(&info_linear->info,
11135                                              desc->array_offset,
11136                                              ptr_to_u64(ptr));
11137                 ptr += count * size;
11138         }
11139
11140         /* step 5: call syscall again to get required arrays */
11141         err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
11142         if (err) {
11143                 pr_debug("can't get prog info: %s", strerror(errno));
11144                 free(info_linear);
11145                 return ERR_PTR(-EFAULT);
11146         }
11147
11148         /* step 6: verify the data */
11149         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
11150                 struct bpf_prog_info_array_desc *desc;
11151                 __u32 v1, v2;
11152
11153                 if ((arrays & (1UL << i)) == 0)
11154                         continue;
11155
11156                 desc = bpf_prog_info_array_desc + i;
11157                 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
11158                 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
11159                                                    desc->count_offset);
11160                 if (v1 != v2)
11161                         pr_warn("%s: mismatch in element count\n", __func__);
11162
11163                 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
11164                 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
11165                                                    desc->size_offset);
11166                 if (v1 != v2)
11167                         pr_warn("%s: mismatch in rec size\n", __func__);
11168         }
11169
11170         /* step 7: update info_len and data_len */
11171         info_linear->info_len = sizeof(struct bpf_prog_info);
11172         info_linear->data_len = data_len;
11173
11174         return info_linear;
11175 }
11176
11177 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
11178 {
11179         int i;
11180
11181         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
11182                 struct bpf_prog_info_array_desc *desc;
11183                 __u64 addr, offs;
11184
11185                 if ((info_linear->arrays & (1UL << i)) == 0)
11186                         continue;
11187
11188                 desc = bpf_prog_info_array_desc + i;
11189                 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
11190                                                      desc->array_offset);
11191                 offs = addr - ptr_to_u64(info_linear->data);
11192                 bpf_prog_info_set_offset_u64(&info_linear->info,
11193                                              desc->array_offset, offs);
11194         }
11195 }
11196
11197 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
11198 {
11199         int i;
11200
11201         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
11202                 struct bpf_prog_info_array_desc *desc;
11203                 __u64 addr, offs;
11204
11205                 if ((info_linear->arrays & (1UL << i)) == 0)
11206                         continue;
11207
11208                 desc = bpf_prog_info_array_desc + i;
11209                 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
11210                                                      desc->array_offset);
11211                 addr = offs + ptr_to_u64(info_linear->data);
11212                 bpf_prog_info_set_offset_u64(&info_linear->info,
11213                                              desc->array_offset, addr);
11214         }
11215 }
11216
11217 int bpf_program__set_attach_target(struct bpf_program *prog,
11218                                    int attach_prog_fd,
11219                                    const char *attach_func_name)
11220 {
11221         int btf_obj_fd = 0, btf_id = 0, err;
11222
11223         if (!prog || attach_prog_fd < 0 || !attach_func_name)
11224                 return -EINVAL;
11225
11226         if (prog->obj->loaded)
11227                 return -EINVAL;
11228
11229         if (attach_prog_fd) {
11230                 btf_id = libbpf_find_prog_btf_id(attach_func_name,
11231                                                  attach_prog_fd);
11232                 if (btf_id < 0)
11233                         return btf_id;
11234         } else {
11235                 /* load btf_vmlinux, if not yet */
11236                 err = bpf_object__load_vmlinux_btf(prog->obj, true);
11237                 if (err)
11238                         return err;
11239                 err = find_kernel_btf_id(prog->obj, attach_func_name,
11240                                          prog->expected_attach_type,
11241                                          &btf_obj_fd, &btf_id);
11242                 if (err)
11243                         return err;
11244         }
11245
11246         prog->attach_btf_id = btf_id;
11247         prog->attach_btf_obj_fd = btf_obj_fd;
11248         prog->attach_prog_fd = attach_prog_fd;
11249         return 0;
11250 }
11251
11252 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
11253 {
11254         int err = 0, n, len, start, end = -1;
11255         bool *tmp;
11256
11257         *mask = NULL;
11258         *mask_sz = 0;
11259
11260         /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
11261         while (*s) {
11262                 if (*s == ',' || *s == '\n') {
11263                         s++;
11264                         continue;
11265                 }
11266                 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
11267                 if (n <= 0 || n > 2) {
11268                         pr_warn("Failed to get CPU range %s: %d\n", s, n);
11269                         err = -EINVAL;
11270                         goto cleanup;
11271                 } else if (n == 1) {
11272                         end = start;
11273                 }
11274                 if (start < 0 || start > end) {
11275                         pr_warn("Invalid CPU range [%d,%d] in %s\n",
11276                                 start, end, s);
11277                         err = -EINVAL;
11278                         goto cleanup;
11279                 }
11280                 tmp = realloc(*mask, end + 1);
11281                 if (!tmp) {
11282                         err = -ENOMEM;
11283                         goto cleanup;
11284                 }
11285                 *mask = tmp;
11286                 memset(tmp + *mask_sz, 0, start - *mask_sz);
11287                 memset(tmp + start, 1, end - start + 1);
11288                 *mask_sz = end + 1;
11289                 s += len;
11290         }
11291         if (!*mask_sz) {
11292                 pr_warn("Empty CPU range\n");
11293                 return -EINVAL;
11294         }
11295         return 0;
11296 cleanup:
11297         free(*mask);
11298         *mask = NULL;
11299         return err;
11300 }
11301
11302 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
11303 {
11304         int fd, err = 0, len;
11305         char buf[128];
11306
11307         fd = open(fcpu, O_RDONLY);
11308         if (fd < 0) {
11309                 err = -errno;
11310                 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
11311                 return err;
11312         }
11313         len = read(fd, buf, sizeof(buf));
11314         close(fd);
11315         if (len <= 0) {
11316                 err = len ? -errno : -EINVAL;
11317                 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
11318                 return err;
11319         }
11320         if (len >= sizeof(buf)) {
11321                 pr_warn("CPU mask is too big in file %s\n", fcpu);
11322                 return -E2BIG;
11323         }
11324         buf[len] = '\0';
11325
11326         return parse_cpu_mask_str(buf, mask, mask_sz);
11327 }
11328
11329 int libbpf_num_possible_cpus(void)
11330 {
11331         static const char *fcpu = "/sys/devices/system/cpu/possible";
11332         static int cpus;
11333         int err, n, i, tmp_cpus;
11334         bool *mask;
11335
11336         tmp_cpus = READ_ONCE(cpus);
11337         if (tmp_cpus > 0)
11338                 return tmp_cpus;
11339
11340         err = parse_cpu_mask_file(fcpu, &mask, &n);
11341         if (err)
11342                 return err;
11343
11344         tmp_cpus = 0;
11345         for (i = 0; i < n; i++) {
11346                 if (mask[i])
11347                         tmp_cpus++;
11348         }
11349         free(mask);
11350
11351         WRITE_ONCE(cpus, tmp_cpus);
11352         return tmp_cpus;
11353 }
11354
11355 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
11356                               const struct bpf_object_open_opts *opts)
11357 {
11358         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
11359                 .object_name = s->name,
11360         );
11361         struct bpf_object *obj;
11362         int i;
11363
11364         /* Attempt to preserve opts->object_name, unless overriden by user
11365          * explicitly. Overwriting object name for skeletons is discouraged,
11366          * as it breaks global data maps, because they contain object name
11367          * prefix as their own map name prefix. When skeleton is generated,
11368          * bpftool is making an assumption that this name will stay the same.
11369          */
11370         if (opts) {
11371                 memcpy(&skel_opts, opts, sizeof(*opts));
11372                 if (!opts->object_name)
11373                         skel_opts.object_name = s->name;
11374         }
11375
11376         obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
11377         if (IS_ERR(obj)) {
11378                 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n",
11379                         s->name, PTR_ERR(obj));
11380                 return PTR_ERR(obj);
11381         }
11382
11383         *s->obj = obj;
11384
11385         for (i = 0; i < s->map_cnt; i++) {
11386                 struct bpf_map **map = s->maps[i].map;
11387                 const char *name = s->maps[i].name;
11388                 void **mmaped = s->maps[i].mmaped;
11389
11390                 *map = bpf_object__find_map_by_name(obj, name);
11391                 if (!*map) {
11392                         pr_warn("failed to find skeleton map '%s'\n", name);
11393                         return -ESRCH;
11394                 }
11395
11396                 /* externs shouldn't be pre-setup from user code */
11397                 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
11398                         *mmaped = (*map)->mmaped;
11399         }
11400
11401         for (i = 0; i < s->prog_cnt; i++) {
11402                 struct bpf_program **prog = s->progs[i].prog;
11403                 const char *name = s->progs[i].name;
11404
11405                 *prog = bpf_object__find_program_by_name(obj, name);
11406                 if (!*prog) {
11407                         pr_warn("failed to find skeleton program '%s'\n", name);
11408                         return -ESRCH;
11409                 }
11410         }
11411
11412         return 0;
11413 }
11414
11415 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
11416 {
11417         int i, err;
11418
11419         err = bpf_object__load(*s->obj);
11420         if (err) {
11421                 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
11422                 return err;
11423         }
11424
11425         for (i = 0; i < s->map_cnt; i++) {
11426                 struct bpf_map *map = *s->maps[i].map;
11427                 size_t mmap_sz = bpf_map_mmap_sz(map);
11428                 int prot, map_fd = bpf_map__fd(map);
11429                 void **mmaped = s->maps[i].mmaped;
11430
11431                 if (!mmaped)
11432                         continue;
11433
11434                 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
11435                         *mmaped = NULL;
11436                         continue;
11437                 }
11438
11439                 if (map->def.map_flags & BPF_F_RDONLY_PROG)
11440                         prot = PROT_READ;
11441                 else
11442                         prot = PROT_READ | PROT_WRITE;
11443
11444                 /* Remap anonymous mmap()-ed "map initialization image" as
11445                  * a BPF map-backed mmap()-ed memory, but preserving the same
11446                  * memory address. This will cause kernel to change process'
11447                  * page table to point to a different piece of kernel memory,
11448                  * but from userspace point of view memory address (and its
11449                  * contents, being identical at this point) will stay the
11450                  * same. This mapping will be released by bpf_object__close()
11451                  * as per normal clean up procedure, so we don't need to worry
11452                  * about it from skeleton's clean up perspective.
11453                  */
11454                 *mmaped = mmap(map->mmaped, mmap_sz, prot,
11455                                 MAP_SHARED | MAP_FIXED, map_fd, 0);
11456                 if (*mmaped == MAP_FAILED) {
11457                         err = -errno;
11458                         *mmaped = NULL;
11459                         pr_warn("failed to re-mmap() map '%s': %d\n",
11460                                  bpf_map__name(map), err);
11461                         return err;
11462                 }
11463         }
11464
11465         return 0;
11466 }
11467
11468 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
11469 {
11470         int i;
11471
11472         for (i = 0; i < s->prog_cnt; i++) {
11473                 struct bpf_program *prog = *s->progs[i].prog;
11474                 struct bpf_link **link = s->progs[i].link;
11475                 const struct bpf_sec_def *sec_def;
11476
11477                 if (!prog->load)
11478                         continue;
11479
11480                 sec_def = find_sec_def(prog->sec_name);
11481                 if (!sec_def || !sec_def->attach_fn)
11482                         continue;
11483
11484                 *link = sec_def->attach_fn(sec_def, prog);
11485                 if (IS_ERR(*link)) {
11486                         pr_warn("failed to auto-attach program '%s': %ld\n",
11487                                 bpf_program__name(prog), PTR_ERR(*link));
11488                         return PTR_ERR(*link);
11489                 }
11490         }
11491
11492         return 0;
11493 }
11494
11495 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
11496 {
11497         int i;
11498
11499         for (i = 0; i < s->prog_cnt; i++) {
11500                 struct bpf_link **link = s->progs[i].link;
11501
11502                 bpf_link__destroy(*link);
11503                 *link = NULL;
11504         }
11505 }
11506
11507 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
11508 {
11509         if (s->progs)
11510                 bpf_object__detach_skeleton(s);
11511         if (s->obj)
11512                 bpf_object__close(*s->obj);
11513         free(s->maps);
11514         free(s->progs);
11515         free(s);
11516 }