tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  * Copyright (C) 2019 Isovalent, Inc.
  11  */
  12
  13 #ifndef _GNU_SOURCE
  14 #define _GNU_SOURCE
  15 #endif
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <stdarg.h>
  19 #include <libgen.h>
  20 #include <inttypes.h>
  21 #include <limits.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <endian.h>
  25 #include <fcntl.h>
  26 #include <errno.h>
  27 #include <ctype.h>
  28 #include <asm/unistd.h>
  29 #include <linux/err.h>
  30 #include <linux/kernel.h>
  31 #include <linux/bpf.h>
  32 #include <linux/btf.h>
  33 #include <linux/filter.h>
  34 #include <linux/list.h>
  35 #include <linux/limits.h>
  36 #include <linux/perf_event.h>
  37 #include <linux/ring_buffer.h>
  38 #include <linux/version.h>
  39 #include <sys/epoll.h>
  40 #include <sys/ioctl.h>
  41 #include <sys/mman.h>
  42 #include <sys/stat.h>
  43 #include <sys/types.h>
  44 #include <sys/vfs.h>
  45 #include <sys/utsname.h>
  46 #include <sys/resource.h>
  47 #include <libelf.h>
  48 #include <gelf.h>
  49 #include <zlib.h>
  50
  51 #include "libbpf.h"
  52 #include "bpf.h"
  53 #include "btf.h"
  54 #include "str_error.h"
  55 #include "libbpf_internal.h"
  56 #include "hashmap.h"
  57
  58 #ifndef EM_BPF
  59 #define EM_BPF 247
  60 #endif
  61
  62 #ifndef BPF_FS_MAGIC
  63 #define BPF_FS_MAGIC            0xcafe4a11
  64 #endif
  65
  66 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
  67
  68 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
  69  * compilation if user enables corresponding warning. Disable it explicitly.
  70  */
  71 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
  72
  73 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  74
  75 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
  76 static const struct btf_type *
  77 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
  78
  79 static int __base_pr(enum libbpf_print_level level, const char *format,
  80                      va_list args)
  81 {
  82         if (level == LIBBPF_DEBUG)
  83                 return 0;
  84
  85         return vfprintf(stderr, format, args);
  86 }
  87
  88 static libbpf_print_fn_t __libbpf_pr = __base_pr;
  89
  90 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
  91 {
  92         libbpf_print_fn_t old_print_fn = __libbpf_pr;
  93
  94         __libbpf_pr = fn;
  95         return old_print_fn;
  96 }
  97
  98 __printf(2, 3)
  99 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
 100 {
 101         va_list args;
 102
 103         if (!__libbpf_pr)
 104                 return;
 105
 106         va_start(args, format);
 107         __libbpf_pr(level, format, args);
 108         va_end(args);
 109 }
 110
 111 static void pr_perm_msg(int err)
 112 {
 113         struct rlimit limit;
 114         char buf[100];
 115
 116         if (err != -EPERM || geteuid() != 0)
 117                 return;
 118
 119         err = getrlimit(RLIMIT_MEMLOCK, &limit);
 120         if (err)
 121                 return;
 122
 123         if (limit.rlim_cur == RLIM_INFINITY)
 124                 return;
 125
 126         if (limit.rlim_cur < 1024)
 127                 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
 128         else if (limit.rlim_cur < 1024*1024)
 129                 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
 130         else
 131                 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
 132
 133         pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
 134                 buf);
 135 }
 136
 137 #define STRERR_BUFSIZE  128
 138
 139 /* Copied from tools/perf/util/util.h */
 140 #ifndef zfree
 141 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 142 #endif
 143
 144 #ifndef zclose
 145 # define zclose(fd) ({                  \
 146         int ___err = 0;                 \
 147         if ((fd) >= 0)                  \
 148                 ___err = close((fd));   \
 149         fd = -1;                        \
 150         ___err; })
 151 #endif
 152
 153 static inline __u64 ptr_to_u64(const void *ptr)
 154 {
 155         return (__u64) (unsigned long) ptr;
 156 }
 157
 158 enum kern_feature_id {
 159         /* v4.14: kernel support for program & map names. */
 160         FEAT_PROG_NAME,
 161         /* v5.2: kernel support for global data sections. */
 162         FEAT_GLOBAL_DATA,
 163         /* BTF support */
 164         FEAT_BTF,
 165         /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
 166         FEAT_BTF_FUNC,
 167         /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
 168         FEAT_BTF_DATASEC,
 169         /* BTF_FUNC_GLOBAL is supported */
 170         FEAT_BTF_GLOBAL_FUNC,
 171         /* BPF_F_MMAPABLE is supported for arrays */
 172         FEAT_ARRAY_MMAP,
 173         /* kernel support for expected_attach_type in BPF_PROG_LOAD */
 174         FEAT_EXP_ATTACH_TYPE,
 175         /* bpf_probe_read_{kernel,user}[_str] helpers */
 176         FEAT_PROBE_READ_KERN,
 177         /* BPF_PROG_BIND_MAP is supported */
 178         FEAT_PROG_BIND_MAP,
 179         /* Kernel support for module BTFs */
 180         FEAT_MODULE_BTF,
 181         __FEAT_CNT,
 182 };
 183
 184 static bool kernel_supports(enum kern_feature_id feat_id);
 185
 186 enum reloc_type {
 187         RELO_LD64,
 188         RELO_CALL,
 189         RELO_DATA,
 190         RELO_EXTERN,
 191 };
 192
 193 struct reloc_desc {
 194         enum reloc_type type;
 195         int insn_idx;
 196         int map_idx;
 197         int sym_off;
 198         bool processed;
 199 };
 200
 201 struct bpf_sec_def;
 202
 203 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
 204                                         struct bpf_program *prog);
 205
 206 struct bpf_sec_def {
 207         const char *sec;
 208         size_t len;
 209         enum bpf_prog_type prog_type;
 210         enum bpf_attach_type expected_attach_type;
 211         bool is_exp_attach_type_optional;
 212         bool is_attachable;
 213         bool is_attach_btf;
 214         bool is_sleepable;
 215         attach_fn_t attach_fn;
 216 };
 217
 218 /*
 219  * bpf_prog should be a better name but it has been used in
 220  * linux/filter.h.
 221  */
 222 struct bpf_program {
 223         const struct bpf_sec_def *sec_def;
 224         char *sec_name;
 225         size_t sec_idx;
 226         /* this program's instruction offset (in number of instructions)
 227          * within its containing ELF section
 228          */
 229         size_t sec_insn_off;
 230         /* number of original instructions in ELF section belonging to this
 231          * program, not taking into account subprogram instructions possible
 232          * appended later during relocation
 233          */
 234         size_t sec_insn_cnt;
 235         /* Offset (in number of instructions) of the start of instruction
 236          * belonging to this BPF program  within its containing main BPF
 237          * program. For the entry-point (main) BPF program, this is always
 238          * zero. For a sub-program, this gets reset before each of main BPF
 239          * programs are processed and relocated and is used to determined
 240          * whether sub-program was already appended to the main program, and
 241          * if yes, at which instruction offset.
 242          */
 243         size_t sub_insn_off;
 244
 245         char *name;
 246         /* sec_name with / replaced by _; makes recursive pinning
 247          * in bpf_object__pin_programs easier
 248          */
 249         char *pin_name;
 250
 251         /* instructions that belong to BPF program; insns[0] is located at
 252          * sec_insn_off instruction within its ELF section in ELF file, so
 253          * when mapping ELF file instruction index to the local instruction,
 254          * one needs to subtract sec_insn_off; and vice versa.
 255          */
 256         struct bpf_insn *insns;
 257         /* actual number of instruction in this BPF program's image; for
 258          * entry-point BPF programs this includes the size of main program
 259          * itself plus all the used sub-programs, appended at the end
 260          */
 261         size_t insns_cnt;
 262
 263         struct reloc_desc *reloc_desc;
 264         int nr_reloc;
 265         int log_level;
 266
 267         struct {
 268                 int nr;
 269                 int *fds;
 270         } instances;
 271         bpf_program_prep_t preprocessor;
 272
 273         struct bpf_object *obj;
 274         void *priv;
 275         bpf_program_clear_priv_t clear_priv;
 276
 277         bool load;
 278         enum bpf_prog_type type;
 279         enum bpf_attach_type expected_attach_type;
 280         int prog_ifindex;
 281         __u32 attach_btf_obj_fd;
 282         __u32 attach_btf_id;
 283         __u32 attach_prog_fd;
 284         void *func_info;
 285         __u32 func_info_rec_size;
 286         __u32 func_info_cnt;
 287
 288         void *line_info;
 289         __u32 line_info_rec_size;
 290         __u32 line_info_cnt;
 291         __u32 prog_flags;
 292 };
 293
 294 struct bpf_struct_ops {
 295         const char *tname;
 296         const struct btf_type *type;
 297         struct bpf_program **progs;
 298         __u32 *kern_func_off;
 299         /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
 300         void *data;
 301         /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
 302          *      btf_vmlinux's format.
 303          * struct bpf_struct_ops_tcp_congestion_ops {
 304          *      [... some other kernel fields ...]
 305          *      struct tcp_congestion_ops data;
 306          * }
 307          * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
 308          * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
 309          * from "data".
 310          */
 311         void *kern_vdata;
 312         __u32 type_id;
 313 };
 314
 315 #define DATA_SEC ".data"
 316 #define BSS_SEC ".bss"
 317 #define RODATA_SEC ".rodata"
 318 #define KCONFIG_SEC ".kconfig"
 319 #define KSYMS_SEC ".ksyms"
 320 #define STRUCT_OPS_SEC ".struct_ops"
 321
 322 enum libbpf_map_type {
 323         LIBBPF_MAP_UNSPEC,
 324         LIBBPF_MAP_DATA,
 325         LIBBPF_MAP_BSS,
 326         LIBBPF_MAP_RODATA,
 327         LIBBPF_MAP_KCONFIG,
 328 };
 329
 330 static const char * const libbpf_type_to_btf_name[] = {
 331         [LIBBPF_MAP_DATA]       = DATA_SEC,
 332         [LIBBPF_MAP_BSS]        = BSS_SEC,
 333         [LIBBPF_MAP_RODATA]     = RODATA_SEC,
 334         [LIBBPF_MAP_KCONFIG]    = KCONFIG_SEC,
 335 };
 336
 337 struct bpf_map {
 338         char *name;
 339         int fd;
 340         int sec_idx;
 341         size_t sec_offset;
 342         int map_ifindex;
 343         int inner_map_fd;
 344         struct bpf_map_def def;
 345         __u32 numa_node;
 346         __u32 btf_var_idx;
 347         __u32 btf_key_type_id;
 348         __u32 btf_value_type_id;
 349         __u32 btf_vmlinux_value_type_id;
 350         void *priv;
 351         bpf_map_clear_priv_t clear_priv;
 352         enum libbpf_map_type libbpf_type;
 353         void *mmaped;
 354         struct bpf_struct_ops *st_ops;
 355         struct bpf_map *inner_map;
 356         void **init_slots;
 357         int init_slots_sz;
 358         char *pin_path;
 359         bool pinned;
 360         bool reused;
 361 };
 362
 363 enum extern_type {
 364         EXT_UNKNOWN,
 365         EXT_KCFG,
 366         EXT_KSYM,
 367 };
 368
 369 enum kcfg_type {
 370         KCFG_UNKNOWN,
 371         KCFG_CHAR,
 372         KCFG_BOOL,
 373         KCFG_INT,
 374         KCFG_TRISTATE,
 375         KCFG_CHAR_ARR,
 376 };
 377
 378 struct extern_desc {
 379         enum extern_type type;
 380         int sym_idx;
 381         int btf_id;
 382         int sec_btf_id;
 383         const char *name;
 384         bool is_set;
 385         bool is_weak;
 386         union {
 387                 struct {
 388                         enum kcfg_type type;
 389                         int sz;
 390                         int align;
 391                         int data_off;
 392                         bool is_signed;
 393                 } kcfg;
 394                 struct {
 395                         unsigned long long addr;
 396
 397                         /* target btf_id of the corresponding kernel var. */
 398                         int vmlinux_btf_id;
 399
 400                         /* local btf_id of the ksym extern's type. */
 401                         __u32 type_id;
 402                 } ksym;
 403         };
 404 };
 405
 406 static LIST_HEAD(bpf_objects_list);
 407
 408 struct module_btf {
 409         struct btf *btf;
 410         char *name;
 411         __u32 id;
 412         int fd;
 413 };
 414
 415 struct bpf_object {
 416         char name[BPF_OBJ_NAME_LEN];
 417         char license[64];
 418         __u32 kern_version;
 419
 420         struct bpf_program *programs;
 421         size_t nr_programs;
 422         struct bpf_map *maps;
 423         size_t nr_maps;
 424         size_t maps_cap;
 425
 426         char *kconfig;
 427         struct extern_desc *externs;
 428         int nr_extern;
 429         int kconfig_map_idx;
 430         int rodata_map_idx;
 431
 432         bool loaded;
 433         bool has_subcalls;
 434
 435         /*
 436          * Information when doing elf related work. Only valid if fd
 437          * is valid.
 438          */
 439         struct {
 440                 int fd;
 441                 const void *obj_buf;
 442                 size_t obj_buf_sz;
 443                 Elf *elf;
 444                 GElf_Ehdr ehdr;
 445                 Elf_Data *symbols;
 446                 Elf_Data *data;
 447                 Elf_Data *rodata;
 448                 Elf_Data *bss;
 449                 Elf_Data *st_ops_data;
 450                 size_t shstrndx; /* section index for section name strings */
 451                 size_t strtabidx;
 452                 struct {
 453                         GElf_Shdr shdr;
 454                         Elf_Data *data;
 455                 } *reloc_sects;
 456                 int nr_reloc_sects;
 457                 int maps_shndx;
 458                 int btf_maps_shndx;
 459                 __u32 btf_maps_sec_btf_id;
 460                 int text_shndx;
 461                 int symbols_shndx;
 462                 int data_shndx;
 463                 int rodata_shndx;
 464                 int bss_shndx;
 465                 int st_ops_shndx;
 466         } efile;
 467         /*
 468          * All loaded bpf_object is linked in a list, which is
 469          * hidden to caller. bpf_objects__<func> handlers deal with
 470          * all objects.
 471          */
 472         struct list_head list;
 473
 474         struct btf *btf;
 475         struct btf_ext *btf_ext;
 476
 477         /* Parse and load BTF vmlinux if any of the programs in the object need
 478          * it at load time.
 479          */
 480         struct btf *btf_vmlinux;
 481         /* vmlinux BTF override for CO-RE relocations */
 482         struct btf *btf_vmlinux_override;
 483         /* Lazily initialized kernel module BTFs */
 484         struct module_btf *btf_modules;
 485         bool btf_modules_loaded;
 486         size_t btf_module_cnt;
 487         size_t btf_module_cap;
 488
 489         void *priv;
 490         bpf_object_clear_priv_t clear_priv;
 491
 492         char path[];
 493 };
 494 #define obj_elf_valid(o)        ((o)->efile.elf)
 495
 496 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
 497 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
 498 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
 499 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
 500 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr);
 501 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
 502 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
 503 static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
 504                               size_t off, __u32 sym_type, GElf_Sym *sym);
 505
 506 void bpf_program__unload(struct bpf_program *prog)
 507 {
 508         int i;
 509
 510         if (!prog)
 511                 return;
 512
 513         /*
 514          * If the object is opened but the program was never loaded,
 515          * it is possible that prog->instances.nr == -1.
 516          */
 517         if (prog->instances.nr > 0) {
 518                 for (i = 0; i < prog->instances.nr; i++)
 519                         zclose(prog->instances.fds[i]);
 520         } else if (prog->instances.nr != -1) {
 521                 pr_warn("Internal error: instances.nr is %d\n",
 522                         prog->instances.nr);
 523         }
 524
 525         prog->instances.nr = -1;
 526         zfree(&prog->instances.fds);
 527
 528         zfree(&prog->func_info);
 529         zfree(&prog->line_info);
 530 }
 531
 532 static void bpf_program__exit(struct bpf_program *prog)
 533 {
 534         if (!prog)
 535                 return;
 536
 537         if (prog->clear_priv)
 538                 prog->clear_priv(prog, prog->priv);
 539
 540         prog->priv = NULL;
 541         prog->clear_priv = NULL;
 542
 543         bpf_program__unload(prog);
 544         zfree(&prog->name);
 545         zfree(&prog->sec_name);
 546         zfree(&prog->pin_name);
 547         zfree(&prog->insns);
 548         zfree(&prog->reloc_desc);
 549
 550         prog->nr_reloc = 0;
 551         prog->insns_cnt = 0;
 552         prog->sec_idx = -1;
 553 }
 554
 555 static char *__bpf_program__pin_name(struct bpf_program *prog)
 556 {
 557         char *name, *p;
 558
 559         name = p = strdup(prog->sec_name);
 560         while ((p = strchr(p, '/')))
 561                 *p = '_';
 562
 563         return name;
 564 }
 565
 566 static bool insn_is_subprog_call(const struct bpf_insn *insn)
 567 {
 568         return BPF_CLASS(insn->code) == BPF_JMP &&
 569                BPF_OP(insn->code) == BPF_CALL &&
 570                BPF_SRC(insn->code) == BPF_K &&
 571                insn->src_reg == BPF_PSEUDO_CALL &&
 572                insn->dst_reg == 0 &&
 573                insn->off == 0;
 574 }
 575
 576 static int
 577 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
 578                       const char *name, size_t sec_idx, const char *sec_name,
 579                       size_t sec_off, void *insn_data, size_t insn_data_sz)
 580 {
 581         if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
 582                 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
 583                         sec_name, name, sec_off, insn_data_sz);
 584                 return -EINVAL;
 585         }
 586
 587         memset(prog, 0, sizeof(*prog));
 588         prog->obj = obj;
 589
 590         prog->sec_idx = sec_idx;
 591         prog->sec_insn_off = sec_off / BPF_INSN_SZ;
 592         prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
 593         /* insns_cnt can later be increased by appending used subprograms */
 594         prog->insns_cnt = prog->sec_insn_cnt;
 595
 596         prog->type = BPF_PROG_TYPE_UNSPEC;
 597         prog->load = true;
 598
 599         prog->instances.fds = NULL;
 600         prog->instances.nr = -1;
 601
 602         prog->sec_name = strdup(sec_name);
 603         if (!prog->sec_name)
 604                 goto errout;
 605
 606         prog->name = strdup(name);
 607         if (!prog->name)
 608                 goto errout;
 609
 610         prog->pin_name = __bpf_program__pin_name(prog);
 611         if (!prog->pin_name)
 612                 goto errout;
 613
 614         prog->insns = malloc(insn_data_sz);
 615         if (!prog->insns)
 616                 goto errout;
 617         memcpy(prog->insns, insn_data, insn_data_sz);
 618
 619         return 0;
 620 errout:
 621         pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
 622         bpf_program__exit(prog);
 623         return -ENOMEM;
 624 }
 625
 626 static int
 627 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
 628                          const char *sec_name, int sec_idx)
 629 {
 630         struct bpf_program *prog, *progs;
 631         void *data = sec_data->d_buf;
 632         size_t sec_sz = sec_data->d_size, sec_off, prog_sz;
 633         int nr_progs, err;
 634         const char *name;
 635         GElf_Sym sym;
 636
 637         progs = obj->programs;
 638         nr_progs = obj->nr_programs;
 639         sec_off = 0;
 640
 641         while (sec_off < sec_sz) {
 642                 if (elf_sym_by_sec_off(obj, sec_idx, sec_off, STT_FUNC, &sym)) {
 643                         pr_warn("sec '%s': failed to find program symbol at offset %zu\n",
 644                                 sec_name, sec_off);
 645                         return -LIBBPF_ERRNO__FORMAT;
 646                 }
 647
 648                 prog_sz = sym.st_size;
 649
 650                 name = elf_sym_str(obj, sym.st_name);
 651                 if (!name) {
 652                         pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
 653                                 sec_name, sec_off);
 654                         return -LIBBPF_ERRNO__FORMAT;
 655                 }
 656
 657                 if (sec_off + prog_sz > sec_sz) {
 658                         pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
 659                                 sec_name, sec_off);
 660                         return -LIBBPF_ERRNO__FORMAT;
 661                 }
 662
 663                 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
 664                          sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
 665
 666                 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
 667                 if (!progs) {
 668                         /*
 669                          * In this case the original obj->programs
 670                          * is still valid, so don't need special treat for
 671                          * bpf_close_object().
 672                          */
 673                         pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
 674                                 sec_name, name);
 675                         return -ENOMEM;
 676                 }
 677                 obj->programs = progs;
 678
 679                 prog = &progs[nr_progs];
 680
 681                 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
 682                                             sec_off, data + sec_off, prog_sz);
 683                 if (err)
 684                         return err;
 685
 686                 nr_progs++;
 687                 obj->nr_programs = nr_progs;
 688
 689                 sec_off += prog_sz;
 690         }
 691
 692         return 0;
 693 }
 694
 695 static __u32 get_kernel_version(void)
 696 {
 697         __u32 major, minor, patch;
 698         struct utsname info;
 699
 700         uname(&info);
 701         if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
 702                 return 0;
 703         return KERNEL_VERSION(major, minor, patch);
 704 }
 705
 706 static const struct btf_member *
 707 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
 708 {
 709         struct btf_member *m;
 710         int i;
 711
 712         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 713                 if (btf_member_bit_offset(t, i) == bit_offset)
 714                         return m;
 715         }
 716
 717         return NULL;
 718 }
 719
 720 static const struct btf_member *
 721 find_member_by_name(const struct btf *btf, const struct btf_type *t,
 722                     const char *name)
 723 {
 724         struct btf_member *m;
 725         int i;
 726
 727         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 728                 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
 729                         return m;
 730         }
 731
 732         return NULL;
 733 }
 734
 735 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
 736 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
 737                                    const char *name, __u32 kind);
 738
 739 static int
 740 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
 741                            const struct btf_type **type, __u32 *type_id,
 742                            const struct btf_type **vtype, __u32 *vtype_id,
 743                            const struct btf_member **data_member)
 744 {
 745         const struct btf_type *kern_type, *kern_vtype;
 746         const struct btf_member *kern_data_member;
 747         __s32 kern_vtype_id, kern_type_id;
 748         __u32 i;
 749
 750         kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
 751         if (kern_type_id < 0) {
 752                 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
 753                         tname);
 754                 return kern_type_id;
 755         }
 756         kern_type = btf__type_by_id(btf, kern_type_id);
 757
 758         /* Find the corresponding "map_value" type that will be used
 759          * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
 760          * find "struct bpf_struct_ops_tcp_congestion_ops" from the
 761          * btf_vmlinux.
 762          */
 763         kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
 764                                                 tname, BTF_KIND_STRUCT);
 765         if (kern_vtype_id < 0) {
 766                 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
 767                         STRUCT_OPS_VALUE_PREFIX, tname);
 768                 return kern_vtype_id;
 769         }
 770         kern_vtype = btf__type_by_id(btf, kern_vtype_id);
 771
 772         /* Find "struct tcp_congestion_ops" from
 773          * struct bpf_struct_ops_tcp_congestion_ops {
 774          *      [ ... ]
 775          *      struct tcp_congestion_ops data;
 776          * }
 777          */
 778         kern_data_member = btf_members(kern_vtype);
 779         for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
 780                 if (kern_data_member->type == kern_type_id)
 781                         break;
 782         }
 783         if (i == btf_vlen(kern_vtype)) {
 784                 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
 785                         tname, STRUCT_OPS_VALUE_PREFIX, tname);
 786                 return -EINVAL;
 787         }
 788
 789         *type = kern_type;
 790         *type_id = kern_type_id;
 791         *vtype = kern_vtype;
 792         *vtype_id = kern_vtype_id;
 793         *data_member = kern_data_member;
 794
 795         return 0;
 796 }
 797
 798 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
 799 {
 800         return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
 801 }
 802
 803 /* Init the map's fields that depend on kern_btf */
 804 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
 805                                          const struct btf *btf,
 806                                          const struct btf *kern_btf)
 807 {
 808         const struct btf_member *member, *kern_member, *kern_data_member;
 809         const struct btf_type *type, *kern_type, *kern_vtype;
 810         __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
 811         struct bpf_struct_ops *st_ops;
 812         void *data, *kern_data;
 813         const char *tname;
 814         int err;
 815
 816         st_ops = map->st_ops;
 817         type = st_ops->type;
 818         tname = st_ops->tname;
 819         err = find_struct_ops_kern_types(kern_btf, tname,
 820                                          &kern_type, &kern_type_id,
 821                                          &kern_vtype, &kern_vtype_id,
 822                                          &kern_data_member);
 823         if (err)
 824                 return err;
 825
 826         pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
 827                  map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
 828
 829         map->def.value_size = kern_vtype->size;
 830         map->btf_vmlinux_value_type_id = kern_vtype_id;
 831
 832         st_ops->kern_vdata = calloc(1, kern_vtype->size);
 833         if (!st_ops->kern_vdata)
 834                 return -ENOMEM;
 835
 836         data = st_ops->data;
 837         kern_data_off = kern_data_member->offset / 8;
 838         kern_data = st_ops->kern_vdata + kern_data_off;
 839
 840         member = btf_members(type);
 841         for (i = 0; i < btf_vlen(type); i++, member++) {
 842                 const struct btf_type *mtype, *kern_mtype;
 843                 __u32 mtype_id, kern_mtype_id;
 844                 void *mdata, *kern_mdata;
 845                 __s64 msize, kern_msize;
 846                 __u32 moff, kern_moff;
 847                 __u32 kern_member_idx;
 848                 const char *mname;
 849
 850                 mname = btf__name_by_offset(btf, member->name_off);
 851                 kern_member = find_member_by_name(kern_btf, kern_type, mname);
 852                 if (!kern_member) {
 853                         pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
 854                                 map->name, mname);
 855                         return -ENOTSUP;
 856                 }
 857
 858                 kern_member_idx = kern_member - btf_members(kern_type);
 859                 if (btf_member_bitfield_size(type, i) ||
 860                     btf_member_bitfield_size(kern_type, kern_member_idx)) {
 861                         pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
 862                                 map->name, mname);
 863                         return -ENOTSUP;
 864                 }
 865
 866                 moff = member->offset / 8;
 867                 kern_moff = kern_member->offset / 8;
 868
 869                 mdata = data + moff;
 870                 kern_mdata = kern_data + kern_moff;
 871
 872                 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
 873                 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
 874                                                     &kern_mtype_id);
 875                 if (BTF_INFO_KIND(mtype->info) !=
 876                     BTF_INFO_KIND(kern_mtype->info)) {
 877                         pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
 878                                 map->name, mname, BTF_INFO_KIND(mtype->info),
 879                                 BTF_INFO_KIND(kern_mtype->info));
 880                         return -ENOTSUP;
 881                 }
 882
 883                 if (btf_is_ptr(mtype)) {
 884                         struct bpf_program *prog;
 885
 886                         mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
 887                         kern_mtype = skip_mods_and_typedefs(kern_btf,
 888                                                             kern_mtype->type,
 889                                                             &kern_mtype_id);
 890                         if (!btf_is_func_proto(mtype) ||
 891                             !btf_is_func_proto(kern_mtype)) {
 892                                 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
 893                                         map->name, mname);
 894                                 return -ENOTSUP;
 895                         }
 896
 897                         prog = st_ops->progs[i];
 898                         if (!prog) {
 899                                 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
 900                                          map->name, mname);
 901                                 continue;
 902                         }
 903
 904                         prog->attach_btf_id = kern_type_id;
 905                         prog->expected_attach_type = kern_member_idx;
 906
 907                         st_ops->kern_func_off[i] = kern_data_off + kern_moff;
 908
 909                         pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
 910                                  map->name, mname, prog->name, moff,
 911                                  kern_moff);
 912
 913                         continue;
 914                 }
 915
 916                 msize = btf__resolve_size(btf, mtype_id);
 917                 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
 918                 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
 919                         pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
 920                                 map->name, mname, (ssize_t)msize,
 921                                 (ssize_t)kern_msize);
 922                         return -ENOTSUP;
 923                 }
 924
 925                 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
 926                          map->name, mname, (unsigned int)msize,
 927                          moff, kern_moff);
 928                 memcpy(kern_mdata, mdata, msize);
 929         }
 930
 931         return 0;
 932 }
 933
 934 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
 935 {
 936         struct bpf_map *map;
 937         size_t i;
 938         int err;
 939
 940         for (i = 0; i < obj->nr_maps; i++) {
 941                 map = &obj->maps[i];
 942
 943                 if (!bpf_map__is_struct_ops(map))
 944                         continue;
 945
 946                 err = bpf_map__init_kern_struct_ops(map, obj->btf,
 947                                                     obj->btf_vmlinux);
 948                 if (err)
 949                         return err;
 950         }
 951
 952         return 0;
 953 }
 954
 955 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
 956 {
 957         const struct btf_type *type, *datasec;
 958         const struct btf_var_secinfo *vsi;
 959         struct bpf_struct_ops *st_ops;
 960         const char *tname, *var_name;
 961         __s32 type_id, datasec_id;
 962         const struct btf *btf;
 963         struct bpf_map *map;
 964         __u32 i;
 965
 966         if (obj->efile.st_ops_shndx == -1)
 967                 return 0;
 968
 969         btf = obj->btf;
 970         datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
 971                                             BTF_KIND_DATASEC);
 972         if (datasec_id < 0) {
 973                 pr_warn("struct_ops init: DATASEC %s not found\n",
 974                         STRUCT_OPS_SEC);
 975                 return -EINVAL;
 976         }
 977
 978         datasec = btf__type_by_id(btf, datasec_id);
 979         vsi = btf_var_secinfos(datasec);
 980         for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
 981                 type = btf__type_by_id(obj->btf, vsi->type);
 982                 var_name = btf__name_by_offset(obj->btf, type->name_off);
 983
 984                 type_id = btf__resolve_type(obj->btf, vsi->type);
 985                 if (type_id < 0) {
 986                         pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
 987                                 vsi->type, STRUCT_OPS_SEC);
 988                         return -EINVAL;
 989                 }
 990
 991                 type = btf__type_by_id(obj->btf, type_id);
 992                 tname = btf__name_by_offset(obj->btf, type->name_off);
 993                 if (!tname[0]) {
 994                         pr_warn("struct_ops init: anonymous type is not supported\n");
 995                         return -ENOTSUP;
 996                 }
 997                 if (!btf_is_struct(type)) {
 998                         pr_warn("struct_ops init: %s is not a struct\n", tname);
 999                         return -EINVAL;
1000                 }
1001
1002                 map = bpf_object__add_map(obj);
1003                 if (IS_ERR(map))
1004                         return PTR_ERR(map);
1005
1006                 map->sec_idx = obj->efile.st_ops_shndx;
1007                 map->sec_offset = vsi->offset;
1008                 map->name = strdup(var_name);
1009                 if (!map->name)
1010                         return -ENOMEM;
1011
1012                 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1013                 map->def.key_size = sizeof(int);
1014                 map->def.value_size = type->size;
1015                 map->def.max_entries = 1;
1016
1017                 map->st_ops = calloc(1, sizeof(*map->st_ops));
1018                 if (!map->st_ops)
1019                         return -ENOMEM;
1020                 st_ops = map->st_ops;
1021                 st_ops->data = malloc(type->size);
1022                 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1023                 st_ops->kern_func_off = malloc(btf_vlen(type) *
1024                                                sizeof(*st_ops->kern_func_off));
1025                 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1026                         return -ENOMEM;
1027
1028                 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1029                         pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1030                                 var_name, STRUCT_OPS_SEC);
1031                         return -EINVAL;
1032                 }
1033
1034                 memcpy(st_ops->data,
1035                        obj->efile.st_ops_data->d_buf + vsi->offset,
1036                        type->size);
1037                 st_ops->tname = tname;
1038                 st_ops->type = type;
1039                 st_ops->type_id = type_id;
1040
1041                 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1042                          tname, type_id, var_name, vsi->offset);
1043         }
1044
1045         return 0;
1046 }
1047
1048 static struct bpf_object *bpf_object__new(const char *path,
1049                                           const void *obj_buf,
1050                                           size_t obj_buf_sz,
1051                                           const char *obj_name)
1052 {
1053         struct bpf_object *obj;
1054         char *end;
1055
1056         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1057         if (!obj) {
1058                 pr_warn("alloc memory failed for %s\n", path);
1059                 return ERR_PTR(-ENOMEM);
1060         }
1061
1062         strcpy(obj->path, path);
1063         if (obj_name) {
1064                 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
1065                 obj->name[sizeof(obj->name) - 1] = 0;
1066         } else {
1067                 /* Using basename() GNU version which doesn't modify arg. */
1068                 strncpy(obj->name, basename((void *)path),
1069                         sizeof(obj->name) - 1);
1070                 end = strchr(obj->name, '.');
1071                 if (end)
1072                         *end = 0;
1073         }
1074
1075         obj->efile.fd = -1;
1076         /*
1077          * Caller of this function should also call
1078          * bpf_object__elf_finish() after data collection to return
1079          * obj_buf to user. If not, we should duplicate the buffer to
1080          * avoid user freeing them before elf finish.
1081          */
1082         obj->efile.obj_buf = obj_buf;
1083         obj->efile.obj_buf_sz = obj_buf_sz;
1084         obj->efile.maps_shndx = -1;
1085         obj->efile.btf_maps_shndx = -1;
1086         obj->efile.data_shndx = -1;
1087         obj->efile.rodata_shndx = -1;
1088         obj->efile.bss_shndx = -1;
1089         obj->efile.st_ops_shndx = -1;
1090         obj->kconfig_map_idx = -1;
1091         obj->rodata_map_idx = -1;
1092
1093         obj->kern_version = get_kernel_version();
1094         obj->loaded = false;
1095
1096         INIT_LIST_HEAD(&obj->list);
1097         list_add(&obj->list, &bpf_objects_list);
1098         return obj;
1099 }
1100
1101 static void bpf_object__elf_finish(struct bpf_object *obj)
1102 {
1103         if (!obj_elf_valid(obj))
1104                 return;
1105
1106         if (obj->efile.elf) {
1107                 elf_end(obj->efile.elf);
1108                 obj->efile.elf = NULL;
1109         }
1110         obj->efile.symbols = NULL;
1111         obj->efile.data = NULL;
1112         obj->efile.rodata = NULL;
1113         obj->efile.bss = NULL;
1114         obj->efile.st_ops_data = NULL;
1115
1116         zfree(&obj->efile.reloc_sects);
1117         obj->efile.nr_reloc_sects = 0;
1118         zclose(obj->efile.fd);
1119         obj->efile.obj_buf = NULL;
1120         obj->efile.obj_buf_sz = 0;
1121 }
1122
1123 /* if libelf is old and doesn't support mmap(), fall back to read() */
1124 #ifndef ELF_C_READ_MMAP
1125 #define ELF_C_READ_MMAP ELF_C_READ
1126 #endif
1127
1128 static int bpf_object__elf_init(struct bpf_object *obj)
1129 {
1130         int err = 0;
1131         GElf_Ehdr *ep;
1132
1133         if (obj_elf_valid(obj)) {
1134                 pr_warn("elf: init internal error\n");
1135                 return -LIBBPF_ERRNO__LIBELF;
1136         }
1137
1138         if (obj->efile.obj_buf_sz > 0) {
1139                 /*
1140                  * obj_buf should have been validated by
1141                  * bpf_object__open_buffer().
1142                  */
1143                 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1144                                             obj->efile.obj_buf_sz);
1145         } else {
1146                 obj->efile.fd = open(obj->path, O_RDONLY);
1147                 if (obj->efile.fd < 0) {
1148                         char errmsg[STRERR_BUFSIZE], *cp;
1149
1150                         err = -errno;
1151                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1152                         pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1153                         return err;
1154                 }
1155
1156                 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1157         }
1158
1159         if (!obj->efile.elf) {
1160                 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1161                 err = -LIBBPF_ERRNO__LIBELF;
1162                 goto errout;
1163         }
1164
1165         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1166                 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1167                 err = -LIBBPF_ERRNO__FORMAT;
1168                 goto errout;
1169         }
1170         ep = &obj->efile.ehdr;
1171
1172         if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) {
1173                 pr_warn("elf: failed to get section names section index for %s: %s\n",
1174                         obj->path, elf_errmsg(-1));
1175                 err = -LIBBPF_ERRNO__FORMAT;
1176                 goto errout;
1177         }
1178
1179         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1180         if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) {
1181                 pr_warn("elf: failed to get section names strings from %s: %s\n",
1182                         obj->path, elf_errmsg(-1));
1183                 return -LIBBPF_ERRNO__FORMAT;
1184         }
1185
1186         /* Old LLVM set e_machine to EM_NONE */
1187         if (ep->e_type != ET_REL ||
1188             (ep->e_machine && ep->e_machine != EM_BPF)) {
1189                 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1190                 err = -LIBBPF_ERRNO__FORMAT;
1191                 goto errout;
1192         }
1193
1194         return 0;
1195 errout:
1196         bpf_object__elf_finish(obj);
1197         return err;
1198 }
1199
1200 static int bpf_object__check_endianness(struct bpf_object *obj)
1201 {
1202 #if __BYTE_ORDER == __LITTLE_ENDIAN
1203         if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1204                 return 0;
1205 #elif __BYTE_ORDER == __BIG_ENDIAN
1206         if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1207                 return 0;
1208 #else
1209 # error "Unrecognized __BYTE_ORDER__"
1210 #endif
1211         pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1212         return -LIBBPF_ERRNO__ENDIAN;
1213 }
1214
1215 static int
1216 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1217 {
1218         memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1219         pr_debug("license of %s is %s\n", obj->path, obj->license);
1220         return 0;
1221 }
1222
1223 static int
1224 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1225 {
1226         __u32 kver;
1227
1228         if (size != sizeof(kver)) {
1229                 pr_warn("invalid kver section in %s\n", obj->path);
1230                 return -LIBBPF_ERRNO__FORMAT;
1231         }
1232         memcpy(&kver, data, sizeof(kver));
1233         obj->kern_version = kver;
1234         pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1235         return 0;
1236 }
1237
1238 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1239 {
1240         if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1241             type == BPF_MAP_TYPE_HASH_OF_MAPS)
1242                 return true;
1243         return false;
1244 }
1245
1246 int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1247                              __u32 *size)
1248 {
1249         int ret = -ENOENT;
1250
1251         *size = 0;
1252         if (!name) {
1253                 return -EINVAL;
1254         } else if (!strcmp(name, DATA_SEC)) {
1255                 if (obj->efile.data)
1256                         *size = obj->efile.data->d_size;
1257         } else if (!strcmp(name, BSS_SEC)) {
1258                 if (obj->efile.bss)
1259                         *size = obj->efile.bss->d_size;
1260         } else if (!strcmp(name, RODATA_SEC)) {
1261                 if (obj->efile.rodata)
1262                         *size = obj->efile.rodata->d_size;
1263         } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1264                 if (obj->efile.st_ops_data)
1265                         *size = obj->efile.st_ops_data->d_size;
1266         } else {
1267                 Elf_Scn *scn = elf_sec_by_name(obj, name);
1268                 Elf_Data *data = elf_sec_data(obj, scn);
1269
1270                 if (data) {
1271                         ret = 0; /* found it */
1272                         *size = data->d_size;
1273                 }
1274         }
1275
1276         return *size ? 0 : ret;
1277 }
1278
1279 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1280                                 __u32 *off)
1281 {
1282         Elf_Data *symbols = obj->efile.symbols;
1283         const char *sname;
1284         size_t si;
1285
1286         if (!name || !off)
1287                 return -EINVAL;
1288
1289         for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1290                 GElf_Sym sym;
1291
1292                 if (!gelf_getsym(symbols, si, &sym))
1293                         continue;
1294                 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1295                     GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1296                         continue;
1297
1298                 sname = elf_sym_str(obj, sym.st_name);
1299                 if (!sname) {
1300                         pr_warn("failed to get sym name string for var %s\n",
1301                                 name);
1302                         return -EIO;
1303                 }
1304                 if (strcmp(name, sname) == 0) {
1305                         *off = sym.st_value;
1306                         return 0;
1307                 }
1308         }
1309
1310         return -ENOENT;
1311 }
1312
1313 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1314 {
1315         struct bpf_map *new_maps;
1316         size_t new_cap;
1317         int i;
1318
1319         if (obj->nr_maps < obj->maps_cap)
1320                 return &obj->maps[obj->nr_maps++];
1321
1322         new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1323         new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps));
1324         if (!new_maps) {
1325                 pr_warn("alloc maps for object failed\n");
1326                 return ERR_PTR(-ENOMEM);
1327         }
1328
1329         obj->maps_cap = new_cap;
1330         obj->maps = new_maps;
1331
1332         /* zero out new maps */
1333         memset(obj->maps + obj->nr_maps, 0,
1334                (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1335         /*
1336          * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
1337          * when failure (zclose won't close negative fd)).
1338          */
1339         for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1340                 obj->maps[i].fd = -1;
1341                 obj->maps[i].inner_map_fd = -1;
1342         }
1343
1344         return &obj->maps[obj->nr_maps++];
1345 }
1346
1347 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1348 {
1349         long page_sz = sysconf(_SC_PAGE_SIZE);
1350         size_t map_sz;
1351
1352         map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1353         map_sz = roundup(map_sz, page_sz);
1354         return map_sz;
1355 }
1356
1357 static char *internal_map_name(struct bpf_object *obj,
1358                                enum libbpf_map_type type)
1359 {
1360         char map_name[BPF_OBJ_NAME_LEN], *p;
1361         const char *sfx = libbpf_type_to_btf_name[type];
1362         int sfx_len = max((size_t)7, strlen(sfx));
1363         int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1364                           strlen(obj->name));
1365
1366         snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1367                  sfx_len, libbpf_type_to_btf_name[type]);
1368
1369         /* sanitise map name to characters allowed by kernel */
1370         for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1371                 if (!isalnum(*p) && *p != '_' && *p != '.')
1372                         *p = '_';
1373
1374         return strdup(map_name);
1375 }
1376
1377 static int
1378 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1379                               int sec_idx, void *data, size_t data_sz)
1380 {
1381         struct bpf_map_def *def;
1382         struct bpf_map *map;
1383         int err;
1384
1385         map = bpf_object__add_map(obj);
1386         if (IS_ERR(map))
1387                 return PTR_ERR(map);
1388
1389         map->libbpf_type = type;
1390         map->sec_idx = sec_idx;
1391         map->sec_offset = 0;
1392         map->name = internal_map_name(obj, type);
1393         if (!map->name) {
1394                 pr_warn("failed to alloc map name\n");
1395                 return -ENOMEM;
1396         }
1397
1398         def = &map->def;
1399         def->type = BPF_MAP_TYPE_ARRAY;
1400         def->key_size = sizeof(int);
1401         def->value_size = data_sz;
1402         def->max_entries = 1;
1403         def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1404                          ? BPF_F_RDONLY_PROG : 0;
1405         def->map_flags |= BPF_F_MMAPABLE;
1406
1407         pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1408                  map->name, map->sec_idx, map->sec_offset, def->map_flags);
1409
1410         map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1411                            MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1412         if (map->mmaped == MAP_FAILED) {
1413                 err = -errno;
1414                 map->mmaped = NULL;
1415                 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1416                         map->name, err);
1417                 zfree(&map->name);
1418                 return err;
1419         }
1420
1421         if (data)
1422                 memcpy(map->mmaped, data, data_sz);
1423
1424         pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1425         return 0;
1426 }
1427
1428 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1429 {
1430         int err;
1431
1432         /*
1433          * Populate obj->maps with libbpf internal maps.
1434          */
1435         if (obj->efile.data_shndx >= 0) {
1436                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1437                                                     obj->efile.data_shndx,
1438                                                     obj->efile.data->d_buf,
1439                                                     obj->efile.data->d_size);
1440                 if (err)
1441                         return err;
1442         }
1443         if (obj->efile.rodata_shndx >= 0) {
1444                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1445                                                     obj->efile.rodata_shndx,
1446                                                     obj->efile.rodata->d_buf,
1447                                                     obj->efile.rodata->d_size);
1448                 if (err)
1449                         return err;
1450
1451                 obj->rodata_map_idx = obj->nr_maps - 1;
1452         }
1453         if (obj->efile.bss_shndx >= 0) {
1454                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1455                                                     obj->efile.bss_shndx,
1456                                                     NULL,
1457                                                     obj->efile.bss->d_size);
1458                 if (err)
1459                         return err;
1460         }
1461         return 0;
1462 }
1463
1464
1465 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1466                                                const void *name)
1467 {
1468         int i;
1469
1470         for (i = 0; i < obj->nr_extern; i++) {
1471                 if (strcmp(obj->externs[i].name, name) == 0)
1472                         return &obj->externs[i];
1473         }
1474         return NULL;
1475 }
1476
1477 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1478                               char value)
1479 {
1480         switch (ext->kcfg.type) {
1481         case KCFG_BOOL:
1482                 if (value == 'm') {
1483                         pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
1484                                 ext->name, value);
1485                         return -EINVAL;
1486                 }
1487                 *(bool *)ext_val = value == 'y' ? true : false;
1488                 break;
1489         case KCFG_TRISTATE:
1490                 if (value == 'y')
1491                         *(enum libbpf_tristate *)ext_val = TRI_YES;
1492                 else if (value == 'm')
1493                         *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1494                 else /* value == 'n' */
1495                         *(enum libbpf_tristate *)ext_val = TRI_NO;
1496                 break;
1497         case KCFG_CHAR:
1498                 *(char *)ext_val = value;
1499                 break;
1500         case KCFG_UNKNOWN:
1501         case KCFG_INT:
1502         case KCFG_CHAR_ARR:
1503         default:
1504                 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
1505                         ext->name, value);
1506                 return -EINVAL;
1507         }
1508         ext->is_set = true;
1509         return 0;
1510 }
1511
1512 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1513                               const char *value)
1514 {
1515         size_t len;
1516
1517         if (ext->kcfg.type != KCFG_CHAR_ARR) {
1518                 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
1519                 return -EINVAL;
1520         }
1521
1522         len = strlen(value);
1523         if (value[len - 1] != '"') {
1524                 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1525                         ext->name, value);
1526                 return -EINVAL;
1527         }
1528
1529         /* strip quotes */
1530         len -= 2;
1531         if (len >= ext->kcfg.sz) {
1532                 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1533                         ext->name, value, len, ext->kcfg.sz - 1);
1534                 len = ext->kcfg.sz - 1;
1535         }
1536         memcpy(ext_val, value + 1, len);
1537         ext_val[len] = '\0';
1538         ext->is_set = true;
1539         return 0;
1540 }
1541
1542 static int parse_u64(const char *value, __u64 *res)
1543 {
1544         char *value_end;
1545         int err;
1546
1547         errno = 0;
1548         *res = strtoull(value, &value_end, 0);
1549         if (errno) {
1550                 err = -errno;
1551                 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1552                 return err;
1553         }
1554         if (*value_end) {
1555                 pr_warn("failed to parse '%s' as integer completely\n", value);
1556                 return -EINVAL;
1557         }
1558         return 0;
1559 }
1560
1561 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1562 {
1563         int bit_sz = ext->kcfg.sz * 8;
1564
1565         if (ext->kcfg.sz == 8)
1566                 return true;
1567
1568         /* Validate that value stored in u64 fits in integer of `ext->sz`
1569          * bytes size without any loss of information. If the target integer
1570          * is signed, we rely on the following limits of integer type of
1571          * Y bits and subsequent transformation:
1572          *
1573          *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
1574          *            0 <= X + 2^(Y-1) <= 2^Y - 1
1575          *            0 <= X + 2^(Y-1) <  2^Y
1576          *
1577          *  For unsigned target integer, check that all the (64 - Y) bits are
1578          *  zero.
1579          */
1580         if (ext->kcfg.is_signed)
1581                 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1582         else
1583                 return (v >> bit_sz) == 0;
1584 }
1585
1586 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1587                               __u64 value)
1588 {
1589         if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1590                 pr_warn("extern (kcfg) %s=%llu should be integer\n",
1591                         ext->name, (unsigned long long)value);
1592                 return -EINVAL;
1593         }
1594         if (!is_kcfg_value_in_range(ext, value)) {
1595                 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
1596                         ext->name, (unsigned long long)value, ext->kcfg.sz);
1597                 return -ERANGE;
1598         }
1599         switch (ext->kcfg.sz) {
1600                 case 1: *(__u8 *)ext_val = value; break;
1601                 case 2: *(__u16 *)ext_val = value; break;
1602                 case 4: *(__u32 *)ext_val = value; break;
1603                 case 8: *(__u64 *)ext_val = value; break;
1604                 default:
1605                         return -EINVAL;
1606         }
1607         ext->is_set = true;
1608         return 0;
1609 }
1610
1611 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1612                                             char *buf, void *data)
1613 {
1614         struct extern_desc *ext;
1615         char *sep, *value;
1616         int len, err = 0;
1617         void *ext_val;
1618         __u64 num;
1619
1620         if (strncmp(buf, "CONFIG_", 7))
1621                 return 0;
1622
1623         sep = strchr(buf, '=');
1624         if (!sep) {
1625                 pr_warn("failed to parse '%s': no separator\n", buf);
1626                 return -EINVAL;
1627         }
1628
1629         /* Trim ending '\n' */
1630         len = strlen(buf);
1631         if (buf[len - 1] == '\n')
1632                 buf[len - 1] = '\0';
1633         /* Split on '=' and ensure that a value is present. */
1634         *sep = '\0';
1635         if (!sep[1]) {
1636                 *sep = '=';
1637                 pr_warn("failed to parse '%s': no value\n", buf);
1638                 return -EINVAL;
1639         }
1640
1641         ext = find_extern_by_name(obj, buf);
1642         if (!ext || ext->is_set)
1643                 return 0;
1644
1645         ext_val = data + ext->kcfg.data_off;
1646         value = sep + 1;
1647
1648         switch (*value) {
1649         case 'y': case 'n': case 'm':
1650                 err = set_kcfg_value_tri(ext, ext_val, *value);
1651                 break;
1652         case '"':
1653                 err = set_kcfg_value_str(ext, ext_val, value);
1654                 break;
1655         default:
1656                 /* assume integer */
1657                 err = parse_u64(value, &num);
1658                 if (err) {
1659                         pr_warn("extern (kcfg) %s=%s should be integer\n",
1660                                 ext->name, value);
1661                         return err;
1662                 }
1663                 err = set_kcfg_value_num(ext, ext_val, num);
1664                 break;
1665         }
1666         if (err)
1667                 return err;
1668         pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
1669         return 0;
1670 }
1671
1672 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1673 {
1674         char buf[PATH_MAX];
1675         struct utsname uts;
1676         int len, err = 0;
1677         gzFile file;
1678
1679         uname(&uts);
1680         len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1681         if (len < 0)
1682                 return -EINVAL;
1683         else if (len >= PATH_MAX)
1684                 return -ENAMETOOLONG;
1685
1686         /* gzopen also accepts uncompressed files. */
1687         file = gzopen(buf, "r");
1688         if (!file)
1689                 file = gzopen("/proc/config.gz", "r");
1690
1691         if (!file) {
1692                 pr_warn("failed to open system Kconfig\n");
1693                 return -ENOENT;
1694         }
1695
1696         while (gzgets(file, buf, sizeof(buf))) {
1697                 err = bpf_object__process_kconfig_line(obj, buf, data);
1698                 if (err) {
1699                         pr_warn("error parsing system Kconfig line '%s': %d\n",
1700                                 buf, err);
1701                         goto out;
1702                 }
1703         }
1704
1705 out:
1706         gzclose(file);
1707         return err;
1708 }
1709
1710 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1711                                         const char *config, void *data)
1712 {
1713         char buf[PATH_MAX];
1714         int err = 0;
1715         FILE *file;
1716
1717         file = fmemopen((void *)config, strlen(config), "r");
1718         if (!file) {
1719                 err = -errno;
1720                 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1721                 return err;
1722         }
1723
1724         while (fgets(buf, sizeof(buf), file)) {
1725                 err = bpf_object__process_kconfig_line(obj, buf, data);
1726                 if (err) {
1727                         pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1728                                 buf, err);
1729                         break;
1730                 }
1731         }
1732
1733         fclose(file);
1734         return err;
1735 }
1736
1737 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1738 {
1739         struct extern_desc *last_ext = NULL, *ext;
1740         size_t map_sz;
1741         int i, err;
1742
1743         for (i = 0; i < obj->nr_extern; i++) {
1744                 ext = &obj->externs[i];
1745                 if (ext->type == EXT_KCFG)
1746                         last_ext = ext;
1747         }
1748
1749         if (!last_ext)
1750                 return 0;
1751
1752         map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1753         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1754                                             obj->efile.symbols_shndx,
1755                                             NULL, map_sz);
1756         if (err)
1757                 return err;
1758
1759         obj->kconfig_map_idx = obj->nr_maps - 1;
1760
1761         return 0;
1762 }
1763
1764 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1765 {
1766         Elf_Data *symbols = obj->efile.symbols;
1767         int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1768         Elf_Data *data = NULL;
1769         Elf_Scn *scn;
1770
1771         if (obj->efile.maps_shndx < 0)
1772                 return 0;
1773
1774         if (!symbols)
1775                 return -EINVAL;
1776
1777
1778         scn = elf_sec_by_idx(obj, obj->efile.maps_shndx);
1779         data = elf_sec_data(obj, scn);
1780         if (!scn || !data) {
1781                 pr_warn("elf: failed to get legacy map definitions for %s\n",
1782                         obj->path);
1783                 return -EINVAL;
1784         }
1785
1786         /*
1787          * Count number of maps. Each map has a name.
1788          * Array of maps is not supported: only the first element is
1789          * considered.
1790          *
1791          * TODO: Detect array of map and report error.
1792          */
1793         nr_syms = symbols->d_size / sizeof(GElf_Sym);
1794         for (i = 0; i < nr_syms; i++) {
1795                 GElf_Sym sym;
1796
1797                 if (!gelf_getsym(symbols, i, &sym))
1798                         continue;
1799                 if (sym.st_shndx != obj->efile.maps_shndx)
1800                         continue;
1801                 nr_maps++;
1802         }
1803         /* Assume equally sized map definitions */
1804         pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n",
1805                  nr_maps, data->d_size, obj->path);
1806
1807         if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1808                 pr_warn("elf: unable to determine legacy map definition size in %s\n",
1809                         obj->path);
1810                 return -EINVAL;
1811         }
1812         map_def_sz = data->d_size / nr_maps;
1813
1814         /* Fill obj->maps using data in "maps" section.  */
1815         for (i = 0; i < nr_syms; i++) {
1816                 GElf_Sym sym;
1817                 const char *map_name;
1818                 struct bpf_map_def *def;
1819                 struct bpf_map *map;
1820
1821                 if (!gelf_getsym(symbols, i, &sym))
1822                         continue;
1823                 if (sym.st_shndx != obj->efile.maps_shndx)
1824                         continue;
1825
1826                 map = bpf_object__add_map(obj);
1827                 if (IS_ERR(map))
1828                         return PTR_ERR(map);
1829
1830                 map_name = elf_sym_str(obj, sym.st_name);
1831                 if (!map_name) {
1832                         pr_warn("failed to get map #%d name sym string for obj %s\n",
1833                                 i, obj->path);
1834                         return -LIBBPF_ERRNO__FORMAT;
1835                 }
1836
1837                 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1838                 map->sec_idx = sym.st_shndx;
1839                 map->sec_offset = sym.st_value;
1840                 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1841                          map_name, map->sec_idx, map->sec_offset);
1842                 if (sym.st_value + map_def_sz > data->d_size) {
1843                         pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1844                                 obj->path, map_name);
1845                         return -EINVAL;
1846                 }
1847
1848                 map->name = strdup(map_name);
1849                 if (!map->name) {
1850                         pr_warn("failed to alloc map name\n");
1851                         return -ENOMEM;
1852                 }
1853                 pr_debug("map %d is \"%s\"\n", i, map->name);
1854                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1855                 /*
1856                  * If the definition of the map in the object file fits in
1857                  * bpf_map_def, copy it.  Any extra fields in our version
1858                  * of bpf_map_def will default to zero as a result of the
1859                  * calloc above.
1860                  */
1861                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1862                         memcpy(&map->def, def, map_def_sz);
1863                 } else {
1864                         /*
1865                          * Here the map structure being read is bigger than what
1866                          * we expect, truncate if the excess bits are all zero.
1867                          * If they are not zero, reject this map as
1868                          * incompatible.
1869                          */
1870                         char *b;
1871
1872                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
1873                              b < ((char *)def) + map_def_sz; b++) {
1874                                 if (*b != 0) {
1875                                         pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1876                                                 obj->path, map_name);
1877                                         if (strict)
1878                                                 return -EINVAL;
1879                                 }
1880                         }
1881                         memcpy(&map->def, def, sizeof(struct bpf_map_def));
1882                 }
1883         }
1884         return 0;
1885 }
1886
1887 static const struct btf_type *
1888 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1889 {
1890         const struct btf_type *t = btf__type_by_id(btf, id);
1891
1892         if (res_id)
1893                 *res_id = id;
1894
1895         while (btf_is_mod(t) || btf_is_typedef(t)) {
1896                 if (res_id)
1897                         *res_id = t->type;
1898                 t = btf__type_by_id(btf, t->type);
1899         }
1900
1901         return t;
1902 }
1903
1904 static const struct btf_type *
1905 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1906 {
1907         const struct btf_type *t;
1908
1909         t = skip_mods_and_typedefs(btf, id, NULL);
1910         if (!btf_is_ptr(t))
1911                 return NULL;
1912
1913         t = skip_mods_and_typedefs(btf, t->type, res_id);
1914
1915         return btf_is_func_proto(t) ? t : NULL;
1916 }
1917
1918 static const char *btf_kind_str(const struct btf_type *t)
1919 {
1920         switch (btf_kind(t)) {
1921         case BTF_KIND_UNKN: return "void";
1922         case BTF_KIND_INT: return "int";
1923         case BTF_KIND_PTR: return "ptr";
1924         case BTF_KIND_ARRAY: return "array";
1925         case BTF_KIND_STRUCT: return "struct";
1926         case BTF_KIND_UNION: return "union";
1927         case BTF_KIND_ENUM: return "enum";
1928         case BTF_KIND_FWD: return "fwd";
1929         case BTF_KIND_TYPEDEF: return "typedef";
1930         case BTF_KIND_VOLATILE: return "volatile";
1931         case BTF_KIND_CONST: return "const";
1932         case BTF_KIND_RESTRICT: return "restrict";
1933         case BTF_KIND_FUNC: return "func";
1934         case BTF_KIND_FUNC_PROTO: return "func_proto";
1935         case BTF_KIND_VAR: return "var";
1936         case BTF_KIND_DATASEC: return "datasec";
1937         default: return "unknown";
1938         }
1939 }
1940
1941 /*
1942  * Fetch integer attribute of BTF map definition. Such attributes are
1943  * represented using a pointer to an array, in which dimensionality of array
1944  * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
1945  * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
1946  * type definition, while using only sizeof(void *) space in ELF data section.
1947  */
1948 static bool get_map_field_int(const char *map_name, const struct btf *btf,
1949                               const struct btf_member *m, __u32 *res)
1950 {
1951         const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1952         const char *name = btf__name_by_offset(btf, m->name_off);
1953         const struct btf_array *arr_info;
1954         const struct btf_type *arr_t;
1955
1956         if (!btf_is_ptr(t)) {
1957                 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
1958                         map_name, name, btf_kind_str(t));
1959                 return false;
1960         }
1961
1962         arr_t = btf__type_by_id(btf, t->type);
1963         if (!arr_t) {
1964                 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1965                         map_name, name, t->type);
1966                 return false;
1967         }
1968         if (!btf_is_array(arr_t)) {
1969                 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
1970                         map_name, name, btf_kind_str(arr_t));
1971                 return false;
1972         }
1973         arr_info = btf_array(arr_t);
1974         *res = arr_info->nelems;
1975         return true;
1976 }
1977
1978 static int build_map_pin_path(struct bpf_map *map, const char *path)
1979 {
1980         char buf[PATH_MAX];
1981         int len;
1982
1983         if (!path)
1984                 path = "/sys/fs/bpf";
1985
1986         len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
1987         if (len < 0)
1988                 return -EINVAL;
1989         else if (len >= PATH_MAX)
1990                 return -ENAMETOOLONG;
1991
1992         return bpf_map__set_pin_path(map, buf);
1993 }
1994
1995
1996 static int parse_btf_map_def(struct bpf_object *obj,
1997                              struct bpf_map *map,
1998                              const struct btf_type *def,
1999                              bool strict, bool is_inner,
2000                              const char *pin_root_path)
2001 {
2002         const struct btf_type *t;
2003         const struct btf_member *m;
2004         int vlen, i;
2005
2006         vlen = btf_vlen(def);
2007         m = btf_members(def);
2008         for (i = 0; i < vlen; i++, m++) {
2009                 const char *name = btf__name_by_offset(obj->btf, m->name_off);
2010
2011                 if (!name) {
2012                         pr_warn("map '%s': invalid field #%d.\n", map->name, i);
2013                         return -EINVAL;
2014                 }
2015                 if (strcmp(name, "type") == 0) {
2016                         if (!get_map_field_int(map->name, obj->btf, m,
2017                                                &map->def.type))
2018                                 return -EINVAL;
2019                         pr_debug("map '%s': found type = %u.\n",
2020                                  map->name, map->def.type);
2021                 } else if (strcmp(name, "max_entries") == 0) {
2022                         if (!get_map_field_int(map->name, obj->btf, m,
2023                                                &map->def.max_entries))
2024                                 return -EINVAL;
2025                         pr_debug("map '%s': found max_entries = %u.\n",
2026                                  map->name, map->def.max_entries);
2027                 } else if (strcmp(name, "map_flags") == 0) {
2028                         if (!get_map_field_int(map->name, obj->btf, m,
2029                                                &map->def.map_flags))
2030                                 return -EINVAL;
2031                         pr_debug("map '%s': found map_flags = %u.\n",
2032                                  map->name, map->def.map_flags);
2033                 } else if (strcmp(name, "numa_node") == 0) {
2034                         if (!get_map_field_int(map->name, obj->btf, m, &map->numa_node))
2035                                 return -EINVAL;
2036                         pr_debug("map '%s': found numa_node = %u.\n", map->name, map->numa_node);
2037                 } else if (strcmp(name, "key_size") == 0) {
2038                         __u32 sz;
2039
2040                         if (!get_map_field_int(map->name, obj->btf, m, &sz))
2041                                 return -EINVAL;
2042                         pr_debug("map '%s': found key_size = %u.\n",
2043                                  map->name, sz);
2044                         if (map->def.key_size && map->def.key_size != sz) {
2045                                 pr_warn("map '%s': conflicting key size %u != %u.\n",
2046                                         map->name, map->def.key_size, sz);
2047                                 return -EINVAL;
2048                         }
2049                         map->def.key_size = sz;
2050                 } else if (strcmp(name, "key") == 0) {
2051                         __s64 sz;
2052
2053                         t = btf__type_by_id(obj->btf, m->type);
2054                         if (!t) {
2055                                 pr_warn("map '%s': key type [%d] not found.\n",
2056                                         map->name, m->type);
2057                                 return -EINVAL;
2058                         }
2059                         if (!btf_is_ptr(t)) {
2060                                 pr_warn("map '%s': key spec is not PTR: %s.\n",
2061                                         map->name, btf_kind_str(t));
2062                                 return -EINVAL;
2063                         }
2064                         sz = btf__resolve_size(obj->btf, t->type);
2065                         if (sz < 0) {
2066                                 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2067                                         map->name, t->type, (ssize_t)sz);
2068                                 return sz;
2069                         }
2070                         pr_debug("map '%s': found key [%u], sz = %zd.\n",
2071                                  map->name, t->type, (ssize_t)sz);
2072                         if (map->def.key_size && map->def.key_size != sz) {
2073                                 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2074                                         map->name, map->def.key_size, (ssize_t)sz);
2075                                 return -EINVAL;
2076                         }
2077                         map->def.key_size = sz;
2078                         map->btf_key_type_id = t->type;
2079                 } else if (strcmp(name, "value_size") == 0) {
2080                         __u32 sz;
2081
2082                         if (!get_map_field_int(map->name, obj->btf, m, &sz))
2083                                 return -EINVAL;
2084                         pr_debug("map '%s': found value_size = %u.\n",
2085                                  map->name, sz);
2086                         if (map->def.value_size && map->def.value_size != sz) {
2087                                 pr_warn("map '%s': conflicting value size %u != %u.\n",
2088                                         map->name, map->def.value_size, sz);
2089                                 return -EINVAL;
2090                         }
2091                         map->def.value_size = sz;
2092                 } else if (strcmp(name, "value") == 0) {
2093                         __s64 sz;
2094
2095                         t = btf__type_by_id(obj->btf, m->type);
2096                         if (!t) {
2097                                 pr_warn("map '%s': value type [%d] not found.\n",
2098                                         map->name, m->type);
2099                                 return -EINVAL;
2100                         }
2101                         if (!btf_is_ptr(t)) {
2102                                 pr_warn("map '%s': value spec is not PTR: %s.\n",
2103                                         map->name, btf_kind_str(t));
2104                                 return -EINVAL;
2105                         }
2106                         sz = btf__resolve_size(obj->btf, t->type);
2107                         if (sz < 0) {
2108                                 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2109                                         map->name, t->type, (ssize_t)sz);
2110                                 return sz;
2111                         }
2112                         pr_debug("map '%s': found value [%u], sz = %zd.\n",
2113                                  map->name, t->type, (ssize_t)sz);
2114                         if (map->def.value_size && map->def.value_size != sz) {
2115                                 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2116                                         map->name, map->def.value_size, (ssize_t)sz);
2117                                 return -EINVAL;
2118                         }
2119                         map->def.value_size = sz;
2120                         map->btf_value_type_id = t->type;
2121                 }
2122                 else if (strcmp(name, "values") == 0) {
2123                         int err;
2124
2125                         if (is_inner) {
2126                                 pr_warn("map '%s': multi-level inner maps not supported.\n",
2127                                         map->name);
2128                                 return -ENOTSUP;
2129                         }
2130                         if (i != vlen - 1) {
2131                                 pr_warn("map '%s': '%s' member should be last.\n",
2132                                         map->name, name);
2133                                 return -EINVAL;
2134                         }
2135                         if (!bpf_map_type__is_map_in_map(map->def.type)) {
2136                                 pr_warn("map '%s': should be map-in-map.\n",
2137                                         map->name);
2138                                 return -ENOTSUP;
2139                         }
2140                         if (map->def.value_size && map->def.value_size != 4) {
2141                                 pr_warn("map '%s': conflicting value size %u != 4.\n",
2142                                         map->name, map->def.value_size);
2143                                 return -EINVAL;
2144                         }
2145                         map->def.value_size = 4;
2146                         t = btf__type_by_id(obj->btf, m->type);
2147                         if (!t) {
2148                                 pr_warn("map '%s': map-in-map inner type [%d] not found.\n",
2149                                         map->name, m->type);
2150                                 return -EINVAL;
2151                         }
2152                         if (!btf_is_array(t) || btf_array(t)->nelems) {
2153                                 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",
2154                                         map->name);
2155                                 return -EINVAL;
2156                         }
2157                         t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type,
2158                                                    NULL);
2159                         if (!btf_is_ptr(t)) {
2160                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2161                                         map->name, btf_kind_str(t));
2162                                 return -EINVAL;
2163                         }
2164                         t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
2165                         if (!btf_is_struct(t)) {
2166                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2167                                         map->name, btf_kind_str(t));
2168                                 return -EINVAL;
2169                         }
2170
2171                         map->inner_map = calloc(1, sizeof(*map->inner_map));
2172                         if (!map->inner_map)
2173                                 return -ENOMEM;
2174                         map->inner_map->sec_idx = obj->efile.btf_maps_shndx;
2175                         map->inner_map->name = malloc(strlen(map->name) +
2176                                                       sizeof(".inner") + 1);
2177                         if (!map->inner_map->name)
2178                                 return -ENOMEM;
2179                         sprintf(map->inner_map->name, "%s.inner", map->name);
2180
2181                         err = parse_btf_map_def(obj, map->inner_map, t, strict,
2182                                                 true /* is_inner */, NULL);
2183                         if (err)
2184                                 return err;
2185                 } else if (strcmp(name, "pinning") == 0) {
2186                         __u32 val;
2187                         int err;
2188
2189                         if (is_inner) {
2190                                 pr_debug("map '%s': inner def can't be pinned.\n",
2191                                          map->name);
2192                                 return -EINVAL;
2193                         }
2194                         if (!get_map_field_int(map->name, obj->btf, m, &val))
2195                                 return -EINVAL;
2196                         pr_debug("map '%s': found pinning = %u.\n",
2197                                  map->name, val);
2198
2199                         if (val != LIBBPF_PIN_NONE &&
2200                             val != LIBBPF_PIN_BY_NAME) {
2201                                 pr_warn("map '%s': invalid pinning value %u.\n",
2202                                         map->name, val);
2203                                 return -EINVAL;
2204                         }
2205                         if (val == LIBBPF_PIN_BY_NAME) {
2206                                 err = build_map_pin_path(map, pin_root_path);
2207                                 if (err) {
2208                                         pr_warn("map '%s': couldn't build pin path.\n",
2209                                                 map->name);
2210                                         return err;
2211                                 }
2212                         }
2213                 } else {
2214                         if (strict) {
2215                                 pr_warn("map '%s': unknown field '%s'.\n",
2216                                         map->name, name);
2217                                 return -ENOTSUP;
2218                         }
2219                         pr_debug("map '%s': ignoring unknown field '%s'.\n",
2220                                  map->name, name);
2221                 }
2222         }
2223
2224         if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
2225                 pr_warn("map '%s': map type isn't specified.\n", map->name);
2226                 return -EINVAL;
2227         }
2228
2229         return 0;
2230 }
2231
2232 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2233                                          const struct btf_type *sec,
2234                                          int var_idx, int sec_idx,
2235                                          const Elf_Data *data, bool strict,
2236                                          const char *pin_root_path)
2237 {
2238         const struct btf_type *var, *def;
2239         const struct btf_var_secinfo *vi;
2240         const struct btf_var *var_extra;
2241         const char *map_name;
2242         struct bpf_map *map;
2243
2244         vi = btf_var_secinfos(sec) + var_idx;
2245         var = btf__type_by_id(obj->btf, vi->type);
2246         var_extra = btf_var(var);
2247         map_name = btf__name_by_offset(obj->btf, var->name_off);
2248
2249         if (map_name == NULL || map_name[0] == '\0') {
2250                 pr_warn("map #%d: empty name.\n", var_idx);
2251                 return -EINVAL;
2252         }
2253         if ((__u64)vi->offset + vi->size > data->d_size) {
2254                 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2255                 return -EINVAL;
2256         }
2257         if (!btf_is_var(var)) {
2258                 pr_warn("map '%s': unexpected var kind %s.\n",
2259                         map_name, btf_kind_str(var));
2260                 return -EINVAL;
2261         }
2262         if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2263             var_extra->linkage != BTF_VAR_STATIC) {
2264                 pr_warn("map '%s': unsupported var linkage %u.\n",
2265                         map_name, var_extra->linkage);
2266                 return -EOPNOTSUPP;
2267         }
2268
2269         def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2270         if (!btf_is_struct(def)) {
2271                 pr_warn("map '%s': unexpected def kind %s.\n",
2272                         map_name, btf_kind_str(var));
2273                 return -EINVAL;
2274         }
2275         if (def->size > vi->size) {
2276                 pr_warn("map '%s': invalid def size.\n", map_name);
2277                 return -EINVAL;
2278         }
2279
2280         map = bpf_object__add_map(obj);
2281         if (IS_ERR(map))
2282                 return PTR_ERR(map);
2283         map->name = strdup(map_name);
2284         if (!map->name) {
2285                 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2286                 return -ENOMEM;
2287         }
2288         map->libbpf_type = LIBBPF_MAP_UNSPEC;
2289         map->def.type = BPF_MAP_TYPE_UNSPEC;
2290         map->sec_idx = sec_idx;
2291         map->sec_offset = vi->offset;
2292         map->btf_var_idx = var_idx;
2293         pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2294                  map_name, map->sec_idx, map->sec_offset);
2295
2296         return parse_btf_map_def(obj, map, def, strict, false, pin_root_path);
2297 }
2298
2299 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2300                                           const char *pin_root_path)
2301 {
2302         const struct btf_type *sec = NULL;
2303         int nr_types, i, vlen, err;
2304         const struct btf_type *t;
2305         const char *name;
2306         Elf_Data *data;
2307         Elf_Scn *scn;
2308
2309         if (obj->efile.btf_maps_shndx < 0)
2310                 return 0;
2311
2312         scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2313         data = elf_sec_data(obj, scn);
2314         if (!scn || !data) {
2315                 pr_warn("elf: failed to get %s map definitions for %s\n",
2316                         MAPS_ELF_SEC, obj->path);
2317                 return -EINVAL;
2318         }
2319
2320         nr_types = btf__get_nr_types(obj->btf);
2321         for (i = 1; i <= nr_types; i++) {
2322                 t = btf__type_by_id(obj->btf, i);
2323                 if (!btf_is_datasec(t))
2324                         continue;
2325                 name = btf__name_by_offset(obj->btf, t->name_off);
2326                 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2327                         sec = t;
2328                         obj->efile.btf_maps_sec_btf_id = i;
2329                         break;
2330                 }
2331         }
2332
2333         if (!sec) {
2334                 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2335                 return -ENOENT;
2336         }
2337
2338         vlen = btf_vlen(sec);
2339         for (i = 0; i < vlen; i++) {
2340                 err = bpf_object__init_user_btf_map(obj, sec, i,
2341                                                     obj->efile.btf_maps_shndx,
2342                                                     data, strict,
2343                                                     pin_root_path);
2344                 if (err)
2345                         return err;
2346         }
2347
2348         return 0;
2349 }
2350
2351 static int bpf_object__init_maps(struct bpf_object *obj,
2352                                  const struct bpf_object_open_opts *opts)
2353 {
2354         const char *pin_root_path;
2355         bool strict;
2356         int err;
2357
2358         strict = !OPTS_GET(opts, relaxed_maps, false);
2359         pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2360
2361         err = bpf_object__init_user_maps(obj, strict);
2362         err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2363         err = err ?: bpf_object__init_global_data_maps(obj);
2364         err = err ?: bpf_object__init_kconfig_map(obj);
2365         err = err ?: bpf_object__init_struct_ops_maps(obj);
2366         if (err)
2367                 return err;
2368
2369         return 0;
2370 }
2371
2372 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2373 {
2374         GElf_Shdr sh;
2375
2376         if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh))
2377                 return false;
2378
2379         return sh.sh_flags & SHF_EXECINSTR;
2380 }
2381
2382 static bool btf_needs_sanitization(struct bpf_object *obj)
2383 {
2384         bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2385         bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2386         bool has_func = kernel_supports(FEAT_BTF_FUNC);
2387
2388         return !has_func || !has_datasec || !has_func_global;
2389 }
2390
2391 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2392 {
2393         bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2394         bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2395         bool has_func = kernel_supports(FEAT_BTF_FUNC);
2396         struct btf_type *t;
2397         int i, j, vlen;
2398
2399         for (i = 1; i <= btf__get_nr_types(btf); i++) {
2400                 t = (struct btf_type *)btf__type_by_id(btf, i);
2401
2402                 if (!has_datasec && btf_is_var(t)) {
2403                         /* replace VAR with INT */
2404                         t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2405                         /*
2406                          * using size = 1 is the safest choice, 4 will be too
2407                          * big and cause kernel BTF validation failure if
2408                          * original variable took less than 4 bytes
2409                          */
2410                         t->size = 1;
2411                         *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2412                 } else if (!has_datasec && btf_is_datasec(t)) {
2413                         /* replace DATASEC with STRUCT */
2414                         const struct btf_var_secinfo *v = btf_var_secinfos(t);
2415                         struct btf_member *m = btf_members(t);
2416                         struct btf_type *vt;
2417                         char *name;
2418
2419                         name = (char *)btf__name_by_offset(btf, t->name_off);
2420                         while (*name) {
2421                                 if (*name == '.')
2422                                         *name = '_';
2423                                 name++;
2424                         }
2425
2426                         vlen = btf_vlen(t);
2427                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2428                         for (j = 0; j < vlen; j++, v++, m++) {
2429                                 /* order of field assignments is important */
2430                                 m->offset = v->offset * 8;
2431                                 m->type = v->type;
2432                                 /* preserve variable name as member name */
2433                                 vt = (void *)btf__type_by_id(btf, v->type);
2434                                 m->name_off = vt->name_off;
2435                         }
2436                 } else if (!has_func && btf_is_func_proto(t)) {
2437                         /* replace FUNC_PROTO with ENUM */
2438                         vlen = btf_vlen(t);
2439                         t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2440                         t->size = sizeof(__u32); /* kernel enforced */
2441                 } else if (!has_func && btf_is_func(t)) {
2442                         /* replace FUNC with TYPEDEF */
2443                         t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2444                 } else if (!has_func_global && btf_is_func(t)) {
2445                         /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2446                         t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2447                 }
2448         }
2449 }
2450
2451 static bool libbpf_needs_btf(const struct bpf_object *obj)
2452 {
2453         return obj->efile.btf_maps_shndx >= 0 ||
2454                obj->efile.st_ops_shndx >= 0 ||
2455                obj->nr_extern > 0;
2456 }
2457
2458 static bool kernel_needs_btf(const struct bpf_object *obj)
2459 {
2460         return obj->efile.st_ops_shndx >= 0;
2461 }
2462
2463 static int bpf_object__init_btf(struct bpf_object *obj,
2464                                 Elf_Data *btf_data,
2465                                 Elf_Data *btf_ext_data)
2466 {
2467         int err = -ENOENT;
2468
2469         if (btf_data) {
2470                 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2471                 if (IS_ERR(obj->btf)) {
2472                         err = PTR_ERR(obj->btf);
2473                         obj->btf = NULL;
2474                         pr_warn("Error loading ELF section %s: %d.\n",
2475                                 BTF_ELF_SEC, err);
2476                         goto out;
2477                 }
2478                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2479                 btf__set_pointer_size(obj->btf, 8);
2480                 err = 0;
2481         }
2482         if (btf_ext_data) {
2483                 if (!obj->btf) {
2484                         pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2485                                  BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2486                         goto out;
2487                 }
2488                 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
2489                                             btf_ext_data->d_size);
2490                 if (IS_ERR(obj->btf_ext)) {
2491                         pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
2492                                 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
2493                         obj->btf_ext = NULL;
2494                         goto out;
2495                 }
2496         }
2497 out:
2498         if (err && libbpf_needs_btf(obj)) {
2499                 pr_warn("BTF is required, but is missing or corrupted.\n");
2500                 return err;
2501         }
2502         return 0;
2503 }
2504
2505 static int bpf_object__finalize_btf(struct bpf_object *obj)
2506 {
2507         int err;
2508
2509         if (!obj->btf)
2510                 return 0;
2511
2512         err = btf__finalize_data(obj, obj->btf);
2513         if (err) {
2514                 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2515                 return err;
2516         }
2517
2518         return 0;
2519 }
2520
2521 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
2522 {
2523         if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2524             prog->type == BPF_PROG_TYPE_LSM)
2525                 return true;
2526
2527         /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2528          * also need vmlinux BTF
2529          */
2530         if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2531                 return true;
2532
2533         return false;
2534 }
2535
2536 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
2537 {
2538         struct bpf_program *prog;
2539         int i;
2540
2541         /* CO-RE relocations need kernel BTF */
2542         if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
2543                 return true;
2544
2545         /* Support for typed ksyms needs kernel BTF */
2546         for (i = 0; i < obj->nr_extern; i++) {
2547                 const struct extern_desc *ext;
2548
2549                 ext = &obj->externs[i];
2550                 if (ext->type == EXT_KSYM && ext->ksym.type_id)
2551                         return true;
2552         }
2553
2554         bpf_object__for_each_program(prog, obj) {
2555                 if (!prog->load)
2556                         continue;
2557                 if (prog_needs_vmlinux_btf(prog))
2558                         return true;
2559         }
2560
2561         return false;
2562 }
2563
2564 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
2565 {
2566         int err;
2567
2568         /* btf_vmlinux could be loaded earlier */
2569         if (obj->btf_vmlinux)
2570                 return 0;
2571
2572         if (!force && !obj_needs_vmlinux_btf(obj))
2573                 return 0;
2574
2575         obj->btf_vmlinux = libbpf_find_kernel_btf();
2576         if (IS_ERR(obj->btf_vmlinux)) {
2577                 err = PTR_ERR(obj->btf_vmlinux);
2578                 pr_warn("Error loading vmlinux BTF: %d\n", err);
2579                 obj->btf_vmlinux = NULL;
2580                 return err;
2581         }
2582         return 0;
2583 }
2584
2585 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2586 {
2587         struct btf *kern_btf = obj->btf;
2588         bool btf_mandatory, sanitize;
2589         int err = 0;
2590
2591         if (!obj->btf)
2592                 return 0;
2593
2594         if (!kernel_supports(FEAT_BTF)) {
2595                 if (kernel_needs_btf(obj)) {
2596                         err = -EOPNOTSUPP;
2597                         goto report;
2598                 }
2599                 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
2600                 return 0;
2601         }
2602
2603         sanitize = btf_needs_sanitization(obj);
2604         if (sanitize) {
2605                 const void *raw_data;
2606                 __u32 sz;
2607
2608                 /* clone BTF to sanitize a copy and leave the original intact */
2609                 raw_data = btf__get_raw_data(obj->btf, &sz);
2610                 kern_btf = btf__new(raw_data, sz);
2611                 if (IS_ERR(kern_btf))
2612                         return PTR_ERR(kern_btf);
2613
2614                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2615                 btf__set_pointer_size(obj->btf, 8);
2616                 bpf_object__sanitize_btf(obj, kern_btf);
2617         }
2618
2619         err = btf__load(kern_btf);
2620         if (sanitize) {
2621                 if (!err) {
2622                         /* move fd to libbpf's BTF */
2623                         btf__set_fd(obj->btf, btf__fd(kern_btf));
2624                         btf__set_fd(kern_btf, -1);
2625                 }
2626                 btf__free(kern_btf);
2627         }
2628 report:
2629         if (err) {
2630                 btf_mandatory = kernel_needs_btf(obj);
2631                 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
2632                         btf_mandatory ? "BTF is mandatory, can't proceed."
2633                                       : "BTF is optional, ignoring.");
2634                 if (!btf_mandatory)
2635                         err = 0;
2636         }
2637         return err;
2638 }
2639
2640 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
2641 {
2642         const char *name;
2643
2644         name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
2645         if (!name) {
2646                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2647                         off, obj->path, elf_errmsg(-1));
2648                 return NULL;
2649         }
2650
2651         return name;
2652 }
2653
2654 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
2655 {
2656         const char *name;
2657
2658         name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
2659         if (!name) {
2660                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2661                         off, obj->path, elf_errmsg(-1));
2662                 return NULL;
2663         }
2664
2665         return name;
2666 }
2667
2668 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
2669 {
2670         Elf_Scn *scn;
2671
2672         scn = elf_getscn(obj->efile.elf, idx);
2673         if (!scn) {
2674                 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
2675                         idx, obj->path, elf_errmsg(-1));
2676                 return NULL;
2677         }
2678         return scn;
2679 }
2680
2681 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
2682 {
2683         Elf_Scn *scn = NULL;
2684         Elf *elf = obj->efile.elf;
2685         const char *sec_name;
2686
2687         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2688                 sec_name = elf_sec_name(obj, scn);
2689                 if (!sec_name)
2690                         return NULL;
2691
2692                 if (strcmp(sec_name, name) != 0)
2693                         continue;
2694
2695                 return scn;
2696         }
2697         return NULL;
2698 }
2699
2700 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr)
2701 {
2702         if (!scn)
2703                 return -EINVAL;
2704
2705         if (gelf_getshdr(scn, hdr) != hdr) {
2706                 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
2707                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2708                 return -EINVAL;
2709         }
2710
2711         return 0;
2712 }
2713
2714 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
2715 {
2716         const char *name;
2717         GElf_Shdr sh;
2718
2719         if (!scn)
2720                 return NULL;
2721
2722         if (elf_sec_hdr(obj, scn, &sh))
2723                 return NULL;
2724
2725         name = elf_sec_str(obj, sh.sh_name);
2726         if (!name) {
2727                 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
2728                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2729                 return NULL;
2730         }
2731
2732         return name;
2733 }
2734
2735 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
2736 {
2737         Elf_Data *data;
2738
2739         if (!scn)
2740                 return NULL;
2741
2742         data = elf_getdata(scn, 0);
2743         if (!data) {
2744                 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
2745                         elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
2746                         obj->path, elf_errmsg(-1));
2747                 return NULL;
2748         }
2749
2750         return data;
2751 }
2752
2753 static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
2754                               size_t off, __u32 sym_type, GElf_Sym *sym)
2755 {
2756         Elf_Data *symbols = obj->efile.symbols;
2757         size_t n = symbols->d_size / sizeof(GElf_Sym);
2758         int i;
2759
2760         for (i = 0; i < n; i++) {
2761                 if (!gelf_getsym(symbols, i, sym))
2762                         continue;
2763                 if (sym->st_shndx != sec_idx || sym->st_value != off)
2764                         continue;
2765                 if (GELF_ST_TYPE(sym->st_info) != sym_type)
2766                         continue;
2767                 return 0;
2768         }
2769
2770         return -ENOENT;
2771 }
2772
2773 static bool is_sec_name_dwarf(const char *name)
2774 {
2775         /* approximation, but the actual list is too long */
2776         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
2777 }
2778
2779 static bool ignore_elf_section(GElf_Shdr *hdr, const char *name)
2780 {
2781         /* no special handling of .strtab */
2782         if (hdr->sh_type == SHT_STRTAB)
2783                 return true;
2784
2785         /* ignore .llvm_addrsig section as well */
2786         if (hdr->sh_type == 0x6FFF4C03 /* SHT_LLVM_ADDRSIG */)
2787                 return true;
2788
2789         /* no subprograms will lead to an empty .text section, ignore it */
2790         if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
2791             strcmp(name, ".text") == 0)
2792                 return true;
2793
2794         /* DWARF sections */
2795         if (is_sec_name_dwarf(name))
2796                 return true;
2797
2798         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
2799                 name += sizeof(".rel") - 1;
2800                 /* DWARF section relocations */
2801                 if (is_sec_name_dwarf(name))
2802                         return true;
2803
2804                 /* .BTF and .BTF.ext don't need relocations */
2805                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
2806                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
2807                         return true;
2808         }
2809
2810         return false;
2811 }
2812
2813 static int cmp_progs(const void *_a, const void *_b)
2814 {
2815         const struct bpf_program *a = _a;
2816         const struct bpf_program *b = _b;
2817
2818         if (a->sec_idx != b->sec_idx)
2819                 return a->sec_idx < b->sec_idx ? -1 : 1;
2820
2821         /* sec_insn_off can't be the same within the section */
2822         return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
2823 }
2824
2825 static int bpf_object__elf_collect(struct bpf_object *obj)
2826 {
2827         Elf *elf = obj->efile.elf;
2828         Elf_Data *btf_ext_data = NULL;
2829         Elf_Data *btf_data = NULL;
2830         int idx = 0, err = 0;
2831         const char *name;
2832         Elf_Data *data;
2833         Elf_Scn *scn;
2834         GElf_Shdr sh;
2835
2836         /* a bunch of ELF parsing functionality depends on processing symbols,
2837          * so do the first pass and find the symbol table
2838          */
2839         scn = NULL;
2840         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2841                 if (elf_sec_hdr(obj, scn, &sh))
2842                         return -LIBBPF_ERRNO__FORMAT;
2843
2844                 if (sh.sh_type == SHT_SYMTAB) {
2845                         if (obj->efile.symbols) {
2846                                 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
2847                                 return -LIBBPF_ERRNO__FORMAT;
2848                         }
2849
2850                         data = elf_sec_data(obj, scn);
2851                         if (!data)
2852                                 return -LIBBPF_ERRNO__FORMAT;
2853
2854                         obj->efile.symbols = data;
2855                         obj->efile.symbols_shndx = elf_ndxscn(scn);
2856                         obj->efile.strtabidx = sh.sh_link;
2857                 }
2858         }
2859
2860         scn = NULL;
2861         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2862                 idx++;
2863
2864                 if (elf_sec_hdr(obj, scn, &sh))
2865                         return -LIBBPF_ERRNO__FORMAT;
2866
2867                 name = elf_sec_str(obj, sh.sh_name);
2868                 if (!name)
2869                         return -LIBBPF_ERRNO__FORMAT;
2870
2871                 if (ignore_elf_section(&sh, name))
2872                         continue;
2873
2874                 data = elf_sec_data(obj, scn);
2875                 if (!data)
2876                         return -LIBBPF_ERRNO__FORMAT;
2877
2878                 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
2879                          idx, name, (unsigned long)data->d_size,
2880                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
2881                          (int)sh.sh_type);
2882
2883                 if (strcmp(name, "license") == 0) {
2884                         err = bpf_object__init_license(obj, data->d_buf, data->d_size);
2885                         if (err)
2886                                 return err;
2887                 } else if (strcmp(name, "version") == 0) {
2888                         err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
2889                         if (err)
2890                                 return err;
2891                 } else if (strcmp(name, "maps") == 0) {
2892                         obj->efile.maps_shndx = idx;
2893                 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
2894                         obj->efile.btf_maps_shndx = idx;
2895                 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
2896                         btf_data = data;
2897                 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
2898                         btf_ext_data = data;
2899                 } else if (sh.sh_type == SHT_SYMTAB) {
2900                         /* already processed during the first pass above */
2901                 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
2902                         if (sh.sh_flags & SHF_EXECINSTR) {
2903                                 if (strcmp(name, ".text") == 0)
2904                                         obj->efile.text_shndx = idx;
2905                                 err = bpf_object__add_programs(obj, data, name, idx);
2906                                 if (err)
2907                                         return err;
2908                         } else if (strcmp(name, DATA_SEC) == 0) {
2909                                 obj->efile.data = data;
2910                                 obj->efile.data_shndx = idx;
2911                         } else if (strcmp(name, RODATA_SEC) == 0) {
2912                                 obj->efile.rodata = data;
2913                                 obj->efile.rodata_shndx = idx;
2914                         } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
2915                                 obj->efile.st_ops_data = data;
2916                                 obj->efile.st_ops_shndx = idx;
2917                         } else {
2918                                 pr_info("elf: skipping unrecognized data section(%d) %s\n",
2919                                         idx, name);
2920                         }
2921                 } else if (sh.sh_type == SHT_REL) {
2922                         int nr_sects = obj->efile.nr_reloc_sects;
2923                         void *sects = obj->efile.reloc_sects;
2924                         int sec = sh.sh_info; /* points to other section */
2925
2926                         /* Only do relo for section with exec instructions */
2927                         if (!section_have_execinstr(obj, sec) &&
2928                             strcmp(name, ".rel" STRUCT_OPS_SEC) &&
2929                             strcmp(name, ".rel" MAPS_ELF_SEC)) {
2930                                 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
2931                                         idx, name, sec,
2932                                         elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>");
2933                                 continue;
2934                         }
2935
2936                         sects = libbpf_reallocarray(sects, nr_sects + 1,
2937                                                     sizeof(*obj->efile.reloc_sects));
2938                         if (!sects)
2939                                 return -ENOMEM;
2940
2941                         obj->efile.reloc_sects = sects;
2942                         obj->efile.nr_reloc_sects++;
2943
2944                         obj->efile.reloc_sects[nr_sects].shdr = sh;
2945                         obj->efile.reloc_sects[nr_sects].data = data;
2946                 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
2947                         obj->efile.bss = data;
2948                         obj->efile.bss_shndx = idx;
2949                 } else {
2950                         pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
2951                                 (size_t)sh.sh_size);
2952                 }
2953         }
2954
2955         if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
2956                 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
2957                 return -LIBBPF_ERRNO__FORMAT;
2958         }
2959
2960         /* sort BPF programs by section name and in-section instruction offset
2961          * for faster search */
2962         qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
2963
2964         return bpf_object__init_btf(obj, btf_data, btf_ext_data);
2965 }
2966
2967 static bool sym_is_extern(const GElf_Sym *sym)
2968 {
2969         int bind = GELF_ST_BIND(sym->st_info);
2970         /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
2971         return sym->st_shndx == SHN_UNDEF &&
2972                (bind == STB_GLOBAL || bind == STB_WEAK) &&
2973                GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
2974 }
2975
2976 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
2977 {
2978         const struct btf_type *t;
2979         const char *var_name;
2980         int i, n;
2981
2982         if (!btf)
2983                 return -ESRCH;
2984
2985         n = btf__get_nr_types(btf);
2986         for (i = 1; i <= n; i++) {
2987                 t = btf__type_by_id(btf, i);
2988
2989                 if (!btf_is_var(t))
2990                         continue;
2991
2992                 var_name = btf__name_by_offset(btf, t->name_off);
2993                 if (strcmp(var_name, ext_name))
2994                         continue;
2995
2996                 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
2997                         return -EINVAL;
2998
2999                 return i;
3000         }
3001
3002         return -ENOENT;
3003 }
3004
3005 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3006         const struct btf_var_secinfo *vs;
3007         const struct btf_type *t;
3008         int i, j, n;
3009
3010         if (!btf)
3011                 return -ESRCH;
3012
3013         n = btf__get_nr_types(btf);
3014         for (i = 1; i <= n; i++) {
3015                 t = btf__type_by_id(btf, i);
3016
3017                 if (!btf_is_datasec(t))
3018                         continue;
3019
3020                 vs = btf_var_secinfos(t);
3021                 for (j = 0; j < btf_vlen(t); j++, vs++) {
3022                         if (vs->type == ext_btf_id)
3023                                 return i;
3024                 }
3025         }
3026
3027         return -ENOENT;
3028 }
3029
3030 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3031                                      bool *is_signed)
3032 {
3033         const struct btf_type *t;
3034         const char *name;
3035
3036         t = skip_mods_and_typedefs(btf, id, NULL);
3037         name = btf__name_by_offset(btf, t->name_off);
3038
3039         if (is_signed)
3040                 *is_signed = false;
3041         switch (btf_kind(t)) {
3042         case BTF_KIND_INT: {
3043                 int enc = btf_int_encoding(t);
3044
3045                 if (enc & BTF_INT_BOOL)
3046                         return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3047                 if (is_signed)
3048                         *is_signed = enc & BTF_INT_SIGNED;
3049                 if (t->size == 1)
3050                         return KCFG_CHAR;
3051                 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3052                         return KCFG_UNKNOWN;
3053                 return KCFG_INT;
3054         }
3055         case BTF_KIND_ENUM:
3056                 if (t->size != 4)
3057                         return KCFG_UNKNOWN;
3058                 if (strcmp(name, "libbpf_tristate"))
3059                         return KCFG_UNKNOWN;
3060                 return KCFG_TRISTATE;
3061         case BTF_KIND_ARRAY:
3062                 if (btf_array(t)->nelems == 0)
3063                         return KCFG_UNKNOWN;
3064                 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3065                         return KCFG_UNKNOWN;
3066                 return KCFG_CHAR_ARR;
3067         default:
3068                 return KCFG_UNKNOWN;
3069         }
3070 }
3071
3072 static int cmp_externs(const void *_a, const void *_b)
3073 {
3074         const struct extern_desc *a = _a;
3075         const struct extern_desc *b = _b;
3076
3077         if (a->type != b->type)
3078                 return a->type < b->type ? -1 : 1;
3079
3080         if (a->type == EXT_KCFG) {
3081                 /* descending order by alignment requirements */
3082                 if (a->kcfg.align != b->kcfg.align)
3083                         return a->kcfg.align > b->kcfg.align ? -1 : 1;
3084                 /* ascending order by size, within same alignment class */
3085                 if (a->kcfg.sz != b->kcfg.sz)
3086                         return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3087         }
3088
3089         /* resolve ties by name */
3090         return strcmp(a->name, b->name);
3091 }
3092
3093 static int find_int_btf_id(const struct btf *btf)
3094 {
3095         const struct btf_type *t;
3096         int i, n;
3097
3098         n = btf__get_nr_types(btf);
3099         for (i = 1; i <= n; i++) {
3100                 t = btf__type_by_id(btf, i);
3101
3102                 if (btf_is_int(t) && btf_int_bits(t) == 32)
3103                         return i;
3104         }
3105
3106         return 0;
3107 }
3108
3109 static int bpf_object__collect_externs(struct bpf_object *obj)
3110 {
3111         struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3112         const struct btf_type *t;
3113         struct extern_desc *ext;
3114         int i, n, off;
3115         const char *ext_name, *sec_name;
3116         Elf_Scn *scn;
3117         GElf_Shdr sh;
3118
3119         if (!obj->efile.symbols)
3120                 return 0;
3121
3122         scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3123         if (elf_sec_hdr(obj, scn, &sh))
3124                 return -LIBBPF_ERRNO__FORMAT;
3125
3126         n = sh.sh_size / sh.sh_entsize;
3127         pr_debug("looking for externs among %d symbols...\n", n);
3128
3129         for (i = 0; i < n; i++) {
3130                 GElf_Sym sym;
3131
3132                 if (!gelf_getsym(obj->efile.symbols, i, &sym))
3133                         return -LIBBPF_ERRNO__FORMAT;
3134                 if (!sym_is_extern(&sym))
3135                         continue;
3136                 ext_name = elf_sym_str(obj, sym.st_name);
3137                 if (!ext_name || !ext_name[0])
3138                         continue;
3139
3140                 ext = obj->externs;
3141                 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3142                 if (!ext)
3143                         return -ENOMEM;
3144                 obj->externs = ext;
3145                 ext = &ext[obj->nr_extern];
3146                 memset(ext, 0, sizeof(*ext));
3147                 obj->nr_extern++;
3148
3149                 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3150                 if (ext->btf_id <= 0) {
3151                         pr_warn("failed to find BTF for extern '%s': %d\n",
3152                                 ext_name, ext->btf_id);
3153                         return ext->btf_id;
3154                 }
3155                 t = btf__type_by_id(obj->btf, ext->btf_id);
3156                 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3157                 ext->sym_idx = i;
3158                 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
3159
3160                 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3161                 if (ext->sec_btf_id <= 0) {
3162                         pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3163                                 ext_name, ext->btf_id, ext->sec_btf_id);
3164                         return ext->sec_btf_id;
3165                 }
3166                 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3167                 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3168
3169                 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3170                         kcfg_sec = sec;
3171                         ext->type = EXT_KCFG;
3172                         ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3173                         if (ext->kcfg.sz <= 0) {
3174                                 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3175                                         ext_name, ext->kcfg.sz);
3176                                 return ext->kcfg.sz;
3177                         }
3178                         ext->kcfg.align = btf__align_of(obj->btf, t->type);
3179                         if (ext->kcfg.align <= 0) {
3180                                 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3181                                         ext_name, ext->kcfg.align);
3182                                 return -EINVAL;
3183                         }
3184                         ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3185                                                         &ext->kcfg.is_signed);
3186                         if (ext->kcfg.type == KCFG_UNKNOWN) {
3187                                 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name);
3188                                 return -ENOTSUP;
3189                         }
3190                 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3191                         ksym_sec = sec;
3192                         ext->type = EXT_KSYM;
3193                         skip_mods_and_typedefs(obj->btf, t->type,
3194                                                &ext->ksym.type_id);
3195                 } else {
3196                         pr_warn("unrecognized extern section '%s'\n", sec_name);
3197                         return -ENOTSUP;
3198                 }
3199         }
3200         pr_debug("collected %d externs total\n", obj->nr_extern);
3201
3202         if (!obj->nr_extern)
3203                 return 0;
3204
3205         /* sort externs by type, for kcfg ones also by (align, size, name) */
3206         qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3207
3208         /* for .ksyms section, we need to turn all externs into allocated
3209          * variables in BTF to pass kernel verification; we do this by
3210          * pretending that each extern is a 8-byte variable
3211          */
3212         if (ksym_sec) {
3213                 /* find existing 4-byte integer type in BTF to use for fake
3214                  * extern variables in DATASEC
3215                  */
3216                 int int_btf_id = find_int_btf_id(obj->btf);
3217
3218                 for (i = 0; i < obj->nr_extern; i++) {
3219                         ext = &obj->externs[i];
3220                         if (ext->type != EXT_KSYM)
3221                                 continue;
3222                         pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3223                                  i, ext->sym_idx, ext->name);
3224                 }
3225
3226                 sec = ksym_sec;
3227                 n = btf_vlen(sec);
3228                 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3229                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3230                         struct btf_type *vt;
3231
3232                         vt = (void *)btf__type_by_id(obj->btf, vs->type);
3233                         ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3234                         ext = find_extern_by_name(obj, ext_name);
3235                         if (!ext) {
3236                                 pr_warn("failed to find extern definition for BTF var '%s'\n",
3237                                         ext_name);
3238                                 return -ESRCH;
3239                         }
3240                         btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3241                         vt->type = int_btf_id;
3242                         vs->offset = off;
3243                         vs->size = sizeof(int);
3244                 }
3245                 sec->size = off;
3246         }
3247
3248         if (kcfg_sec) {
3249                 sec = kcfg_sec;
3250                 /* for kcfg externs calculate their offsets within a .kconfig map */
3251                 off = 0;
3252                 for (i = 0; i < obj->nr_extern; i++) {
3253                         ext = &obj->externs[i];
3254                         if (ext->type != EXT_KCFG)
3255                                 continue;
3256
3257                         ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3258                         off = ext->kcfg.data_off + ext->kcfg.sz;
3259                         pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3260                                  i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3261                 }
3262                 sec->size = off;
3263                 n = btf_vlen(sec);
3264                 for (i = 0; i < n; i++) {
3265                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3266
3267                         t = btf__type_by_id(obj->btf, vs->type);
3268                         ext_name = btf__name_by_offset(obj->btf, t->name_off);
3269                         ext = find_extern_by_name(obj, ext_name);
3270                         if (!ext) {
3271                                 pr_warn("failed to find extern definition for BTF var '%s'\n",
3272                                         ext_name);
3273                                 return -ESRCH;
3274                         }
3275                         btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3276                         vs->offset = ext->kcfg.data_off;
3277                 }
3278         }
3279         return 0;
3280 }
3281
3282 struct bpf_program *
3283 bpf_object__find_program_by_title(const struct bpf_object *obj,
3284                                   const char *title)
3285 {
3286         struct bpf_program *pos;
3287
3288         bpf_object__for_each_program(pos, obj) {
3289                 if (pos->sec_name && !strcmp(pos->sec_name, title))
3290                         return pos;
3291         }
3292         return NULL;
3293 }
3294
3295 static bool prog_is_subprog(const struct bpf_object *obj,
3296                             const struct bpf_program *prog)
3297 {
3298         /* For legacy reasons, libbpf supports an entry-point BPF programs
3299          * without SEC() attribute, i.e., those in the .text section. But if
3300          * there are 2 or more such programs in the .text section, they all
3301          * must be subprograms called from entry-point BPF programs in
3302          * designated SEC()'tions, otherwise there is no way to distinguish
3303          * which of those programs should be loaded vs which are a subprogram.
3304          * Similarly, if there is a function/program in .text and at least one
3305          * other BPF program with custom SEC() attribute, then we just assume
3306          * .text programs are subprograms (even if they are not called from
3307          * other programs), because libbpf never explicitly supported mixing
3308          * SEC()-designated BPF programs and .text entry-point BPF programs.
3309          */
3310         return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3311 }
3312
3313 struct bpf_program *
3314 bpf_object__find_program_by_name(const struct bpf_object *obj,
3315                                  const char *name)
3316 {
3317         struct bpf_program *prog;
3318
3319         bpf_object__for_each_program(prog, obj) {
3320                 if (prog_is_subprog(obj, prog))
3321                         continue;
3322                 if (!strcmp(prog->name, name))
3323                         return prog;
3324         }
3325         return NULL;
3326 }
3327
3328 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3329                                       int shndx)
3330 {
3331         return shndx == obj->efile.data_shndx ||
3332                shndx == obj->efile.bss_shndx ||
3333                shndx == obj->efile.rodata_shndx;
3334 }
3335
3336 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3337                                       int shndx)
3338 {
3339         return shndx == obj->efile.maps_shndx ||
3340                shndx == obj->efile.btf_maps_shndx;
3341 }
3342
3343 static enum libbpf_map_type
3344 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3345 {
3346         if (shndx == obj->efile.data_shndx)
3347                 return LIBBPF_MAP_DATA;
3348         else if (shndx == obj->efile.bss_shndx)
3349                 return LIBBPF_MAP_BSS;
3350         else if (shndx == obj->efile.rodata_shndx)
3351                 return LIBBPF_MAP_RODATA;
3352         else if (shndx == obj->efile.symbols_shndx)
3353                 return LIBBPF_MAP_KCONFIG;
3354         else
3355                 return LIBBPF_MAP_UNSPEC;
3356 }
3357
3358 static int bpf_program__record_reloc(struct bpf_program *prog,
3359                                      struct reloc_desc *reloc_desc,
3360                                      __u32 insn_idx, const char *sym_name,
3361                                      const GElf_Sym *sym, const GElf_Rel *rel)
3362 {
3363         struct bpf_insn *insn = &prog->insns[insn_idx];
3364         size_t map_idx, nr_maps = prog->obj->nr_maps;
3365         struct bpf_object *obj = prog->obj;
3366         __u32 shdr_idx = sym->st_shndx;
3367         enum libbpf_map_type type;
3368         const char *sym_sec_name;
3369         struct bpf_map *map;
3370
3371         reloc_desc->processed = false;
3372
3373         /* sub-program call relocation */
3374         if (insn->code == (BPF_JMP | BPF_CALL)) {
3375                 if (insn->src_reg != BPF_PSEUDO_CALL) {
3376                         pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3377                         return -LIBBPF_ERRNO__RELOC;
3378                 }
3379                 /* text_shndx can be 0, if no default "main" program exists */
3380                 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3381                         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3382                         pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3383                                 prog->name, sym_name, sym_sec_name);
3384                         return -LIBBPF_ERRNO__RELOC;
3385                 }
3386                 if (sym->st_value % BPF_INSN_SZ) {
3387                         pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3388                                 prog->name, sym_name, (size_t)sym->st_value);
3389                         return -LIBBPF_ERRNO__RELOC;
3390                 }
3391                 reloc_desc->type = RELO_CALL;
3392                 reloc_desc->insn_idx = insn_idx;
3393                 reloc_desc->sym_off = sym->st_value;
3394                 return 0;
3395         }
3396
3397         if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
3398                 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3399                         prog->name, sym_name, insn_idx, insn->code);
3400                 return -LIBBPF_ERRNO__RELOC;
3401         }
3402
3403         if (sym_is_extern(sym)) {
3404                 int sym_idx = GELF_R_SYM(rel->r_info);
3405                 int i, n = obj->nr_extern;
3406                 struct extern_desc *ext;
3407
3408                 for (i = 0; i < n; i++) {
3409                         ext = &obj->externs[i];
3410                         if (ext->sym_idx == sym_idx)
3411                                 break;
3412                 }
3413                 if (i >= n) {
3414                         pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3415                                 prog->name, sym_name, sym_idx);
3416                         return -LIBBPF_ERRNO__RELOC;
3417                 }
3418                 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3419                          prog->name, i, ext->name, ext->sym_idx, insn_idx);
3420                 reloc_desc->type = RELO_EXTERN;
3421                 reloc_desc->insn_idx = insn_idx;
3422                 reloc_desc->sym_off = i; /* sym_off stores extern index */
3423                 return 0;
3424         }
3425
3426         if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
3427                 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
3428                         prog->name, sym_name, shdr_idx);
3429                 return -LIBBPF_ERRNO__RELOC;
3430         }
3431
3432         type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
3433         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3434
3435         /* generic map reference relocation */
3436         if (type == LIBBPF_MAP_UNSPEC) {
3437                 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
3438                         pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
3439                                 prog->name, sym_name, sym_sec_name);
3440                         return -LIBBPF_ERRNO__RELOC;
3441                 }
3442                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3443                         map = &obj->maps[map_idx];
3444                         if (map->libbpf_type != type ||
3445                             map->sec_idx != sym->st_shndx ||
3446                             map->sec_offset != sym->st_value)
3447                                 continue;
3448                         pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
3449                                  prog->name, map_idx, map->name, map->sec_idx,
3450                                  map->sec_offset, insn_idx);
3451                         break;
3452                 }
3453                 if (map_idx >= nr_maps) {
3454                         pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
3455                                 prog->name, sym_sec_name, (size_t)sym->st_value);
3456                         return -LIBBPF_ERRNO__RELOC;
3457                 }
3458                 reloc_desc->type = RELO_LD64;
3459                 reloc_desc->insn_idx = insn_idx;
3460                 reloc_desc->map_idx = map_idx;
3461                 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
3462                 return 0;
3463         }
3464
3465         /* global data map relocation */
3466         if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
3467                 pr_warn("prog '%s': bad data relo against section '%s'\n",
3468                         prog->name, sym_sec_name);
3469                 return -LIBBPF_ERRNO__RELOC;
3470         }
3471         for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3472                 map = &obj->maps[map_idx];
3473                 if (map->libbpf_type != type)
3474                         continue;
3475                 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
3476                          prog->name, map_idx, map->name, map->sec_idx,
3477                          map->sec_offset, insn_idx);
3478                 break;
3479         }
3480         if (map_idx >= nr_maps) {
3481                 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
3482                         prog->name, sym_sec_name);
3483                 return -LIBBPF_ERRNO__RELOC;
3484         }
3485
3486         reloc_desc->type = RELO_DATA;
3487         reloc_desc->insn_idx = insn_idx;
3488         reloc_desc->map_idx = map_idx;
3489         reloc_desc->sym_off = sym->st_value;
3490         return 0;
3491 }
3492
3493 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
3494 {
3495         return insn_idx >= prog->sec_insn_off &&
3496                insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
3497 }
3498
3499 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
3500                                                  size_t sec_idx, size_t insn_idx)
3501 {
3502         int l = 0, r = obj->nr_programs - 1, m;
3503         struct bpf_program *prog;
3504
3505         while (l < r) {
3506                 m = l + (r - l + 1) / 2;
3507                 prog = &obj->programs[m];
3508
3509                 if (prog->sec_idx < sec_idx ||
3510                     (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
3511                         l = m;
3512                 else
3513                         r = m - 1;
3514         }
3515         /* matching program could be at index l, but it still might be the
3516          * wrong one, so we need to double check conditions for the last time
3517          */
3518         prog = &obj->programs[l];
3519         if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
3520                 return prog;
3521         return NULL;
3522 }
3523
3524 static int
3525 bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data)
3526 {
3527         Elf_Data *symbols = obj->efile.symbols;
3528         const char *relo_sec_name, *sec_name;
3529         size_t sec_idx = shdr->sh_info;
3530         struct bpf_program *prog;
3531         struct reloc_desc *relos;
3532         int err, i, nrels;
3533         const char *sym_name;
3534         __u32 insn_idx;
3535         GElf_Sym sym;
3536         GElf_Rel rel;
3537
3538         relo_sec_name = elf_sec_str(obj, shdr->sh_name);
3539         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
3540         if (!relo_sec_name || !sec_name)
3541                 return -EINVAL;
3542
3543         pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
3544                  relo_sec_name, sec_idx, sec_name);
3545         nrels = shdr->sh_size / shdr->sh_entsize;
3546
3547         for (i = 0; i < nrels; i++) {
3548                 if (!gelf_getrel(data, i, &rel)) {
3549                         pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
3550                         return -LIBBPF_ERRNO__FORMAT;
3551                 }
3552                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3553                         pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n",
3554                                 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3555                         return -LIBBPF_ERRNO__FORMAT;
3556                 }
3557                 if (rel.r_offset % BPF_INSN_SZ) {
3558                         pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
3559                                 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3560                         return -LIBBPF_ERRNO__FORMAT;
3561                 }
3562
3563                 insn_idx = rel.r_offset / BPF_INSN_SZ;
3564                 /* relocations against static functions are recorded as
3565                  * relocations against the section that contains a function;
3566                  * in such case, symbol will be STT_SECTION and sym.st_name
3567                  * will point to empty string (0), so fetch section name
3568                  * instead
3569                  */
3570                 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0)
3571                         sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx));
3572                 else
3573                         sym_name = elf_sym_str(obj, sym.st_name);
3574                 sym_name = sym_name ?: "<?";
3575
3576                 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
3577                          relo_sec_name, i, insn_idx, sym_name);
3578
3579                 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
3580                 if (!prog) {
3581                         pr_warn("sec '%s': relo #%d: program not found in section '%s' for insn #%u\n",
3582                                 relo_sec_name, i, sec_name, insn_idx);
3583                         return -LIBBPF_ERRNO__RELOC;
3584                 }
3585
3586                 relos = libbpf_reallocarray(prog->reloc_desc,
3587                                             prog->nr_reloc + 1, sizeof(*relos));
3588                 if (!relos)
3589                         return -ENOMEM;
3590                 prog->reloc_desc = relos;
3591
3592                 /* adjust insn_idx to local BPF program frame of reference */
3593                 insn_idx -= prog->sec_insn_off;
3594                 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
3595                                                 insn_idx, sym_name, &sym, &rel);
3596                 if (err)
3597                         return err;
3598
3599                 prog->nr_reloc++;
3600         }
3601         return 0;
3602 }
3603
3604 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3605 {
3606         struct bpf_map_def *def = &map->def;
3607         __u32 key_type_id = 0, value_type_id = 0;
3608         int ret;
3609
3610         /* if it's BTF-defined map, we don't need to search for type IDs.
3611          * For struct_ops map, it does not need btf_key_type_id and
3612          * btf_value_type_id.
3613          */
3614         if (map->sec_idx == obj->efile.btf_maps_shndx ||
3615             bpf_map__is_struct_ops(map))
3616                 return 0;
3617
3618         if (!bpf_map__is_internal(map)) {
3619                 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3620                                            def->value_size, &key_type_id,
3621                                            &value_type_id);
3622         } else {
3623                 /*
3624                  * LLVM annotates global data differently in BTF, that is,
3625                  * only as '.data', '.bss' or '.rodata'.
3626                  */
3627                 ret = btf__find_by_name(obj->btf,
3628                                 libbpf_type_to_btf_name[map->libbpf_type]);
3629         }
3630         if (ret < 0)
3631                 return ret;
3632
3633         map->btf_key_type_id = key_type_id;
3634         map->btf_value_type_id = bpf_map__is_internal(map) ?
3635                                  ret : value_type_id;
3636         return 0;
3637 }
3638
3639 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3640 {
3641         struct bpf_map_info info = {};
3642         __u32 len = sizeof(info);
3643         int new_fd, err;
3644         char *new_name;
3645
3646         err = bpf_obj_get_info_by_fd(fd, &info, &len);
3647         if (err)
3648                 return err;
3649
3650         new_name = strdup(info.name);
3651         if (!new_name)
3652                 return -errno;
3653
3654         new_fd = open("/", O_RDONLY | O_CLOEXEC);
3655         if (new_fd < 0) {
3656                 err = -errno;
3657                 goto err_free_new_name;
3658         }
3659
3660         new_fd = dup3(fd, new_fd, O_CLOEXEC);
3661         if (new_fd < 0) {
3662                 err = -errno;
3663                 goto err_close_new_fd;
3664         }
3665
3666         err = zclose(map->fd);
3667         if (err) {
3668                 err = -errno;
3669                 goto err_close_new_fd;
3670         }
3671         free(map->name);
3672
3673         map->fd = new_fd;
3674         map->name = new_name;
3675         map->def.type = info.type;
3676         map->def.key_size = info.key_size;
3677         map->def.value_size = info.value_size;
3678         map->def.max_entries = info.max_entries;
3679         map->def.map_flags = info.map_flags;
3680         map->btf_key_type_id = info.btf_key_type_id;
3681         map->btf_value_type_id = info.btf_value_type_id;
3682         map->reused = true;
3683
3684         return 0;
3685
3686 err_close_new_fd:
3687         close(new_fd);
3688 err_free_new_name:
3689         free(new_name);
3690         return err;
3691 }
3692
3693 __u32 bpf_map__max_entries(const struct bpf_map *map)
3694 {
3695         return map->def.max_entries;
3696 }
3697
3698 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
3699 {
3700         if (map->fd >= 0)
3701                 return -EBUSY;
3702         map->def.max_entries = max_entries;
3703         return 0;
3704 }
3705
3706 int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
3707 {
3708         if (!map || !max_entries)
3709                 return -EINVAL;
3710
3711         return bpf_map__set_max_entries(map, max_entries);
3712 }
3713
3714 static int
3715 bpf_object__probe_loading(struct bpf_object *obj)
3716 {
3717         struct bpf_load_program_attr attr;
3718         char *cp, errmsg[STRERR_BUFSIZE];
3719         struct bpf_insn insns[] = {
3720                 BPF_MOV64_IMM(BPF_REG_0, 0),
3721                 BPF_EXIT_INSN(),
3722         };
3723         int ret;
3724
3725         /* make sure basic loading works */
3726
3727         memset(&attr, 0, sizeof(attr));
3728         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3729         attr.insns = insns;
3730         attr.insns_cnt = ARRAY_SIZE(insns);
3731         attr.license = "GPL";
3732
3733         ret = bpf_load_program_xattr(&attr, NULL, 0);
3734         if (ret < 0) {
3735                 ret = errno;
3736                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3737                 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
3738                         "program. Make sure your kernel supports BPF "
3739                         "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
3740                         "set to big enough value.\n", __func__, cp, ret);
3741                 return -ret;
3742         }
3743         close(ret);
3744
3745         return 0;
3746 }
3747
3748 static int probe_fd(int fd)
3749 {
3750         if (fd >= 0)
3751                 close(fd);
3752         return fd >= 0;
3753 }
3754
3755 static int probe_kern_prog_name(void)
3756 {
3757         struct bpf_load_program_attr attr;
3758         struct bpf_insn insns[] = {
3759                 BPF_MOV64_IMM(BPF_REG_0, 0),
3760                 BPF_EXIT_INSN(),
3761         };
3762         int ret;
3763
3764         /* make sure loading with name works */
3765
3766         memset(&attr, 0, sizeof(attr));
3767         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3768         attr.insns = insns;
3769         attr.insns_cnt = ARRAY_SIZE(insns);
3770         attr.license = "GPL";
3771         attr.name = "test";
3772         ret = bpf_load_program_xattr(&attr, NULL, 0);
3773         return probe_fd(ret);
3774 }
3775
3776 static int probe_kern_global_data(void)
3777 {
3778         struct bpf_load_program_attr prg_attr;
3779         struct bpf_create_map_attr map_attr;
3780         char *cp, errmsg[STRERR_BUFSIZE];
3781         struct bpf_insn insns[] = {
3782                 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
3783                 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
3784                 BPF_MOV64_IMM(BPF_REG_0, 0),
3785                 BPF_EXIT_INSN(),
3786         };
3787         int ret, map;
3788
3789         memset(&map_attr, 0, sizeof(map_attr));
3790         map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3791         map_attr.key_size = sizeof(int);
3792         map_attr.value_size = 32;
3793         map_attr.max_entries = 1;
3794
3795         map = bpf_create_map_xattr(&map_attr);
3796         if (map < 0) {
3797                 ret = -errno;
3798                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3799                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3800                         __func__, cp, -ret);
3801                 return ret;
3802         }
3803
3804         insns[0].imm = map;
3805
3806         memset(&prg_attr, 0, sizeof(prg_attr));
3807         prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3808         prg_attr.insns = insns;
3809         prg_attr.insns_cnt = ARRAY_SIZE(insns);
3810         prg_attr.license = "GPL";
3811
3812         ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
3813         close(map);
3814         return probe_fd(ret);
3815 }
3816
3817 static int probe_kern_btf(void)
3818 {
3819         static const char strs[] = "\0int";
3820         __u32 types[] = {
3821                 /* int */
3822                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3823         };
3824
3825         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3826                                              strs, sizeof(strs)));
3827 }
3828
3829 static int probe_kern_btf_func(void)
3830 {
3831         static const char strs[] = "\0int\0x\0a";
3832         /* void x(int a) {} */
3833         __u32 types[] = {
3834                 /* int */
3835                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
3836                 /* FUNC_PROTO */                                /* [2] */
3837                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3838                 BTF_PARAM_ENC(7, 1),
3839                 /* FUNC x */                                    /* [3] */
3840                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
3841         };
3842
3843         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3844                                              strs, sizeof(strs)));
3845 }
3846
3847 static int probe_kern_btf_func_global(void)
3848 {
3849         static const char strs[] = "\0int\0x\0a";
3850         /* static void x(int a) {} */
3851         __u32 types[] = {
3852                 /* int */
3853                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
3854                 /* FUNC_PROTO */                                /* [2] */
3855                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3856                 BTF_PARAM_ENC(7, 1),
3857                 /* FUNC x BTF_FUNC_GLOBAL */                    /* [3] */
3858                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
3859         };
3860
3861         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3862                                              strs, sizeof(strs)));
3863 }
3864
3865 static int probe_kern_btf_datasec(void)
3866 {
3867         static const char strs[] = "\0x\0.data";
3868         /* static int a; */
3869         __u32 types[] = {
3870                 /* int */
3871                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
3872                 /* VAR x */                                     /* [2] */
3873                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
3874                 BTF_VAR_STATIC,
3875                 /* DATASEC val */                               /* [3] */
3876                 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
3877                 BTF_VAR_SECINFO_ENC(2, 0, 4),
3878         };
3879
3880         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3881                                              strs, sizeof(strs)));
3882 }
3883
3884 static int probe_kern_array_mmap(void)
3885 {
3886         struct bpf_create_map_attr attr = {
3887                 .map_type = BPF_MAP_TYPE_ARRAY,
3888                 .map_flags = BPF_F_MMAPABLE,
3889                 .key_size = sizeof(int),
3890                 .value_size = sizeof(int),
3891                 .max_entries = 1,
3892         };
3893
3894         return probe_fd(bpf_create_map_xattr(&attr));
3895 }
3896
3897 static int probe_kern_exp_attach_type(void)
3898 {
3899         struct bpf_load_program_attr attr;
3900         struct bpf_insn insns[] = {
3901                 BPF_MOV64_IMM(BPF_REG_0, 0),
3902                 BPF_EXIT_INSN(),
3903         };
3904
3905         memset(&attr, 0, sizeof(attr));
3906         /* use any valid combination of program type and (optional)
3907          * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
3908          * to see if kernel supports expected_attach_type field for
3909          * BPF_PROG_LOAD command
3910          */
3911         attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK;
3912         attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE;
3913         attr.insns = insns;
3914         attr.insns_cnt = ARRAY_SIZE(insns);
3915         attr.license = "GPL";
3916
3917         return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
3918 }
3919
3920 static int probe_kern_probe_read_kernel(void)
3921 {
3922         struct bpf_load_program_attr attr;
3923         struct bpf_insn insns[] = {
3924                 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),   /* r1 = r10 (fp) */
3925                 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),  /* r1 += -8 */
3926                 BPF_MOV64_IMM(BPF_REG_2, 8),            /* r2 = 8 */
3927                 BPF_MOV64_IMM(BPF_REG_3, 0),            /* r3 = 0 */
3928                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
3929                 BPF_EXIT_INSN(),
3930         };
3931
3932         memset(&attr, 0, sizeof(attr));
3933         attr.prog_type = BPF_PROG_TYPE_KPROBE;
3934         attr.insns = insns;
3935         attr.insns_cnt = ARRAY_SIZE(insns);
3936         attr.license = "GPL";
3937
3938         return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
3939 }
3940
3941 static int probe_prog_bind_map(void)
3942 {
3943         struct bpf_load_program_attr prg_attr;
3944         struct bpf_create_map_attr map_attr;
3945         char *cp, errmsg[STRERR_BUFSIZE];
3946         struct bpf_insn insns[] = {
3947                 BPF_MOV64_IMM(BPF_REG_0, 0),
3948                 BPF_EXIT_INSN(),
3949         };
3950         int ret, map, prog;
3951
3952         memset(&map_attr, 0, sizeof(map_attr));
3953         map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3954         map_attr.key_size = sizeof(int);
3955         map_attr.value_size = 32;
3956         map_attr.max_entries = 1;
3957
3958         map = bpf_create_map_xattr(&map_attr);
3959         if (map < 0) {
3960                 ret = -errno;
3961                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3962                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3963                         __func__, cp, -ret);
3964                 return ret;
3965         }
3966
3967         memset(&prg_attr, 0, sizeof(prg_attr));
3968         prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3969         prg_attr.insns = insns;
3970         prg_attr.insns_cnt = ARRAY_SIZE(insns);
3971         prg_attr.license = "GPL";
3972
3973         prog = bpf_load_program_xattr(&prg_attr, NULL, 0);
3974         if (prog < 0) {
3975                 close(map);
3976                 return 0;
3977         }
3978
3979         ret = bpf_prog_bind_map(prog, map, NULL);
3980
3981         close(map);
3982         close(prog);
3983
3984         return ret >= 0;
3985 }
3986
3987 static int probe_module_btf(void)
3988 {
3989         static const char strs[] = "\0int";
3990         __u32 types[] = {
3991                 /* int */
3992                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3993         };
3994         struct bpf_btf_info info;
3995         __u32 len = sizeof(info);
3996         char name[16];
3997         int fd, err;
3998
3999         fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4000         if (fd < 0)
4001                 return 0; /* BTF not supported at all */
4002
4003         memset(&info, 0, sizeof(info));
4004         info.name = ptr_to_u64(name);
4005         info.name_len = sizeof(name);
4006
4007         /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4008          * kernel's module BTF support coincides with support for
4009          * name/name_len fields in struct bpf_btf_info.
4010          */
4011         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4012         close(fd);
4013         return !err;
4014 }
4015
4016 enum kern_feature_result {
4017         FEAT_UNKNOWN = 0,
4018         FEAT_SUPPORTED = 1,
4019         FEAT_MISSING = 2,
4020 };
4021
4022 typedef int (*feature_probe_fn)(void);
4023
4024 static struct kern_feature_desc {
4025         const char *desc;
4026         feature_probe_fn probe;
4027         enum kern_feature_result res;
4028 } feature_probes[__FEAT_CNT] = {
4029         [FEAT_PROG_NAME] = {
4030                 "BPF program name", probe_kern_prog_name,
4031         },
4032         [FEAT_GLOBAL_DATA] = {
4033                 "global variables", probe_kern_global_data,
4034         },
4035         [FEAT_BTF] = {
4036                 "minimal BTF", probe_kern_btf,
4037         },
4038         [FEAT_BTF_FUNC] = {
4039                 "BTF functions", probe_kern_btf_func,
4040         },
4041         [FEAT_BTF_GLOBAL_FUNC] = {
4042                 "BTF global function", probe_kern_btf_func_global,
4043         },
4044         [FEAT_BTF_DATASEC] = {
4045                 "BTF data section and variable", probe_kern_btf_datasec,
4046         },
4047         [FEAT_ARRAY_MMAP] = {
4048                 "ARRAY map mmap()", probe_kern_array_mmap,
4049         },
4050         [FEAT_EXP_ATTACH_TYPE] = {
4051                 "BPF_PROG_LOAD expected_attach_type attribute",
4052                 probe_kern_exp_attach_type,
4053         },
4054         [FEAT_PROBE_READ_KERN] = {
4055                 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4056         },
4057         [FEAT_PROG_BIND_MAP] = {
4058                 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4059         },
4060         [FEAT_MODULE_BTF] = {
4061                 "module BTF support", probe_module_btf,
4062         },
4063 };
4064
4065 static bool kernel_supports(enum kern_feature_id feat_id)
4066 {
4067         struct kern_feature_desc *feat = &feature_probes[feat_id];
4068         int ret;
4069
4070         if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4071                 ret = feat->probe();
4072                 if (ret > 0) {
4073                         WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4074                 } else if (ret == 0) {
4075                         WRITE_ONCE(feat->res, FEAT_MISSING);
4076                 } else {
4077                         pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4078                         WRITE_ONCE(feat->res, FEAT_MISSING);
4079                 }
4080         }
4081
4082         return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4083 }
4084
4085 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4086 {
4087         struct bpf_map_info map_info = {};
4088         char msg[STRERR_BUFSIZE];
4089         __u32 map_info_len;
4090
4091         map_info_len = sizeof(map_info);
4092
4093         if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
4094                 pr_warn("failed to get map info for map FD %d: %s\n",
4095                         map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
4096                 return false;
4097         }
4098
4099         return (map_info.type == map->def.type &&
4100                 map_info.key_size == map->def.key_size &&
4101                 map_info.value_size == map->def.value_size &&
4102                 map_info.max_entries == map->def.max_entries &&
4103                 map_info.map_flags == map->def.map_flags);
4104 }
4105
4106 static int
4107 bpf_object__reuse_map(struct bpf_map *map)
4108 {
4109         char *cp, errmsg[STRERR_BUFSIZE];
4110         int err, pin_fd;
4111
4112         pin_fd = bpf_obj_get(map->pin_path);
4113         if (pin_fd < 0) {
4114                 err = -errno;
4115                 if (err == -ENOENT) {
4116                         pr_debug("found no pinned map to reuse at '%s'\n",
4117                                  map->pin_path);
4118                         return 0;
4119                 }
4120
4121                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4122                 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4123                         map->pin_path, cp);
4124                 return err;
4125         }
4126
4127         if (!map_is_reuse_compat(map, pin_fd)) {
4128                 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4129                         map->pin_path);
4130                 close(pin_fd);
4131                 return -EINVAL;
4132         }
4133
4134         err = bpf_map__reuse_fd(map, pin_fd);
4135         if (err) {
4136                 close(pin_fd);
4137                 return err;
4138         }
4139         map->pinned = true;
4140         pr_debug("reused pinned map at '%s'\n", map->pin_path);
4141
4142         return 0;
4143 }
4144
4145 static int
4146 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4147 {
4148         enum libbpf_map_type map_type = map->libbpf_type;
4149         char *cp, errmsg[STRERR_BUFSIZE];
4150         int err, zero = 0;
4151
4152         err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4153         if (err) {
4154                 err = -errno;
4155                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4156                 pr_warn("Error setting initial map(%s) contents: %s\n",
4157                         map->name, cp);
4158                 return err;
4159         }
4160
4161         /* Freeze .rodata and .kconfig map as read-only from syscall side. */
4162         if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4163                 err = bpf_map_freeze(map->fd);
4164                 if (err) {
4165                         err = -errno;
4166                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4167                         pr_warn("Error freezing map(%s) as read-only: %s\n",
4168                                 map->name, cp);
4169                         return err;
4170                 }
4171         }
4172         return 0;
4173 }
4174
4175 static void bpf_map__destroy(struct bpf_map *map);
4176
4177 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map)
4178 {
4179         struct bpf_create_map_attr create_attr;
4180         struct bpf_map_def *def = &map->def;
4181
4182         memset(&create_attr, 0, sizeof(create_attr));
4183
4184         if (kernel_supports(FEAT_PROG_NAME))
4185                 create_attr.name = map->name;
4186         create_attr.map_ifindex = map->map_ifindex;
4187         create_attr.map_type = def->type;
4188         create_attr.map_flags = def->map_flags;
4189         create_attr.key_size = def->key_size;
4190         create_attr.value_size = def->value_size;
4191         create_attr.numa_node = map->numa_node;
4192
4193         if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) {
4194                 int nr_cpus;
4195
4196                 nr_cpus = libbpf_num_possible_cpus();
4197                 if (nr_cpus < 0) {
4198                         pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
4199                                 map->name, nr_cpus);
4200                         return nr_cpus;
4201                 }
4202                 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
4203                 create_attr.max_entries = nr_cpus;
4204         } else {
4205                 create_attr.max_entries = def->max_entries;
4206         }
4207
4208         if (bpf_map__is_struct_ops(map))
4209                 create_attr.btf_vmlinux_value_type_id =
4210                         map->btf_vmlinux_value_type_id;
4211
4212         create_attr.btf_fd = 0;
4213         create_attr.btf_key_type_id = 0;
4214         create_attr.btf_value_type_id = 0;
4215         if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) {
4216                 create_attr.btf_fd = btf__fd(obj->btf);
4217                 create_attr.btf_key_type_id = map->btf_key_type_id;
4218                 create_attr.btf_value_type_id = map->btf_value_type_id;
4219         }
4220
4221         if (bpf_map_type__is_map_in_map(def->type)) {
4222                 if (map->inner_map) {
4223                         int err;
4224
4225                         err = bpf_object__create_map(obj, map->inner_map);
4226                         if (err) {
4227                                 pr_warn("map '%s': failed to create inner map: %d\n",
4228                                         map->name, err);
4229                                 return err;
4230                         }
4231                         map->inner_map_fd = bpf_map__fd(map->inner_map);
4232                 }
4233                 if (map->inner_map_fd >= 0)
4234                         create_attr.inner_map_fd = map->inner_map_fd;
4235         }
4236
4237         map->fd = bpf_create_map_xattr(&create_attr);
4238         if (map->fd < 0 && (create_attr.btf_key_type_id ||
4239                             create_attr.btf_value_type_id)) {
4240                 char *cp, errmsg[STRERR_BUFSIZE];
4241                 int err = -errno;
4242
4243                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4244                 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
4245                         map->name, cp, err);
4246                 create_attr.btf_fd = 0;
4247                 create_attr.btf_key_type_id = 0;
4248                 create_attr.btf_value_type_id = 0;
4249                 map->btf_key_type_id = 0;
4250                 map->btf_value_type_id = 0;
4251                 map->fd = bpf_create_map_xattr(&create_attr);
4252         }
4253
4254         if (map->fd < 0)
4255                 return -errno;
4256
4257         if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
4258                 bpf_map__destroy(map->inner_map);
4259                 zfree(&map->inner_map);
4260         }
4261
4262         return 0;
4263 }
4264
4265 static int init_map_slots(struct bpf_map *map)
4266 {
4267         const struct bpf_map *targ_map;
4268         unsigned int i;
4269         int fd, err;
4270
4271         for (i = 0; i < map->init_slots_sz; i++) {
4272                 if (!map->init_slots[i])
4273                         continue;
4274
4275                 targ_map = map->init_slots[i];
4276                 fd = bpf_map__fd(targ_map);
4277                 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
4278                 if (err) {
4279                         err = -errno;
4280                         pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
4281                                 map->name, i, targ_map->name,
4282                                 fd, err);
4283                         return err;
4284                 }
4285                 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
4286                          map->name, i, targ_map->name, fd);
4287         }
4288
4289         zfree(&map->init_slots);
4290         map->init_slots_sz = 0;
4291
4292         return 0;
4293 }
4294
4295 static int
4296 bpf_object__create_maps(struct bpf_object *obj)
4297 {
4298         struct bpf_map *map;
4299         char *cp, errmsg[STRERR_BUFSIZE];
4300         unsigned int i, j;
4301         int err;
4302
4303         for (i = 0; i < obj->nr_maps; i++) {
4304                 map = &obj->maps[i];
4305
4306                 if (map->pin_path) {
4307                         err = bpf_object__reuse_map(map);
4308                         if (err) {
4309                                 pr_warn("map '%s': error reusing pinned map\n",
4310                                         map->name);
4311                                 goto err_out;
4312                         }
4313                 }
4314
4315                 if (map->fd >= 0) {
4316                         pr_debug("map '%s': skipping creation (preset fd=%d)\n",
4317                                  map->name, map->fd);
4318                 } else {
4319                         err = bpf_object__create_map(obj, map);
4320                         if (err)
4321                                 goto err_out;
4322
4323                         pr_debug("map '%s': created successfully, fd=%d\n",
4324                                  map->name, map->fd);
4325
4326                         if (bpf_map__is_internal(map)) {
4327                                 err = bpf_object__populate_internal_map(obj, map);
4328                                 if (err < 0) {
4329                                         zclose(map->fd);
4330                                         goto err_out;
4331                                 }
4332                         }
4333
4334                         if (map->init_slots_sz) {
4335                                 err = init_map_slots(map);
4336                                 if (err < 0) {
4337                                         zclose(map->fd);
4338                                         goto err_out;
4339                                 }
4340                         }
4341                 }
4342
4343                 if (map->pin_path && !map->pinned) {
4344                         err = bpf_map__pin(map, NULL);
4345                         if (err) {
4346                                 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
4347                                         map->name, map->pin_path, err);
4348                                 zclose(map->fd);
4349                                 goto err_out;
4350                         }
4351                 }
4352         }
4353
4354         return 0;
4355
4356 err_out:
4357         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4358         pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
4359         pr_perm_msg(err);
4360         for (j = 0; j < i; j++)
4361                 zclose(obj->maps[j].fd);
4362         return err;
4363 }
4364
4365 #define BPF_CORE_SPEC_MAX_LEN 64
4366
4367 /* represents BPF CO-RE field or array element accessor */
4368 struct bpf_core_accessor {
4369         __u32 type_id;          /* struct/union type or array element type */
4370         __u32 idx;              /* field index or array index */
4371         const char *name;       /* field name or NULL for array accessor */
4372 };
4373
4374 struct bpf_core_spec {
4375         const struct btf *btf;
4376         /* high-level spec: named fields and array indices only */
4377         struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
4378         /* original unresolved (no skip_mods_or_typedefs) root type ID */
4379         __u32 root_type_id;
4380         /* CO-RE relocation kind */
4381         enum bpf_core_relo_kind relo_kind;
4382         /* high-level spec length */
4383         int len;
4384         /* raw, low-level spec: 1-to-1 with accessor spec string */
4385         int raw_spec[BPF_CORE_SPEC_MAX_LEN];
4386         /* raw spec length */
4387         int raw_len;
4388         /* field bit offset represented by spec */
4389         __u32 bit_offset;
4390 };
4391
4392 static bool str_is_empty(const char *s)
4393 {
4394         return !s || !s[0];
4395 }
4396
4397 static bool is_flex_arr(const struct btf *btf,
4398                         const struct bpf_core_accessor *acc,
4399                         const struct btf_array *arr)
4400 {
4401         const struct btf_type *t;
4402
4403         /* not a flexible array, if not inside a struct or has non-zero size */
4404         if (!acc->name || arr->nelems > 0)
4405                 return false;
4406
4407         /* has to be the last member of enclosing struct */
4408         t = btf__type_by_id(btf, acc->type_id);
4409         return acc->idx == btf_vlen(t) - 1;
4410 }
4411
4412 static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
4413 {
4414         switch (kind) {
4415         case BPF_FIELD_BYTE_OFFSET: return "byte_off";
4416         case BPF_FIELD_BYTE_SIZE: return "byte_sz";
4417         case BPF_FIELD_EXISTS: return "field_exists";
4418         case BPF_FIELD_SIGNED: return "signed";
4419         case BPF_FIELD_LSHIFT_U64: return "lshift_u64";
4420         case BPF_FIELD_RSHIFT_U64: return "rshift_u64";
4421         case BPF_TYPE_ID_LOCAL: return "local_type_id";
4422         case BPF_TYPE_ID_TARGET: return "target_type_id";
4423         case BPF_TYPE_EXISTS: return "type_exists";
4424         case BPF_TYPE_SIZE: return "type_size";
4425         case BPF_ENUMVAL_EXISTS: return "enumval_exists";
4426         case BPF_ENUMVAL_VALUE: return "enumval_value";
4427         default: return "unknown";
4428         }
4429 }
4430
4431 static bool core_relo_is_field_based(enum bpf_core_relo_kind kind)
4432 {
4433         switch (kind) {
4434         case BPF_FIELD_BYTE_OFFSET:
4435         case BPF_FIELD_BYTE_SIZE:
4436         case BPF_FIELD_EXISTS:
4437         case BPF_FIELD_SIGNED:
4438         case BPF_FIELD_LSHIFT_U64:
4439         case BPF_FIELD_RSHIFT_U64:
4440                 return true;
4441         default:
4442                 return false;
4443         }
4444 }
4445
4446 static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
4447 {
4448         switch (kind) {
4449         case BPF_TYPE_ID_LOCAL:
4450         case BPF_TYPE_ID_TARGET:
4451         case BPF_TYPE_EXISTS:
4452         case BPF_TYPE_SIZE:
4453                 return true;
4454         default:
4455                 return false;
4456         }
4457 }
4458
4459 static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
4460 {
4461         switch (kind) {
4462         case BPF_ENUMVAL_EXISTS:
4463         case BPF_ENUMVAL_VALUE:
4464                 return true;
4465         default:
4466                 return false;
4467         }
4468 }
4469
4470 /*
4471  * Turn bpf_core_relo into a low- and high-level spec representation,
4472  * validating correctness along the way, as well as calculating resulting
4473  * field bit offset, specified by accessor string. Low-level spec captures
4474  * every single level of nestedness, including traversing anonymous
4475  * struct/union members. High-level one only captures semantically meaningful
4476  * "turning points": named fields and array indicies.
4477  * E.g., for this case:
4478  *
4479  *   struct sample {
4480  *       int __unimportant;
4481  *       struct {
4482  *           int __1;
4483  *           int __2;
4484  *           int a[7];
4485  *       };
4486  *   };
4487  *
4488  *   struct sample *s = ...;
4489  *
4490  *   int x = &s->a[3]; // access string = '0:1:2:3'
4491  *
4492  * Low-level spec has 1:1 mapping with each element of access string (it's
4493  * just a parsed access string representation): [0, 1, 2, 3].
4494  *
4495  * High-level spec will capture only 3 points:
4496  *   - intial zero-index access by pointer (&s->... is the same as &s[0]...);
4497  *   - field 'a' access (corresponds to '2' in low-level spec);
4498  *   - array element #3 access (corresponds to '3' in low-level spec).
4499  *
4500  * Type-based relocations (TYPE_EXISTS/TYPE_SIZE,
4501  * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their
4502  * spec and raw_spec are kept empty.
4503  *
4504  * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access
4505  * string to specify enumerator's value index that need to be relocated.
4506  */
4507 static int bpf_core_parse_spec(const struct btf *btf,
4508                                __u32 type_id,
4509                                const char *spec_str,
4510                                enum bpf_core_relo_kind relo_kind,
4511                                struct bpf_core_spec *spec)
4512 {
4513         int access_idx, parsed_len, i;
4514         struct bpf_core_accessor *acc;
4515         const struct btf_type *t;
4516         const char *name;
4517         __u32 id;
4518         __s64 sz;
4519
4520         if (str_is_empty(spec_str) || *spec_str == ':')
4521                 return -EINVAL;
4522
4523         memset(spec, 0, sizeof(*spec));
4524         spec->btf = btf;
4525         spec->root_type_id = type_id;
4526         spec->relo_kind = relo_kind;
4527
4528         /* type-based relocations don't have a field access string */
4529         if (core_relo_is_type_based(relo_kind)) {
4530                 if (strcmp(spec_str, "0"))
4531                         return -EINVAL;
4532                 return 0;
4533         }
4534
4535         /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
4536         while (*spec_str) {
4537                 if (*spec_str == ':')
4538                         ++spec_str;
4539                 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
4540                         return -EINVAL;
4541                 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4542                         return -E2BIG;
4543                 spec_str += parsed_len;
4544                 spec->raw_spec[spec->raw_len++] = access_idx;
4545         }
4546
4547         if (spec->raw_len == 0)
4548                 return -EINVAL;
4549
4550         t = skip_mods_and_typedefs(btf, type_id, &id);
4551         if (!t)
4552                 return -EINVAL;
4553
4554         access_idx = spec->raw_spec[0];
4555         acc = &spec->spec[0];
4556         acc->type_id = id;
4557         acc->idx = access_idx;
4558         spec->len++;
4559
4560         if (core_relo_is_enumval_based(relo_kind)) {
4561                 if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
4562                         return -EINVAL;
4563
4564                 /* record enumerator name in a first accessor */
4565                 acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off);
4566                 return 0;
4567         }
4568
4569         if (!core_relo_is_field_based(relo_kind))
4570                 return -EINVAL;
4571
4572         sz = btf__resolve_size(btf, id);
4573         if (sz < 0)
4574                 return sz;
4575         spec->bit_offset = access_idx * sz * 8;
4576
4577         for (i = 1; i < spec->raw_len; i++) {
4578                 t = skip_mods_and_typedefs(btf, id, &id);
4579                 if (!t)
4580                         return -EINVAL;
4581
4582                 access_idx = spec->raw_spec[i];
4583                 acc = &spec->spec[spec->len];
4584
4585                 if (btf_is_composite(t)) {
4586                         const struct btf_member *m;
4587                         __u32 bit_offset;
4588
4589                         if (access_idx >= btf_vlen(t))
4590                                 return -EINVAL;
4591
4592                         bit_offset = btf_member_bit_offset(t, access_idx);
4593                         spec->bit_offset += bit_offset;
4594
4595                         m = btf_members(t) + access_idx;
4596                         if (m->name_off) {
4597                                 name = btf__name_by_offset(btf, m->name_off);
4598                                 if (str_is_empty(name))
4599                                         return -EINVAL;
4600
4601                                 acc->type_id = id;
4602                                 acc->idx = access_idx;
4603                                 acc->name = name;
4604                                 spec->len++;
4605                         }
4606
4607                         id = m->type;
4608                 } else if (btf_is_array(t)) {
4609                         const struct btf_array *a = btf_array(t);
4610                         bool flex;
4611
4612                         t = skip_mods_and_typedefs(btf, a->type, &id);
4613                         if (!t)
4614                                 return -EINVAL;
4615
4616                         flex = is_flex_arr(btf, acc - 1, a);
4617                         if (!flex && access_idx >= a->nelems)
4618                                 return -EINVAL;
4619
4620                         spec->spec[spec->len].type_id = id;
4621                         spec->spec[spec->len].idx = access_idx;
4622                         spec->len++;
4623
4624                         sz = btf__resolve_size(btf, id);
4625                         if (sz < 0)
4626                                 return sz;
4627                         spec->bit_offset += access_idx * sz * 8;
4628                 } else {
4629                         pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
4630                                 type_id, spec_str, i, id, btf_kind_str(t));
4631                         return -EINVAL;
4632                 }
4633         }
4634
4635         return 0;
4636 }
4637
4638 static bool bpf_core_is_flavor_sep(const char *s)
4639 {
4640         /* check X___Y name pattern, where X and Y are not underscores */
4641         return s[0] != '_' &&                                 /* X */
4642                s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */
4643                s[4] != '_';                                   /* Y */
4644 }
4645
4646 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
4647  * before last triple underscore. Struct name part after last triple
4648  * underscore is ignored by BPF CO-RE relocation during relocation matching.
4649  */
4650 static size_t bpf_core_essential_name_len(const char *name)
4651 {
4652         size_t n = strlen(name);
4653         int i;
4654
4655         for (i = n - 5; i >= 0; i--) {
4656                 if (bpf_core_is_flavor_sep(name + i))
4657                         return i + 1;
4658         }
4659         return n;
4660 }
4661
4662 struct core_cand
4663 {
4664         const struct btf *btf;
4665         const struct btf_type *t;
4666         const char *name;
4667         __u32 id;
4668 };
4669
4670 /* dynamically sized list of type IDs and its associated struct btf */
4671 struct core_cand_list {
4672         struct core_cand *cands;
4673         int len;
4674 };
4675
4676 static void bpf_core_free_cands(struct core_cand_list *cands)
4677 {
4678         free(cands->cands);
4679         free(cands);
4680 }
4681
4682 static int bpf_core_add_cands(struct core_cand *local_cand,
4683                               size_t local_essent_len,
4684                               const struct btf *targ_btf,
4685                               const char *targ_btf_name,
4686                               int targ_start_id,
4687                               struct core_cand_list *cands)
4688 {
4689         struct core_cand *new_cands, *cand;
4690         const struct btf_type *t;
4691         const char *targ_name;
4692         size_t targ_essent_len;
4693         int n, i;
4694
4695         n = btf__get_nr_types(targ_btf);
4696         for (i = targ_start_id; i <= n; i++) {
4697                 t = btf__type_by_id(targ_btf, i);
4698                 if (btf_kind(t) != btf_kind(local_cand->t))
4699                         continue;
4700
4701                 targ_name = btf__name_by_offset(targ_btf, t->name_off);
4702                 if (str_is_empty(targ_name))
4703                         continue;
4704
4705                 targ_essent_len = bpf_core_essential_name_len(targ_name);
4706                 if (targ_essent_len != local_essent_len)
4707                         continue;
4708
4709                 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
4710                         continue;
4711
4712                 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
4713                          local_cand->id, btf_kind_str(local_cand->t),
4714                          local_cand->name, i, btf_kind_str(t), targ_name,
4715                          targ_btf_name);
4716                 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
4717                                               sizeof(*cands->cands));
4718                 if (!new_cands)
4719                         return -ENOMEM;
4720
4721                 cand = &new_cands[cands->len];
4722                 cand->btf = targ_btf;
4723                 cand->t = t;
4724                 cand->name = targ_name;
4725                 cand->id = i;
4726
4727                 cands->cands = new_cands;
4728                 cands->len++;
4729         }
4730         return 0;
4731 }
4732
4733 static int load_module_btfs(struct bpf_object *obj)
4734 {
4735         struct bpf_btf_info info;
4736         struct module_btf *mod_btf;
4737         struct btf *btf;
4738         char name[64];
4739         __u32 id = 0, len;
4740         int err, fd;
4741
4742         if (obj->btf_modules_loaded)
4743                 return 0;
4744
4745         /* don't do this again, even if we find no module BTFs */
4746         obj->btf_modules_loaded = true;
4747
4748         /* kernel too old to support module BTFs */
4749         if (!kernel_supports(FEAT_MODULE_BTF))
4750                 return 0;
4751
4752         while (true) {
4753                 err = bpf_btf_get_next_id(id, &id);
4754                 if (err && errno == ENOENT)
4755                         return 0;
4756                 if (err) {
4757                         err = -errno;
4758                         pr_warn("failed to iterate BTF objects: %d\n", err);
4759                         return err;
4760                 }
4761
4762                 fd = bpf_btf_get_fd_by_id(id);
4763                 if (fd < 0) {
4764                         if (errno == ENOENT)
4765                                 continue; /* expected race: BTF was unloaded */
4766                         err = -errno;
4767                         pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
4768                         return err;
4769                 }
4770
4771                 len = sizeof(info);
4772                 memset(&info, 0, sizeof(info));
4773                 info.name = ptr_to_u64(name);
4774                 info.name_len = sizeof(name);
4775
4776                 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4777                 if (err) {
4778                         err = -errno;
4779                         pr_warn("failed to get BTF object #%d info: %d\n", id, err);
4780                         goto err_out;
4781                 }
4782
4783                 /* ignore non-module BTFs */
4784                 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
4785                         close(fd);
4786                         continue;
4787                 }
4788
4789                 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
4790                 if (IS_ERR(btf)) {
4791                         pr_warn("failed to load module [%s]'s BTF object #%d: %ld\n",
4792                                 name, id, PTR_ERR(btf));
4793                         err = PTR_ERR(btf);
4794                         goto err_out;
4795                 }
4796
4797                 err = btf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
4798                                      sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
4799                 if (err)
4800                         goto err_out;
4801
4802                 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
4803
4804                 mod_btf->btf = btf;
4805                 mod_btf->id = id;
4806                 mod_btf->fd = fd;
4807                 mod_btf->name = strdup(name);
4808                 if (!mod_btf->name) {
4809                         err = -ENOMEM;
4810                         goto err_out;
4811                 }
4812                 continue;
4813
4814 err_out:
4815                 close(fd);
4816                 return err;
4817         }
4818
4819         return 0;
4820 }
4821
4822 static struct core_cand_list *
4823 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
4824 {
4825         struct core_cand local_cand = {};
4826         struct core_cand_list *cands;
4827         const struct btf *main_btf;
4828         size_t local_essent_len;
4829         int err, i;
4830
4831         local_cand.btf = local_btf;
4832         local_cand.t = btf__type_by_id(local_btf, local_type_id);
4833         if (!local_cand.t)
4834                 return ERR_PTR(-EINVAL);
4835
4836         local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
4837         if (str_is_empty(local_cand.name))
4838                 return ERR_PTR(-EINVAL);
4839         local_essent_len = bpf_core_essential_name_len(local_cand.name);
4840
4841         cands = calloc(1, sizeof(*cands));
4842         if (!cands)
4843                 return ERR_PTR(-ENOMEM);
4844
4845         /* Attempt to find target candidates in vmlinux BTF first */
4846         main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
4847         err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
4848         if (err)
4849                 goto err_out;
4850
4851         /* if vmlinux BTF has any candidate, don't got for module BTFs */
4852         if (cands->len)
4853                 return cands;
4854
4855         /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
4856         if (obj->btf_vmlinux_override)
4857                 return cands;
4858
4859         /* now look through module BTFs, trying to still find candidates */
4860         err = load_module_btfs(obj);
4861         if (err)
4862                 goto err_out;
4863
4864         for (i = 0; i < obj->btf_module_cnt; i++) {
4865                 err = bpf_core_add_cands(&local_cand, local_essent_len,
4866                                          obj->btf_modules[i].btf,
4867                                          obj->btf_modules[i].name,
4868                                          btf__get_nr_types(obj->btf_vmlinux) + 1,
4869                                          cands);
4870                 if (err)
4871                         goto err_out;
4872         }
4873
4874         return cands;
4875 err_out:
4876         bpf_core_free_cands(cands);
4877         return ERR_PTR(err);
4878 }
4879
4880 /* Check two types for compatibility for the purpose of field access
4881  * relocation. const/volatile/restrict and typedefs are skipped to ensure we
4882  * are relocating semantically compatible entities:
4883  *   - any two STRUCTs/UNIONs are compatible and can be mixed;
4884  *   - any two FWDs are compatible, if their names match (modulo flavor suffix);
4885  *   - any two PTRs are always compatible;
4886  *   - for ENUMs, names should be the same (ignoring flavor suffix) or at
4887  *     least one of enums should be anonymous;
4888  *   - for ENUMs, check sizes, names are ignored;
4889  *   - for INT, size and signedness are ignored;
4890  *   - for ARRAY, dimensionality is ignored, element types are checked for
4891  *     compatibility recursively;
4892  *   - everything else shouldn't be ever a target of relocation.
4893  * These rules are not set in stone and probably will be adjusted as we get
4894  * more experience with using BPF CO-RE relocations.
4895  */
4896 static int bpf_core_fields_are_compat(const struct btf *local_btf,
4897                                       __u32 local_id,
4898                                       const struct btf *targ_btf,
4899                                       __u32 targ_id)
4900 {
4901         const struct btf_type *local_type, *targ_type;
4902
4903 recur:
4904         local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
4905         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4906         if (!local_type || !targ_type)
4907                 return -EINVAL;
4908
4909         if (btf_is_composite(local_type) && btf_is_composite(targ_type))
4910                 return 1;
4911         if (btf_kind(local_type) != btf_kind(targ_type))
4912                 return 0;
4913
4914         switch (btf_kind(local_type)) {
4915         case BTF_KIND_PTR:
4916                 return 1;
4917         case BTF_KIND_FWD:
4918         case BTF_KIND_ENUM: {
4919                 const char *local_name, *targ_name;
4920                 size_t local_len, targ_len;
4921
4922                 local_name = btf__name_by_offset(local_btf,
4923                                                  local_type->name_off);
4924                 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
4925                 local_len = bpf_core_essential_name_len(local_name);
4926                 targ_len = bpf_core_essential_name_len(targ_name);
4927                 /* one of them is anonymous or both w/ same flavor-less names */
4928                 return local_len == 0 || targ_len == 0 ||
4929                        (local_len == targ_len &&
4930                         strncmp(local_name, targ_name, local_len) == 0);
4931         }
4932         case BTF_KIND_INT:
4933                 /* just reject deprecated bitfield-like integers; all other
4934                  * integers are by default compatible between each other
4935                  */
4936                 return btf_int_offset(local_type) == 0 &&
4937                        btf_int_offset(targ_type) == 0;
4938         case BTF_KIND_ARRAY:
4939                 local_id = btf_array(local_type)->type;
4940                 targ_id = btf_array(targ_type)->type;
4941                 goto recur;
4942         default:
4943                 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
4944                         btf_kind(local_type), local_id, targ_id);
4945                 return 0;
4946         }
4947 }
4948
4949 /*
4950  * Given single high-level named field accessor in local type, find
4951  * corresponding high-level accessor for a target type. Along the way,
4952  * maintain low-level spec for target as well. Also keep updating target
4953  * bit offset.
4954  *
4955  * Searching is performed through recursive exhaustive enumeration of all
4956  * fields of a struct/union. If there are any anonymous (embedded)
4957  * structs/unions, they are recursively searched as well. If field with
4958  * desired name is found, check compatibility between local and target types,
4959  * before returning result.
4960  *
4961  * 1 is returned, if field is found.
4962  * 0 is returned if no compatible field is found.
4963  * <0 is returned on error.
4964  */
4965 static int bpf_core_match_member(const struct btf *local_btf,
4966                                  const struct bpf_core_accessor *local_acc,
4967                                  const struct btf *targ_btf,
4968                                  __u32 targ_id,
4969                                  struct bpf_core_spec *spec,
4970                                  __u32 *next_targ_id)
4971 {
4972         const struct btf_type *local_type, *targ_type;
4973         const struct btf_member *local_member, *m;
4974         const char *local_name, *targ_name;
4975         __u32 local_id;
4976         int i, n, found;
4977
4978         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4979         if (!targ_type)
4980                 return -EINVAL;
4981         if (!btf_is_composite(targ_type))
4982                 return 0;
4983
4984         local_id = local_acc->type_id;
4985         local_type = btf__type_by_id(local_btf, local_id);
4986         local_member = btf_members(local_type) + local_acc->idx;
4987         local_name = btf__name_by_offset(local_btf, local_member->name_off);
4988
4989         n = btf_vlen(targ_type);
4990         m = btf_members(targ_type);
4991         for (i = 0; i < n; i++, m++) {
4992                 __u32 bit_offset;
4993
4994                 bit_offset = btf_member_bit_offset(targ_type, i);
4995
4996                 /* too deep struct/union/array nesting */
4997                 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4998                         return -E2BIG;
4999
5000                 /* speculate this member will be the good one */
5001                 spec->bit_offset += bit_offset;
5002                 spec->raw_spec[spec->raw_len++] = i;
5003
5004                 targ_name = btf__name_by_offset(targ_btf, m->name_off);
5005                 if (str_is_empty(targ_name)) {
5006                         /* embedded struct/union, we need to go deeper */
5007                         found = bpf_core_match_member(local_btf, local_acc,
5008                                                       targ_btf, m->type,
5009                                                       spec, next_targ_id);
5010                         if (found) /* either found or error */
5011                                 return found;
5012                 } else if (strcmp(local_name, targ_name) == 0) {
5013                         /* matching named field */
5014                         struct bpf_core_accessor *targ_acc;
5015
5016                         targ_acc = &spec->spec[spec->len++];
5017                         targ_acc->type_id = targ_id;
5018                         targ_acc->idx = i;
5019                         targ_acc->name = targ_name;
5020
5021                         *next_targ_id = m->type;
5022                         found = bpf_core_fields_are_compat(local_btf,
5023                                                            local_member->type,
5024                                                            targ_btf, m->type);
5025                         if (!found)
5026                                 spec->len--; /* pop accessor */
5027                         return found;
5028                 }
5029                 /* member turned out not to be what we looked for */
5030                 spec->bit_offset -= bit_offset;
5031                 spec->raw_len--;
5032         }
5033
5034         return 0;
5035 }
5036
5037 /* Check local and target types for compatibility. This check is used for
5038  * type-based CO-RE relocations and follow slightly different rules than
5039  * field-based relocations. This function assumes that root types were already
5040  * checked for name match. Beyond that initial root-level name check, names
5041  * are completely ignored. Compatibility rules are as follows:
5042  *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5043  *     kind should match for local and target types (i.e., STRUCT is not
5044  *     compatible with UNION);
5045  *   - for ENUMs, the size is ignored;
5046  *   - for INT, size and signedness are ignored;
5047  *   - for ARRAY, dimensionality is ignored, element types are checked for
5048  *     compatibility recursively;
5049  *   - CONST/VOLATILE/RESTRICT modifiers are ignored;
5050  *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5051  *   - FUNC_PROTOs are compatible if they have compatible signature: same
5052  *     number of input args and compatible return and argument types.
5053  * These rules are not set in stone and probably will be adjusted as we get
5054  * more experience with using BPF CO-RE relocations.
5055  */
5056 static int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5057                                      const struct btf *targ_btf, __u32 targ_id)
5058 {
5059         const struct btf_type *local_type, *targ_type;
5060         int depth = 32; /* max recursion depth */
5061
5062         /* caller made sure that names match (ignoring flavor suffix) */
5063         local_type = btf__type_by_id(local_btf, local_id);
5064         targ_type = btf__type_by_id(targ_btf, targ_id);
5065         if (btf_kind(local_type) != btf_kind(targ_type))
5066                 return 0;
5067
5068 recur:
5069         depth--;
5070         if (depth < 0)
5071                 return -EINVAL;
5072
5073         local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
5074         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
5075         if (!local_type || !targ_type)
5076                 return -EINVAL;
5077
5078         if (btf_kind(local_type) != btf_kind(targ_type))
5079                 return 0;
5080
5081         switch (btf_kind(local_type)) {
5082         case BTF_KIND_UNKN:
5083         case BTF_KIND_STRUCT:
5084         case BTF_KIND_UNION:
5085         case BTF_KIND_ENUM:
5086         case BTF_KIND_FWD:
5087                 return 1;
5088         case BTF_KIND_INT:
5089                 /* just reject deprecated bitfield-like integers; all other
5090                  * integers are by default compatible between each other
5091                  */
5092                 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
5093         case BTF_KIND_PTR:
5094                 local_id = local_type->type;
5095                 targ_id = targ_type->type;
5096                 goto recur;
5097         case BTF_KIND_ARRAY:
5098                 local_id = btf_array(local_type)->type;
5099                 targ_id = btf_array(targ_type)->type;
5100                 goto recur;
5101         case BTF_KIND_FUNC_PROTO: {
5102                 struct btf_param *local_p = btf_params(local_type);
5103                 struct btf_param *targ_p = btf_params(targ_type);
5104                 __u16 local_vlen = btf_vlen(local_type);
5105                 __u16 targ_vlen = btf_vlen(targ_type);
5106                 int i, err;
5107
5108                 if (local_vlen != targ_vlen)
5109                         return 0;
5110
5111                 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
5112                         skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
5113                         skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
5114                         err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id);
5115                         if (err <= 0)
5116                                 return err;
5117                 }
5118
5119                 /* tail recurse for return type check */
5120                 skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
5121                 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
5122                 goto recur;
5123         }
5124         default:
5125                 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
5126                         btf_kind_str(local_type), local_id, targ_id);
5127                 return 0;
5128         }
5129 }
5130
5131 /*
5132  * Try to match local spec to a target type and, if successful, produce full
5133  * target spec (high-level, low-level + bit offset).
5134  */
5135 static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
5136                                const struct btf *targ_btf, __u32 targ_id,
5137                                struct bpf_core_spec *targ_spec)
5138 {
5139         const struct btf_type *targ_type;
5140         const struct bpf_core_accessor *local_acc;
5141         struct bpf_core_accessor *targ_acc;
5142         int i, sz, matched;
5143
5144         memset(targ_spec, 0, sizeof(*targ_spec));
5145         targ_spec->btf = targ_btf;
5146         targ_spec->root_type_id = targ_id;
5147         targ_spec->relo_kind = local_spec->relo_kind;
5148
5149         if (core_relo_is_type_based(local_spec->relo_kind)) {
5150                 return bpf_core_types_are_compat(local_spec->btf,
5151                                                  local_spec->root_type_id,
5152                                                  targ_btf, targ_id);
5153         }
5154
5155         local_acc = &local_spec->spec[0];
5156         targ_acc = &targ_spec->spec[0];
5157
5158         if (core_relo_is_enumval_based(local_spec->relo_kind)) {
5159                 size_t local_essent_len, targ_essent_len;
5160                 const struct btf_enum *e;
5161                 const char *targ_name;
5162
5163                 /* has to resolve to an enum */
5164                 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
5165                 if (!btf_is_enum(targ_type))
5166                         return 0;
5167
5168                 local_essent_len = bpf_core_essential_name_len(local_acc->name);
5169
5170                 for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) {
5171                         targ_name = btf__name_by_offset(targ_spec->btf, e->name_off);
5172                         targ_essent_len = bpf_core_essential_name_len(targ_name);
5173                         if (targ_essent_len != local_essent_len)
5174                                 continue;
5175                         if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {
5176                                 targ_acc->type_id = targ_id;
5177                                 targ_acc->idx = i;
5178                                 targ_acc->name = targ_name;
5179                                 targ_spec->len++;
5180                                 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5181                                 targ_spec->raw_len++;
5182                                 return 1;
5183                         }
5184                 }
5185                 return 0;
5186         }
5187
5188         if (!core_relo_is_field_based(local_spec->relo_kind))
5189                 return -EINVAL;
5190
5191         for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
5192                 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
5193                                                    &targ_id);
5194                 if (!targ_type)
5195                         return -EINVAL;
5196
5197                 if (local_acc->name) {
5198                         matched = bpf_core_match_member(local_spec->btf,
5199                                                         local_acc,
5200                                                         targ_btf, targ_id,
5201                                                         targ_spec, &targ_id);
5202                         if (matched <= 0)
5203                                 return matched;
5204                 } else {
5205                         /* for i=0, targ_id is already treated as array element
5206                          * type (because it's the original struct), for others
5207                          * we should find array element type first
5208                          */
5209                         if (i > 0) {
5210                                 const struct btf_array *a;
5211                                 bool flex;
5212
5213                                 if (!btf_is_array(targ_type))
5214                                         return 0;
5215
5216                                 a = btf_array(targ_type);
5217                                 flex = is_flex_arr(targ_btf, targ_acc - 1, a);
5218                                 if (!flex && local_acc->idx >= a->nelems)
5219                                         return 0;
5220                                 if (!skip_mods_and_typedefs(targ_btf, a->type,
5221                                                             &targ_id))
5222                                         return -EINVAL;
5223                         }
5224
5225                         /* too deep struct/union/array nesting */
5226                         if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
5227                                 return -E2BIG;
5228
5229                         targ_acc->type_id = targ_id;
5230                         targ_acc->idx = local_acc->idx;
5231                         targ_acc->name = NULL;
5232                         targ_spec->len++;
5233                         targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5234                         targ_spec->raw_len++;
5235
5236                         sz = btf__resolve_size(targ_btf, targ_id);
5237                         if (sz < 0)
5238                                 return sz;
5239                         targ_spec->bit_offset += local_acc->idx * sz * 8;
5240                 }
5241         }
5242
5243         return 1;
5244 }
5245
5246 static int bpf_core_calc_field_relo(const struct bpf_program *prog,
5247                                     const struct bpf_core_relo *relo,
5248                                     const struct bpf_core_spec *spec,
5249                                     __u32 *val, __u32 *field_sz, __u32 *type_id,
5250                                     bool *validate)
5251 {
5252         const struct bpf_core_accessor *acc;
5253         const struct btf_type *t;
5254         __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
5255         const struct btf_member *m;
5256         const struct btf_type *mt;
5257         bool bitfield;
5258         __s64 sz;
5259
5260         *field_sz = 0;
5261
5262         if (relo->kind == BPF_FIELD_EXISTS) {
5263                 *val = spec ? 1 : 0;
5264                 return 0;
5265         }
5266
5267         if (!spec)
5268                 return -EUCLEAN; /* request instruction poisoning */
5269
5270         acc = &spec->spec[spec->len - 1];
5271         t = btf__type_by_id(spec->btf, acc->type_id);
5272
5273         /* a[n] accessor needs special handling */
5274         if (!acc->name) {
5275                 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
5276                         *val = spec->bit_offset / 8;
5277                         /* remember field size for load/store mem size */
5278                         sz = btf__resolve_size(spec->btf, acc->type_id);
5279                         if (sz < 0)
5280                                 return -EINVAL;
5281                         *field_sz = sz;
5282                         *type_id = acc->type_id;
5283                 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
5284                         sz = btf__resolve_size(spec->btf, acc->type_id);
5285                         if (sz < 0)
5286                                 return -EINVAL;
5287                         *val = sz;
5288                 } else {
5289                         pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
5290                                 prog->name, relo->kind, relo->insn_off / 8);
5291                         return -EINVAL;
5292                 }
5293                 if (validate)
5294                         *validate = true;
5295                 return 0;
5296         }
5297
5298         m = btf_members(t) + acc->idx;
5299         mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
5300         bit_off = spec->bit_offset;
5301         bit_sz = btf_member_bitfield_size(t, acc->idx);
5302
5303         bitfield = bit_sz > 0;
5304         if (bitfield) {
5305                 byte_sz = mt->size;
5306                 byte_off = bit_off / 8 / byte_sz * byte_sz;
5307                 /* figure out smallest int size necessary for bitfield load */
5308                 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
5309                         if (byte_sz >= 8) {
5310                                 /* bitfield can't be read with 64-bit read */
5311                                 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
5312                                         prog->name, relo->kind, relo->insn_off / 8);
5313                                 return -E2BIG;
5314                         }
5315                         byte_sz *= 2;
5316                         byte_off = bit_off / 8 / byte_sz * byte_sz;
5317                 }
5318         } else {
5319                 sz = btf__resolve_size(spec->btf, field_type_id);
5320                 if (sz < 0)
5321                         return -EINVAL;
5322                 byte_sz = sz;
5323                 byte_off = spec->bit_offset / 8;
5324                 bit_sz = byte_sz * 8;
5325         }
5326
5327         /* for bitfields, all the relocatable aspects are ambiguous and we
5328          * might disagree with compiler, so turn off validation of expected
5329          * value, except for signedness
5330          */
5331         if (validate)
5332                 *validate = !bitfield;
5333
5334         switch (relo->kind) {
5335         case BPF_FIELD_BYTE_OFFSET:
5336                 *val = byte_off;
5337                 if (!bitfield) {
5338                         *field_sz = byte_sz;
5339                         *type_id = field_type_id;
5340                 }
5341                 break;
5342         case BPF_FIELD_BYTE_SIZE:
5343                 *val = byte_sz;
5344                 break;
5345         case BPF_FIELD_SIGNED:
5346                 /* enums will be assumed unsigned */
5347                 *val = btf_is_enum(mt) ||
5348                        (btf_int_encoding(mt) & BTF_INT_SIGNED);
5349                 if (validate)
5350                         *validate = true; /* signedness is never ambiguous */
5351                 break;
5352         case BPF_FIELD_LSHIFT_U64:
5353 #if __BYTE_ORDER == __LITTLE_ENDIAN
5354                 *val = 64 - (bit_off + bit_sz - byte_off  * 8);
5355 #else
5356                 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
5357 #endif
5358                 break;
5359         case BPF_FIELD_RSHIFT_U64:
5360                 *val = 64 - bit_sz;
5361                 if (validate)
5362                         *validate = true; /* right shift is never ambiguous */
5363                 break;
5364         case BPF_FIELD_EXISTS:
5365         default:
5366                 return -EOPNOTSUPP;
5367         }
5368
5369         return 0;
5370 }
5371
5372 static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
5373                                    const struct bpf_core_spec *spec,
5374                                    __u32 *val)
5375 {
5376         __s64 sz;
5377
5378         /* type-based relos return zero when target type is not found */
5379         if (!spec) {
5380                 *val = 0;
5381                 return 0;
5382         }
5383
5384         switch (relo->kind) {
5385         case BPF_TYPE_ID_TARGET:
5386                 *val = spec->root_type_id;
5387                 break;
5388         case BPF_TYPE_EXISTS:
5389                 *val = 1;
5390                 break;
5391         case BPF_TYPE_SIZE:
5392                 sz = btf__resolve_size(spec->btf, spec->root_type_id);
5393                 if (sz < 0)
5394                         return -EINVAL;
5395                 *val = sz;
5396                 break;
5397         case BPF_TYPE_ID_LOCAL:
5398         /* BPF_TYPE_ID_LOCAL is handled specially and shouldn't get here */
5399         default:
5400                 return -EOPNOTSUPP;
5401         }
5402
5403         return 0;
5404 }
5405
5406 static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
5407                                       const struct bpf_core_spec *spec,
5408                                       __u32 *val)
5409 {
5410         const struct btf_type *t;
5411         const struct btf_enum *e;
5412
5413         switch (relo->kind) {
5414         case BPF_ENUMVAL_EXISTS:
5415                 *val = spec ? 1 : 0;
5416                 break;
5417         case BPF_ENUMVAL_VALUE:
5418                 if (!spec)
5419                         return -EUCLEAN; /* request instruction poisoning */
5420                 t = btf__type_by_id(spec->btf, spec->spec[0].type_id);
5421                 e = btf_enum(t) + spec->spec[0].idx;
5422                 *val = e->val;
5423                 break;
5424         default:
5425                 return -EOPNOTSUPP;
5426         }
5427
5428         return 0;
5429 }
5430
5431 struct bpf_core_relo_res
5432 {
5433         /* expected value in the instruction, unless validate == false */
5434         __u32 orig_val;
5435         /* new value that needs to be patched up to */
5436         __u32 new_val;
5437         /* relocation unsuccessful, poison instruction, but don't fail load */
5438         bool poison;
5439         /* some relocations can't be validated against orig_val */
5440         bool validate;
5441         /* for field byte offset relocations or the forms:
5442          *     *(T *)(rX + <off>) = rY
5443          *     rX = *(T *)(rY + <off>),
5444          * we remember original and resolved field size to adjust direct
5445          * memory loads of pointers and integers; this is necessary for 32-bit
5446          * host kernel architectures, but also allows to automatically
5447          * relocate fields that were resized from, e.g., u32 to u64, etc.
5448          */
5449         bool fail_memsz_adjust;
5450         __u32 orig_sz;
5451         __u32 orig_type_id;
5452         __u32 new_sz;
5453         __u32 new_type_id;
5454 };
5455
5456 /* Calculate original and target relocation values, given local and target
5457  * specs and relocation kind. These values are calculated for each candidate.
5458  * If there are multiple candidates, resulting values should all be consistent
5459  * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity.
5460  * If instruction has to be poisoned, *poison will be set to true.
5461  */
5462 static int bpf_core_calc_relo(const struct bpf_program *prog,
5463                               const struct bpf_core_relo *relo,
5464                               int relo_idx,
5465                               const struct bpf_core_spec *local_spec,
5466                               const struct bpf_core_spec *targ_spec,
5467                               struct bpf_core_relo_res *res)
5468 {
5469         int err = -EOPNOTSUPP;
5470
5471         res->orig_val = 0;
5472         res->new_val = 0;
5473         res->poison = false;
5474         res->validate = true;
5475         res->fail_memsz_adjust = false;
5476         res->orig_sz = res->new_sz = 0;
5477         res->orig_type_id = res->new_type_id = 0;
5478
5479         if (core_relo_is_field_based(relo->kind)) {
5480                 err = bpf_core_calc_field_relo(prog, relo, local_spec,
5481                                                &res->orig_val, &res->orig_sz,
5482                                                &res->orig_type_id, &res->validate);
5483                 err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec,
5484                                                       &res->new_val, &res->new_sz,
5485                                                       &res->new_type_id, NULL);
5486                 if (err)
5487                         goto done;
5488                 /* Validate if it's safe to adjust load/store memory size.
5489                  * Adjustments are performed only if original and new memory
5490                  * sizes differ.
5491                  */
5492                 res->fail_memsz_adjust = false;
5493                 if (res->orig_sz != res->new_sz) {
5494                         const struct btf_type *orig_t, *new_t;
5495
5496                         orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id);
5497                         new_t = btf__type_by_id(targ_spec->btf, res->new_type_id);
5498
5499                         /* There are two use cases in which it's safe to
5500                          * adjust load/store's mem size:
5501                          *   - reading a 32-bit kernel pointer, while on BPF
5502                          *   size pointers are always 64-bit; in this case
5503                          *   it's safe to "downsize" instruction size due to
5504                          *   pointer being treated as unsigned integer with
5505                          *   zero-extended upper 32-bits;
5506                          *   - reading unsigned integers, again due to
5507                          *   zero-extension is preserving the value correctly.
5508                          *
5509                          * In all other cases it's incorrect to attempt to
5510                          * load/store field because read value will be
5511                          * incorrect, so we poison relocated instruction.
5512                          */
5513                         if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
5514                                 goto done;
5515                         if (btf_is_int(orig_t) && btf_is_int(new_t) &&
5516                             btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
5517                             btf_int_encoding(new_t) != BTF_INT_SIGNED)
5518                                 goto done;
5519
5520                         /* mark as invalid mem size adjustment, but this will
5521                          * only be checked for LDX/STX/ST insns
5522                          */
5523                         res->fail_memsz_adjust = true;
5524                 }
5525         } else if (core_relo_is_type_based(relo->kind)) {
5526                 err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
5527                 err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
5528         } else if (core_relo_is_enumval_based(relo->kind)) {
5529                 err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
5530                 err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
5531         }
5532
5533 done:
5534         if (err == -EUCLEAN) {
5535                 /* EUCLEAN is used to signal instruction poisoning request */
5536                 res->poison = true;
5537                 err = 0;
5538         } else if (err == -EOPNOTSUPP) {
5539                 /* EOPNOTSUPP means unknown/unsupported relocation */
5540                 pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",
5541                         prog->name, relo_idx, core_relo_kind_str(relo->kind),
5542                         relo->kind, relo->insn_off / 8);
5543         }
5544
5545         return err;
5546 }
5547
5548 /*
5549  * Turn instruction for which CO_RE relocation failed into invalid one with
5550  * distinct signature.
5551  */
5552 static void bpf_core_poison_insn(struct bpf_program *prog, int relo_idx,
5553                                  int insn_idx, struct bpf_insn *insn)
5554 {
5555         pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
5556                  prog->name, relo_idx, insn_idx);
5557         insn->code = BPF_JMP | BPF_CALL;
5558         insn->dst_reg = 0;
5559         insn->src_reg = 0;
5560         insn->off = 0;
5561         /* if this instruction is reachable (not a dead code),
5562          * verifier will complain with the following message:
5563          * invalid func unknown#195896080
5564          */
5565         insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
5566 }
5567
5568 static bool is_ldimm64(struct bpf_insn *insn)
5569 {
5570         return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
5571 }
5572
5573 static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
5574 {
5575         switch (BPF_SIZE(insn->code)) {
5576         case BPF_DW: return 8;
5577         case BPF_W: return 4;
5578         case BPF_H: return 2;
5579         case BPF_B: return 1;
5580         default: return -1;
5581         }
5582 }
5583
5584 static int insn_bytes_to_bpf_size(__u32 sz)
5585 {
5586         switch (sz) {
5587         case 8: return BPF_DW;
5588         case 4: return BPF_W;
5589         case 2: return BPF_H;
5590         case 1: return BPF_B;
5591         default: return -1;
5592         }
5593 }
5594
5595 /*
5596  * Patch relocatable BPF instruction.
5597  *
5598  * Patched value is determined by relocation kind and target specification.
5599  * For existence relocations target spec will be NULL if field/type is not found.
5600  * Expected insn->imm value is determined using relocation kind and local
5601  * spec, and is checked before patching instruction. If actual insn->imm value
5602  * is wrong, bail out with error.
5603  *
5604  * Currently supported classes of BPF instruction are:
5605  * 1. rX = <imm> (assignment with immediate operand);
5606  * 2. rX += <imm> (arithmetic operations with immediate operand);
5607  * 3. rX = <imm64> (load with 64-bit immediate value);
5608  * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64};
5609  * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
5610  * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
5611  */
5612 static int bpf_core_patch_insn(struct bpf_program *prog,
5613                                const struct bpf_core_relo *relo,
5614                                int relo_idx,
5615                                const struct bpf_core_relo_res *res)
5616 {
5617         __u32 orig_val, new_val;
5618         struct bpf_insn *insn;
5619         int insn_idx;
5620         __u8 class;
5621
5622         if (relo->insn_off % BPF_INSN_SZ)
5623                 return -EINVAL;
5624         insn_idx = relo->insn_off / BPF_INSN_SZ;
5625         /* adjust insn_idx from section frame of reference to the local
5626          * program's frame of reference; (sub-)program code is not yet
5627          * relocated, so it's enough to just subtract in-section offset
5628          */
5629         insn_idx = insn_idx - prog->sec_insn_off;
5630         insn = &prog->insns[insn_idx];
5631         class = BPF_CLASS(insn->code);
5632
5633         if (res->poison) {
5634 poison:
5635                 /* poison second part of ldimm64 to avoid confusing error from
5636                  * verifier about "unknown opcode 00"
5637                  */
5638                 if (is_ldimm64(insn))
5639                         bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);
5640                 bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);
5641                 return 0;
5642         }
5643
5644         orig_val = res->orig_val;
5645         new_val = res->new_val;
5646
5647         switch (class) {
5648         case BPF_ALU:
5649         case BPF_ALU64:
5650                 if (BPF_SRC(insn->code) != BPF_K)
5651                         return -EINVAL;
5652                 if (res->validate && insn->imm != orig_val) {
5653                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
5654                                 prog->name, relo_idx,
5655                                 insn_idx, insn->imm, orig_val, new_val);
5656                         return -EINVAL;
5657                 }
5658                 orig_val = insn->imm;
5659                 insn->imm = new_val;
5660                 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
5661                          prog->name, relo_idx, insn_idx,
5662                          orig_val, new_val);
5663                 break;
5664         case BPF_LDX:
5665         case BPF_ST:
5666         case BPF_STX:
5667                 if (res->validate && insn->off != orig_val) {
5668                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n",
5669                                 prog->name, relo_idx, insn_idx, insn->off, orig_val, new_val);
5670                         return -EINVAL;
5671                 }
5672                 if (new_val > SHRT_MAX) {
5673                         pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
5674                                 prog->name, relo_idx, insn_idx, new_val);
5675                         return -ERANGE;
5676                 }
5677                 if (res->fail_memsz_adjust) {
5678                         pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
5679                                 "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
5680                                 prog->name, relo_idx, insn_idx);
5681                         goto poison;
5682                 }
5683
5684                 orig_val = insn->off;
5685                 insn->off = new_val;
5686                 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
5687                          prog->name, relo_idx, insn_idx, orig_val, new_val);
5688
5689                 if (res->new_sz != res->orig_sz) {
5690                         int insn_bytes_sz, insn_bpf_sz;
5691
5692                         insn_bytes_sz = insn_bpf_size_to_bytes(insn);
5693                         if (insn_bytes_sz != res->orig_sz) {
5694                                 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
5695                                         prog->name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
5696                                 return -EINVAL;
5697                         }
5698
5699                         insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
5700                         if (insn_bpf_sz < 0) {
5701                                 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
5702                                         prog->name, relo_idx, insn_idx, res->new_sz);
5703                                 return -EINVAL;
5704                         }
5705
5706                         insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code);
5707                         pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
5708                                  prog->name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
5709                 }
5710                 break;
5711         case BPF_LD: {
5712                 __u64 imm;
5713
5714                 if (!is_ldimm64(insn) ||
5715                     insn[0].src_reg != 0 || insn[0].off != 0 ||
5716                     insn_idx + 1 >= prog->insns_cnt ||
5717                     insn[1].code != 0 || insn[1].dst_reg != 0 ||
5718                     insn[1].src_reg != 0 || insn[1].off != 0) {
5719                         pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",
5720                                 prog->name, relo_idx, insn_idx);
5721                         return -EINVAL;
5722                 }
5723
5724                 imm = insn[0].imm + ((__u64)insn[1].imm << 32);
5725                 if (res->validate && imm != orig_val) {
5726                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n",
5727                                 prog->name, relo_idx,
5728                                 insn_idx, (unsigned long long)imm,
5729                                 orig_val, new_val);
5730                         return -EINVAL;
5731                 }
5732
5733                 insn[0].imm = new_val;
5734                 insn[1].imm = 0; /* currently only 32-bit values are supported */
5735                 pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n",
5736                          prog->name, relo_idx, insn_idx,
5737                          (unsigned long long)imm, new_val);
5738                 break;
5739         }
5740         default:
5741                 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
5742                         prog->name, relo_idx, insn_idx, insn->code,
5743                         insn->src_reg, insn->dst_reg, insn->off, insn->imm);
5744                 return -EINVAL;
5745         }
5746
5747         return 0;
5748 }
5749
5750 /* Output spec definition in the format:
5751  * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
5752  * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
5753  */
5754 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
5755 {
5756         const struct btf_type *t;
5757         const struct btf_enum *e;
5758         const char *s;
5759         __u32 type_id;
5760         int i;
5761
5762         type_id = spec->root_type_id;
5763         t = btf__type_by_id(spec->btf, type_id);
5764         s = btf__name_by_offset(spec->btf, t->name_off);
5765
5766         libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
5767
5768         if (core_relo_is_type_based(spec->relo_kind))
5769                 return;
5770
5771         if (core_relo_is_enumval_based(spec->relo_kind)) {
5772                 t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
5773                 e = btf_enum(t) + spec->raw_spec[0];
5774                 s = btf__name_by_offset(spec->btf, e->name_off);
5775
5776                 libbpf_print(level, "::%s = %u", s, e->val);
5777                 return;
5778         }
5779
5780         if (core_relo_is_field_based(spec->relo_kind)) {
5781                 for (i = 0; i < spec->len; i++) {
5782                         if (spec->spec[i].name)
5783                                 libbpf_print(level, ".%s", spec->spec[i].name);
5784                         else if (i > 0 || spec->spec[i].idx > 0)
5785                                 libbpf_print(level, "[%u]", spec->spec[i].idx);
5786                 }
5787
5788                 libbpf_print(level, " (");
5789                 for (i = 0; i < spec->raw_len; i++)
5790                         libbpf_print(level, "%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
5791
5792                 if (spec->bit_offset % 8)
5793                         libbpf_print(level, " @ offset %u.%u)",
5794                                      spec->bit_offset / 8, spec->bit_offset % 8);
5795                 else
5796                         libbpf_print(level, " @ offset %u)", spec->bit_offset / 8);
5797                 return;
5798         }
5799 }
5800
5801 static size_t bpf_core_hash_fn(const void *key, void *ctx)
5802 {
5803         return (size_t)key;
5804 }
5805
5806 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
5807 {
5808         return k1 == k2;
5809 }
5810
5811 static void *u32_as_hash_key(__u32 x)
5812 {
5813         return (void *)(uintptr_t)x;
5814 }
5815
5816 /*
5817  * CO-RE relocate single instruction.
5818  *
5819  * The outline and important points of the algorithm:
5820  * 1. For given local type, find corresponding candidate target types.
5821  *    Candidate type is a type with the same "essential" name, ignoring
5822  *    everything after last triple underscore (___). E.g., `sample`,
5823  *    `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
5824  *    for each other. Names with triple underscore are referred to as
5825  *    "flavors" and are useful, among other things, to allow to
5826  *    specify/support incompatible variations of the same kernel struct, which
5827  *    might differ between different kernel versions and/or build
5828  *    configurations.
5829  *
5830  *    N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
5831  *    converter, when deduplicated BTF of a kernel still contains more than
5832  *    one different types with the same name. In that case, ___2, ___3, etc
5833  *    are appended starting from second name conflict. But start flavors are
5834  *    also useful to be defined "locally", in BPF program, to extract same
5835  *    data from incompatible changes between different kernel
5836  *    versions/configurations. For instance, to handle field renames between
5837  *    kernel versions, one can use two flavors of the struct name with the
5838  *    same common name and use conditional relocations to extract that field,
5839  *    depending on target kernel version.
5840  * 2. For each candidate type, try to match local specification to this
5841  *    candidate target type. Matching involves finding corresponding
5842  *    high-level spec accessors, meaning that all named fields should match,
5843  *    as well as all array accesses should be within the actual bounds. Also,
5844  *    types should be compatible (see bpf_core_fields_are_compat for details).
5845  * 3. It is supported and expected that there might be multiple flavors
5846  *    matching the spec. As long as all the specs resolve to the same set of
5847  *    offsets across all candidates, there is no error. If there is any
5848  *    ambiguity, CO-RE relocation will fail. This is necessary to accomodate
5849  *    imprefection of BTF deduplication, which can cause slight duplication of
5850  *    the same BTF type, if some directly or indirectly referenced (by
5851  *    pointer) type gets resolved to different actual types in different
5852  *    object files. If such situation occurs, deduplicated BTF will end up
5853  *    with two (or more) structurally identical types, which differ only in
5854  *    types they refer to through pointer. This should be OK in most cases and
5855  *    is not an error.
5856  * 4. Candidate types search is performed by linearly scanning through all
5857  *    types in target BTF. It is anticipated that this is overall more
5858  *    efficient memory-wise and not significantly worse (if not better)
5859  *    CPU-wise compared to prebuilding a map from all local type names to
5860  *    a list of candidate type names. It's also sped up by caching resolved
5861  *    list of matching candidates per each local "root" type ID, that has at
5862  *    least one bpf_core_relo associated with it. This list is shared
5863  *    between multiple relocations for the same type ID and is updated as some
5864  *    of the candidates are pruned due to structural incompatibility.
5865  */
5866 static int bpf_core_apply_relo(struct bpf_program *prog,
5867                                const struct bpf_core_relo *relo,
5868                                int relo_idx,
5869                                const struct btf *local_btf,
5870                                struct hashmap *cand_cache)
5871 {
5872         struct bpf_core_spec local_spec, cand_spec, targ_spec = {};
5873         const void *type_key = u32_as_hash_key(relo->type_id);
5874         struct bpf_core_relo_res cand_res, targ_res;
5875         const struct btf_type *local_type;
5876         const char *local_name;
5877         struct core_cand_list *cands = NULL;
5878         __u32 local_id;
5879         const char *spec_str;
5880         int i, j, err;
5881
5882         local_id = relo->type_id;
5883         local_type = btf__type_by_id(local_btf, local_id);
5884         if (!local_type)
5885                 return -EINVAL;
5886
5887         local_name = btf__name_by_offset(local_btf, local_type->name_off);
5888         if (!local_name)
5889                 return -EINVAL;
5890
5891         spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
5892         if (str_is_empty(spec_str))
5893                 return -EINVAL;
5894
5895         err = bpf_core_parse_spec(local_btf, local_id, spec_str, relo->kind, &local_spec);
5896         if (err) {
5897                 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
5898                         prog->name, relo_idx, local_id, btf_kind_str(local_type),
5899                         str_is_empty(local_name) ? "<anon>" : local_name,
5900                         spec_str, err);
5901                 return -EINVAL;
5902         }
5903
5904         pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog->name,
5905                  relo_idx, core_relo_kind_str(relo->kind), relo->kind);
5906         bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
5907         libbpf_print(LIBBPF_DEBUG, "\n");
5908
5909         /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
5910         if (relo->kind == BPF_TYPE_ID_LOCAL) {
5911                 targ_res.validate = true;
5912                 targ_res.poison = false;
5913                 targ_res.orig_val = local_spec.root_type_id;
5914                 targ_res.new_val = local_spec.root_type_id;
5915                 goto patch_insn;
5916         }
5917
5918         /* libbpf doesn't support candidate search for anonymous types */
5919         if (str_is_empty(spec_str)) {
5920                 pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
5921                         prog->name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
5922                 return -EOPNOTSUPP;
5923         }
5924
5925         if (!hashmap__find(cand_cache, type_key, (void **)&cands)) {
5926                 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5927                 if (IS_ERR(cands)) {
5928                         pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5929                                 prog->name, relo_idx, local_id, btf_kind_str(local_type),
5930                                 local_name, PTR_ERR(cands));
5931                         return PTR_ERR(cands);
5932                 }
5933                 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
5934                 if (err) {
5935                         bpf_core_free_cands(cands);
5936                         return err;
5937                 }
5938         }
5939
5940         for (i = 0, j = 0; i < cands->len; i++) {
5941                 err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,
5942                                           cands->cands[i].id, &cand_spec);
5943                 if (err < 0) {
5944                         pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
5945                                 prog->name, relo_idx, i);
5946                         bpf_core_dump_spec(LIBBPF_WARN, &cand_spec);
5947                         libbpf_print(LIBBPF_WARN, ": %d\n", err);
5948                         return err;
5949                 }
5950
5951                 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog->name,
5952                          relo_idx, err == 0 ? "non-matching" : "matching", i);
5953                 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
5954                 libbpf_print(LIBBPF_DEBUG, "\n");
5955
5956                 if (err == 0)
5957                         continue;
5958
5959                 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, &cand_spec, &cand_res);
5960                 if (err)
5961                         return err;
5962
5963                 if (j == 0) {
5964                         targ_res = cand_res;
5965                         targ_spec = cand_spec;
5966                 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
5967                         /* if there are many field relo candidates, they
5968                          * should all resolve to the same bit offset
5969                          */
5970                         pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
5971                                 prog->name, relo_idx, cand_spec.bit_offset,
5972                                 targ_spec.bit_offset);
5973                         return -EINVAL;
5974                 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {
5975                         /* all candidates should result in the same relocation
5976                          * decision and value, otherwise it's dangerous to
5977                          * proceed due to ambiguity
5978                          */
5979                         pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n",
5980                                 prog->name, relo_idx,
5981                                 cand_res.poison ? "failure" : "success", cand_res.new_val,
5982                                 targ_res.poison ? "failure" : "success", targ_res.new_val);
5983                         return -EINVAL;
5984                 }
5985
5986                 cands->cands[j++] = cands->cands[i];
5987         }
5988
5989         /*
5990          * For BPF_FIELD_EXISTS relo or when used BPF program has field
5991          * existence checks or kernel version/config checks, it's expected
5992          * that we might not find any candidates. In this case, if field
5993          * wasn't found in any candidate, the list of candidates shouldn't
5994          * change at all, we'll just handle relocating appropriately,
5995          * depending on relo's kind.
5996          */
5997         if (j > 0)
5998                 cands->len = j;
5999
6000         /*
6001          * If no candidates were found, it might be both a programmer error,
6002          * as well as expected case, depending whether instruction w/
6003          * relocation is guarded in some way that makes it unreachable (dead
6004          * code) if relocation can't be resolved. This is handled in
6005          * bpf_core_patch_insn() uniformly by replacing that instruction with
6006          * BPF helper call insn (using invalid helper ID). If that instruction
6007          * is indeed unreachable, then it will be ignored and eliminated by
6008          * verifier. If it was an error, then verifier will complain and point
6009          * to a specific instruction number in its log.
6010          */
6011         if (j == 0) {
6012                 pr_debug("prog '%s': relo #%d: no matching targets found\n",
6013                          prog->name, relo_idx);
6014
6015                 /* calculate single target relo result explicitly */
6016                 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, NULL, &targ_res);
6017                 if (err)
6018                         return err;
6019         }
6020
6021 patch_insn:
6022         /* bpf_core_patch_insn() should know how to handle missing targ_spec */
6023         err = bpf_core_patch_insn(prog, relo, relo_idx, &targ_res);
6024         if (err) {
6025                 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
6026                         prog->name, relo_idx, relo->insn_off, err);
6027                 return -EINVAL;
6028         }
6029
6030         return 0;
6031 }
6032
6033 static int
6034 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
6035 {
6036         const struct btf_ext_info_sec *sec;
6037         const struct bpf_core_relo *rec;
6038         const struct btf_ext_info *seg;
6039         struct hashmap_entry *entry;
6040         struct hashmap *cand_cache = NULL;
6041         struct bpf_program *prog;
6042         const char *sec_name;
6043         int i, err = 0, insn_idx, sec_idx;
6044
6045         if (obj->btf_ext->core_relo_info.len == 0)
6046                 return 0;
6047
6048         if (targ_btf_path) {
6049                 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6050                 if (IS_ERR_OR_NULL(obj->btf_vmlinux_override)) {
6051                         err = PTR_ERR(obj->btf_vmlinux_override);
6052                         pr_warn("failed to parse target BTF: %d\n", err);
6053                         return err;
6054                 }
6055         }
6056
6057         cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
6058         if (IS_ERR(cand_cache)) {
6059                 err = PTR_ERR(cand_cache);
6060                 goto out;
6061         }
6062
6063         seg = &obj->btf_ext->core_relo_info;
6064         for_each_btf_ext_sec(seg, sec) {
6065                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6066                 if (str_is_empty(sec_name)) {
6067                         err = -EINVAL;
6068                         goto out;
6069                 }
6070                 /* bpf_object's ELF is gone by now so it's not easy to find
6071                  * section index by section name, but we can find *any*
6072                  * bpf_program within desired section name and use it's
6073                  * prog->sec_idx to do a proper search by section index and
6074                  * instruction offset
6075                  */
6076                 prog = NULL;
6077                 for (i = 0; i < obj->nr_programs; i++) {
6078                         prog = &obj->programs[i];
6079                         if (strcmp(prog->sec_name, sec_name) == 0)
6080                                 break;
6081                 }
6082                 if (!prog) {
6083                         pr_warn("sec '%s': failed to find a BPF program\n", sec_name);
6084                         return -ENOENT;
6085                 }
6086                 sec_idx = prog->sec_idx;
6087
6088                 pr_debug("sec '%s': found %d CO-RE relocations\n",
6089                          sec_name, sec->num_info);
6090
6091                 for_each_btf_ext_rec(seg, sec, i, rec) {
6092                         insn_idx = rec->insn_off / BPF_INSN_SZ;
6093                         prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
6094                         if (!prog) {
6095                                 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n",
6096                                         sec_name, insn_idx, i);
6097                                 err = -EINVAL;
6098                                 goto out;
6099                         }
6100                         /* no need to apply CO-RE relocation if the program is
6101                          * not going to be loaded
6102                          */
6103                         if (!prog->load)
6104                                 continue;
6105
6106                         err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache);
6107                         if (err) {
6108                                 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
6109                                         prog->name, i, err);
6110                                 goto out;
6111                         }
6112                 }
6113         }
6114
6115 out:
6116         /* obj->btf_vmlinux and module BTFs are freed after object load */
6117         btf__free(obj->btf_vmlinux_override);
6118         obj->btf_vmlinux_override = NULL;
6119
6120         if (!IS_ERR_OR_NULL(cand_cache)) {
6121                 hashmap__for_each_entry(cand_cache, entry, i) {
6122                         bpf_core_free_cands(entry->value);
6123                 }
6124                 hashmap__free(cand_cache);
6125         }
6126         return err;
6127 }
6128
6129 /* Relocate data references within program code:
6130  *  - map references;
6131  *  - global variable references;
6132  *  - extern references.
6133  */
6134 static int
6135 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
6136 {
6137         int i;
6138
6139         for (i = 0; i < prog->nr_reloc; i++) {
6140                 struct reloc_desc *relo = &prog->reloc_desc[i];
6141                 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6142                 struct extern_desc *ext;
6143
6144                 switch (relo->type) {
6145                 case RELO_LD64:
6146                         insn[0].src_reg = BPF_PSEUDO_MAP_FD;
6147                         insn[0].imm = obj->maps[relo->map_idx].fd;
6148                         relo->processed = true;
6149                         break;
6150                 case RELO_DATA:
6151                         insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6152                         insn[1].imm = insn[0].imm + relo->sym_off;
6153                         insn[0].imm = obj->maps[relo->map_idx].fd;
6154                         relo->processed = true;
6155                         break;
6156                 case RELO_EXTERN:
6157                         ext = &obj->externs[relo->sym_off];
6158                         if (ext->type == EXT_KCFG) {
6159                                 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6160                                 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6161                                 insn[1].imm = ext->kcfg.data_off;
6162                         } else /* EXT_KSYM */ {
6163                                 if (ext->ksym.type_id) { /* typed ksyms */
6164                                         insn[0].src_reg = BPF_PSEUDO_BTF_ID;
6165                                         insn[0].imm = ext->ksym.vmlinux_btf_id;
6166                                 } else { /* typeless ksyms */
6167                                         insn[0].imm = (__u32)ext->ksym.addr;
6168                                         insn[1].imm = ext->ksym.addr >> 32;
6169                                 }
6170                         }
6171                         relo->processed = true;
6172                         break;
6173                 case RELO_CALL:
6174                         /* will be handled as a follow up pass */
6175                         break;
6176                 default:
6177                         pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6178                                 prog->name, i, relo->type);
6179                         return -EINVAL;
6180                 }
6181         }
6182
6183         return 0;
6184 }
6185
6186 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
6187                                     const struct bpf_program *prog,
6188                                     const struct btf_ext_info *ext_info,
6189                                     void **prog_info, __u32 *prog_rec_cnt,
6190                                     __u32 *prog_rec_sz)
6191 {
6192         void *copy_start = NULL, *copy_end = NULL;
6193         void *rec, *rec_end, *new_prog_info;
6194         const struct btf_ext_info_sec *sec;
6195         size_t old_sz, new_sz;
6196         const char *sec_name;
6197         int i, off_adj;
6198
6199         for_each_btf_ext_sec(ext_info, sec) {
6200                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6201                 if (!sec_name)
6202                         return -EINVAL;
6203                 if (strcmp(sec_name, prog->sec_name) != 0)
6204                         continue;
6205
6206                 for_each_btf_ext_rec(ext_info, sec, i, rec) {
6207                         __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
6208
6209                         if (insn_off < prog->sec_insn_off)
6210                                 continue;
6211                         if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6212                                 break;
6213
6214                         if (!copy_start)
6215                                 copy_start = rec;
6216                         copy_end = rec + ext_info->rec_size;
6217                 }
6218
6219                 if (!copy_start)
6220                         return -ENOENT;
6221
6222                 /* append func/line info of a given (sub-)program to the main
6223                  * program func/line info
6224                  */
6225                 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6226                 new_sz = old_sz + (copy_end - copy_start);
6227                 new_prog_info = realloc(*prog_info, new_sz);
6228                 if (!new_prog_info)
6229                         return -ENOMEM;
6230                 *prog_info = new_prog_info;
6231                 *prog_rec_cnt = new_sz / ext_info->rec_size;
6232                 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6233
6234                 /* Kernel instruction offsets are in units of 8-byte
6235                  * instructions, while .BTF.ext instruction offsets generated
6236                  * by Clang are in units of bytes. So convert Clang offsets
6237                  * into kernel offsets and adjust offset according to program
6238                  * relocated position.
6239                  */
6240                 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6241                 rec = new_prog_info + old_sz;
6242                 rec_end = new_prog_info + new_sz;
6243                 for (; rec < rec_end; rec += ext_info->rec_size) {
6244                         __u32 *insn_off = rec;
6245
6246                         *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
6247                 }
6248                 *prog_rec_sz = ext_info->rec_size;
6249                 return 0;
6250         }
6251
6252         return -ENOENT;
6253 }
6254
6255 static int
6256 reloc_prog_func_and_line_info(const struct bpf_object *obj,
6257                               struct bpf_program *main_prog,
6258                               const struct bpf_program *prog)
6259 {
6260         int err;
6261
6262         /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
6263          * supprot func/line info
6264          */
6265         if (!obj->btf_ext || !kernel_supports(FEAT_BTF_FUNC))
6266                 return 0;
6267
6268         /* only attempt func info relocation if main program's func_info
6269          * relocation was successful
6270          */
6271         if (main_prog != prog && !main_prog->func_info)
6272                 goto line_info;
6273
6274         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6275                                        &main_prog->func_info,
6276                                        &main_prog->func_info_cnt,
6277                                        &main_prog->func_info_rec_size);
6278         if (err) {
6279                 if (err != -ENOENT) {
6280                         pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6281                                 prog->name, err);
6282                         return err;
6283                 }
6284                 if (main_prog->func_info) {
6285                         /*
6286                          * Some info has already been found but has problem
6287                          * in the last btf_ext reloc. Must have to error out.
6288                          */
6289                         pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6290                         return err;
6291                 }
6292                 /* Have problem loading the very first info. Ignore the rest. */
6293                 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6294                         prog->name);
6295         }
6296
6297 line_info:
6298         /* don't relocate line info if main program's relocation failed */
6299         if (main_prog != prog && !main_prog->line_info)
6300                 return 0;
6301
6302         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6303                                        &main_prog->line_info,
6304                                        &main_prog->line_info_cnt,
6305                                        &main_prog->line_info_rec_size);
6306         if (err) {
6307                 if (err != -ENOENT) {
6308                         pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6309                                 prog->name, err);
6310                         return err;
6311                 }
6312                 if (main_prog->line_info) {
6313                         /*
6314                          * Some info has already been found but has problem
6315                          * in the last btf_ext reloc. Must have to error out.
6316                          */
6317                         pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6318                         return err;
6319                 }
6320                 /* Have problem loading the very first info. Ignore the rest. */
6321                 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6322                         prog->name);
6323         }
6324         return 0;
6325 }
6326
6327 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6328 {
6329         size_t insn_idx = *(const size_t *)key;
6330         const struct reloc_desc *relo = elem;
6331
6332         if (insn_idx == relo->insn_idx)
6333                 return 0;
6334         return insn_idx < relo->insn_idx ? -1 : 1;
6335 }
6336
6337 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6338 {
6339         return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6340                        sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6341 }
6342
6343 static int
6344 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6345                        struct bpf_program *prog)
6346 {
6347         size_t sub_insn_idx, insn_idx, new_cnt;
6348         struct bpf_program *subprog;
6349         struct bpf_insn *insns, *insn;
6350         struct reloc_desc *relo;
6351         int err;
6352
6353         err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6354         if (err)
6355                 return err;
6356
6357         for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6358                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6359                 if (!insn_is_subprog_call(insn))
6360                         continue;
6361
6362                 relo = find_prog_insn_relo(prog, insn_idx);
6363                 if (relo && relo->type != RELO_CALL) {
6364                         pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6365                                 prog->name, insn_idx, relo->type);
6366                         return -LIBBPF_ERRNO__RELOC;
6367                 }
6368                 if (relo) {
6369                         /* sub-program instruction index is a combination of
6370                          * an offset of a symbol pointed to by relocation and
6371                          * call instruction's imm field; for global functions,
6372                          * call always has imm = -1, but for static functions
6373                          * relocation is against STT_SECTION and insn->imm
6374                          * points to a start of a static function
6375                          */
6376                         sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6377                 } else {
6378                         /* if subprogram call is to a static function within
6379                          * the same ELF section, there won't be any relocation
6380                          * emitted, but it also means there is no additional
6381                          * offset necessary, insns->imm is relative to
6382                          * instruction's original position within the section
6383                          */
6384                         sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6385                 }
6386
6387                 /* we enforce that sub-programs should be in .text section */
6388                 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6389                 if (!subprog) {
6390                         pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6391                                 prog->name);
6392                         return -LIBBPF_ERRNO__RELOC;
6393                 }
6394
6395                 /* if it's the first call instruction calling into this
6396                  * subprogram (meaning this subprog hasn't been processed
6397                  * yet) within the context of current main program:
6398                  *   - append it at the end of main program's instructions blog;
6399                  *   - process is recursively, while current program is put on hold;
6400                  *   - if that subprogram calls some other not yet processes
6401                  *   subprogram, same thing will happen recursively until
6402                  *   there are no more unprocesses subprograms left to append
6403                  *   and relocate.
6404                  */
6405                 if (subprog->sub_insn_off == 0) {
6406                         subprog->sub_insn_off = main_prog->insns_cnt;
6407
6408                         new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6409                         insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6410                         if (!insns) {
6411                                 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6412                                 return -ENOMEM;
6413                         }
6414                         main_prog->insns = insns;
6415                         main_prog->insns_cnt = new_cnt;
6416
6417                         memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6418                                subprog->insns_cnt * sizeof(*insns));
6419
6420                         pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6421                                  main_prog->name, subprog->insns_cnt, subprog->name);
6422
6423                         err = bpf_object__reloc_code(obj, main_prog, subprog);
6424                         if (err)
6425                                 return err;
6426                 }
6427
6428                 /* main_prog->insns memory could have been re-allocated, so
6429                  * calculate pointer again
6430                  */
6431                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6432                 /* calculate correct instruction position within current main
6433                  * prog; each main prog can have a different set of
6434                  * subprograms appended (potentially in different order as
6435                  * well), so position of any subprog can be different for
6436                  * different main programs */
6437                 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6438
6439                 if (relo)
6440                         relo->processed = true;
6441
6442                 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6443                          prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6444         }
6445
6446         return 0;
6447 }
6448
6449 /*
6450  * Relocate sub-program calls.
6451  *
6452  * Algorithm operates as follows. Each entry-point BPF program (referred to as
6453  * main prog) is processed separately. For each subprog (non-entry functions,
6454  * that can be called from either entry progs or other subprogs) gets their
6455  * sub_insn_off reset to zero. This serves as indicator that this subprogram
6456  * hasn't been yet appended and relocated within current main prog. Once its
6457  * relocated, sub_insn_off will point at the position within current main prog
6458  * where given subprog was appended. This will further be used to relocate all
6459  * the call instructions jumping into this subprog.
6460  *
6461  * We start with main program and process all call instructions. If the call
6462  * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6463  * is zero), subprog instructions are appended at the end of main program's
6464  * instruction array. Then main program is "put on hold" while we recursively
6465  * process newly appended subprogram. If that subprogram calls into another
6466  * subprogram that hasn't been appended, new subprogram is appended again to
6467  * the *main* prog's instructions (subprog's instructions are always left
6468  * untouched, as they need to be in unmodified state for subsequent main progs
6469  * and subprog instructions are always sent only as part of a main prog) and
6470  * the process continues recursively. Once all the subprogs called from a main
6471  * prog or any of its subprogs are appended (and relocated), all their
6472  * positions within finalized instructions array are known, so it's easy to
6473  * rewrite call instructions with correct relative offsets, corresponding to
6474  * desired target subprog.
6475  *
6476  * Its important to realize that some subprogs might not be called from some
6477  * main prog and any of its called/used subprogs. Those will keep their
6478  * subprog->sub_insn_off as zero at all times and won't be appended to current
6479  * main prog and won't be relocated within the context of current main prog.
6480  * They might still be used from other main progs later.
6481  *
6482  * Visually this process can be shown as below. Suppose we have two main
6483  * programs mainA and mainB and BPF object contains three subprogs: subA,
6484  * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6485  * subC both call subB:
6486  *
6487  *        +--------+ +-------+
6488  *        |        v v       |
6489  *     +--+---+ +--+-+-+ +---+--+
6490  *     | subA | | subB | | subC |
6491  *     +--+---+ +------+ +---+--+
6492  *        ^                  ^
6493  *        |                  |
6494  *    +---+-------+   +------+----+
6495  *    |   mainA   |   |   mainB   |
6496  *    +-----------+   +-----------+
6497  *
6498  * We'll start relocating mainA, will find subA, append it and start
6499  * processing sub A recursively:
6500  *
6501  *    +-----------+------+
6502  *    |   mainA   | subA |
6503  *    +-----------+------+
6504  *
6505  * At this point we notice that subB is used from subA, so we append it and
6506  * relocate (there are no further subcalls from subB):
6507  *
6508  *    +-----------+------+------+
6509  *    |   mainA   | subA | subB |
6510  *    +-----------+------+------+
6511  *
6512  * At this point, we relocate subA calls, then go one level up and finish with
6513  * relocatin mainA calls. mainA is done.
6514  *
6515  * For mainB process is similar but results in different order. We start with
6516  * mainB and skip subA and subB, as mainB never calls them (at least
6517  * directly), but we see subC is needed, so we append and start processing it:
6518  *
6519  *    +-----------+------+
6520  *    |   mainB   | subC |
6521  *    +-----------+------+
6522  * Now we see subC needs subB, so we go back to it, append and relocate it:
6523  *
6524  *    +-----------+------+------+
6525  *    |   mainB   | subC | subB |
6526  *    +-----------+------+------+
6527  *
6528  * At this point we unwind recursion, relocate calls in subC, then in mainB.
6529  */
6530 static int
6531 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6532 {
6533         struct bpf_program *subprog;
6534         int i, j, err;
6535
6536         /* mark all subprogs as not relocated (yet) within the context of
6537          * current main program
6538          */
6539         for (i = 0; i < obj->nr_programs; i++) {
6540                 subprog = &obj->programs[i];
6541                 if (!prog_is_subprog(obj, subprog))
6542                         continue;
6543
6544                 subprog->sub_insn_off = 0;
6545                 for (j = 0; j < subprog->nr_reloc; j++)
6546                         if (subprog->reloc_desc[j].type == RELO_CALL)
6547                                 subprog->reloc_desc[j].processed = false;
6548         }
6549
6550         err = bpf_object__reloc_code(obj, prog, prog);
6551         if (err)
6552                 return err;
6553
6554
6555         return 0;
6556 }
6557
6558 static int
6559 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6560 {
6561         struct bpf_program *prog;
6562         size_t i;
6563         int err;
6564
6565         if (obj->btf_ext) {
6566                 err = bpf_object__relocate_core(obj, targ_btf_path);
6567                 if (err) {
6568                         pr_warn("failed to perform CO-RE relocations: %d\n",
6569                                 err);
6570                         return err;
6571                 }
6572         }
6573         /* relocate data references first for all programs and sub-programs,
6574          * as they don't change relative to code locations, so subsequent
6575          * subprogram processing won't need to re-calculate any of them
6576          */
6577         for (i = 0; i < obj->nr_programs; i++) {
6578                 prog = &obj->programs[i];
6579                 err = bpf_object__relocate_data(obj, prog);
6580                 if (err) {
6581                         pr_warn("prog '%s': failed to relocate data references: %d\n",
6582                                 prog->name, err);
6583                         return err;
6584                 }
6585         }
6586         /* now relocate subprogram calls and append used subprograms to main
6587          * programs; each copy of subprogram code needs to be relocated
6588          * differently for each main program, because its code location might
6589          * have changed
6590          */
6591         for (i = 0; i < obj->nr_programs; i++) {
6592                 prog = &obj->programs[i];
6593                 /* sub-program's sub-calls are relocated within the context of
6594                  * its main program only
6595                  */
6596                 if (prog_is_subprog(obj, prog))
6597                         continue;
6598
6599                 err = bpf_object__relocate_calls(obj, prog);
6600                 if (err) {
6601                         pr_warn("prog '%s': failed to relocate calls: %d\n",
6602                                 prog->name, err);
6603                         return err;
6604                 }
6605         }
6606         /* free up relocation descriptors */
6607         for (i = 0; i < obj->nr_programs; i++) {
6608                 prog = &obj->programs[i];
6609                 zfree(&prog->reloc_desc);
6610                 prog->nr_reloc = 0;
6611         }
6612         return 0;
6613 }
6614
6615 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6616                                             GElf_Shdr *shdr, Elf_Data *data);
6617
6618 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6619                                          GElf_Shdr *shdr, Elf_Data *data)
6620 {
6621         const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6622         int i, j, nrels, new_sz;
6623         const struct btf_var_secinfo *vi = NULL;
6624         const struct btf_type *sec, *var, *def;
6625         struct bpf_map *map = NULL, *targ_map;
6626         const struct btf_member *member;
6627         const char *name, *mname;
6628         Elf_Data *symbols;
6629         unsigned int moff;
6630         GElf_Sym sym;
6631         GElf_Rel rel;
6632         void *tmp;
6633
6634         if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6635                 return -EINVAL;
6636         sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6637         if (!sec)
6638                 return -EINVAL;
6639
6640         symbols = obj->efile.symbols;
6641         nrels = shdr->sh_size / shdr->sh_entsize;
6642         for (i = 0; i < nrels; i++) {
6643                 if (!gelf_getrel(data, i, &rel)) {
6644                         pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6645                         return -LIBBPF_ERRNO__FORMAT;
6646                 }
6647                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
6648                         pr_warn(".maps relo #%d: symbol %zx not found\n",
6649                                 i, (size_t)GELF_R_SYM(rel.r_info));
6650                         return -LIBBPF_ERRNO__FORMAT;
6651                 }
6652                 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
6653                 if (sym.st_shndx != obj->efile.btf_maps_shndx) {
6654                         pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6655                                 i, name);
6656                         return -LIBBPF_ERRNO__RELOC;
6657                 }
6658
6659                 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n",
6660                          i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value,
6661                          (size_t)rel.r_offset, sym.st_name, name);
6662
6663                 for (j = 0; j < obj->nr_maps; j++) {
6664                         map = &obj->maps[j];
6665                         if (map->sec_idx != obj->efile.btf_maps_shndx)
6666                                 continue;
6667
6668                         vi = btf_var_secinfos(sec) + map->btf_var_idx;
6669                         if (vi->offset <= rel.r_offset &&
6670                             rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6671                                 break;
6672                 }
6673                 if (j == obj->nr_maps) {
6674                         pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n",
6675                                 i, name, (size_t)rel.r_offset);
6676                         return -EINVAL;
6677                 }
6678
6679                 if (!bpf_map_type__is_map_in_map(map->def.type))
6680                         return -EINVAL;
6681                 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6682                     map->def.key_size != sizeof(int)) {
6683                         pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6684                                 i, map->name, sizeof(int));
6685                         return -EINVAL;
6686                 }
6687
6688                 targ_map = bpf_object__find_map_by_name(obj, name);
6689                 if (!targ_map)
6690                         return -ESRCH;
6691
6692                 var = btf__type_by_id(obj->btf, vi->type);
6693                 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6694                 if (btf_vlen(def) == 0)
6695                         return -EINVAL;
6696                 member = btf_members(def) + btf_vlen(def) - 1;
6697                 mname = btf__name_by_offset(obj->btf, member->name_off);
6698                 if (strcmp(mname, "values"))
6699                         return -EINVAL;
6700
6701                 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6702                 if (rel.r_offset - vi->offset < moff)
6703                         return -EINVAL;
6704
6705                 moff = rel.r_offset - vi->offset - moff;
6706                 /* here we use BPF pointer size, which is always 64 bit, as we
6707                  * are parsing ELF that was built for BPF target
6708                  */
6709                 if (moff % bpf_ptr_sz)
6710                         return -EINVAL;
6711                 moff /= bpf_ptr_sz;
6712                 if (moff >= map->init_slots_sz) {
6713                         new_sz = moff + 1;
6714                         tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6715                         if (!tmp)
6716                                 return -ENOMEM;
6717                         map->init_slots = tmp;
6718                         memset(map->init_slots + map->init_slots_sz, 0,
6719                                (new_sz - map->init_slots_sz) * host_ptr_sz);
6720                         map->init_slots_sz = new_sz;
6721                 }
6722                 map->init_slots[moff] = targ_map;
6723
6724                 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n",
6725                          i, map->name, moff, name);
6726         }
6727
6728         return 0;
6729 }
6730
6731 static int cmp_relocs(const void *_a, const void *_b)
6732 {
6733         const struct reloc_desc *a = _a;
6734         const struct reloc_desc *b = _b;
6735
6736         if (a->insn_idx != b->insn_idx)
6737                 return a->insn_idx < b->insn_idx ? -1 : 1;
6738
6739         /* no two relocations should have the same insn_idx, but ... */
6740         if (a->type != b->type)
6741                 return a->type < b->type ? -1 : 1;
6742
6743         return 0;
6744 }
6745
6746 static int bpf_object__collect_relos(struct bpf_object *obj)
6747 {
6748         int i, err;
6749
6750         for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
6751                 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
6752                 Elf_Data *data = obj->efile.reloc_sects[i].data;
6753                 int idx = shdr->sh_info;
6754
6755                 if (shdr->sh_type != SHT_REL) {
6756                         pr_warn("internal error at %d\n", __LINE__);
6757                         return -LIBBPF_ERRNO__INTERNAL;
6758                 }
6759
6760                 if (idx == obj->efile.st_ops_shndx)
6761                         err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6762                 else if (idx == obj->efile.btf_maps_shndx)
6763                         err = bpf_object__collect_map_relos(obj, shdr, data);
6764                 else
6765                         err = bpf_object__collect_prog_relos(obj, shdr, data);
6766                 if (err)
6767                         return err;
6768         }
6769
6770         for (i = 0; i < obj->nr_programs; i++) {
6771                 struct bpf_program *p = &obj->programs[i];
6772
6773                 if (!p->nr_reloc)
6774                         continue;
6775
6776                 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6777         }
6778         return 0;
6779 }
6780
6781 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6782 {
6783         if (BPF_CLASS(insn->code) == BPF_JMP &&
6784             BPF_OP(insn->code) == BPF_CALL &&
6785             BPF_SRC(insn->code) == BPF_K &&
6786             insn->src_reg == 0 &&
6787             insn->dst_reg == 0) {
6788                     *func_id = insn->imm;
6789                     return true;
6790         }
6791         return false;
6792 }
6793
6794 static int bpf_object__sanitize_prog(struct bpf_object* obj, struct bpf_program *prog)
6795 {
6796         struct bpf_insn *insn = prog->insns;
6797         enum bpf_func_id func_id;
6798         int i;
6799
6800         for (i = 0; i < prog->insns_cnt; i++, insn++) {
6801                 if (!insn_is_helper_call(insn, &func_id))
6802                         continue;
6803
6804                 /* on kernels that don't yet support
6805                  * bpf_probe_read_{kernel,user}[_str] helpers, fall back
6806                  * to bpf_probe_read() which works well for old kernels
6807                  */
6808                 switch (func_id) {
6809                 case BPF_FUNC_probe_read_kernel:
6810                 case BPF_FUNC_probe_read_user:
6811                         if (!kernel_supports(FEAT_PROBE_READ_KERN))
6812                                 insn->imm = BPF_FUNC_probe_read;
6813                         break;
6814                 case BPF_FUNC_probe_read_kernel_str:
6815                 case BPF_FUNC_probe_read_user_str:
6816                         if (!kernel_supports(FEAT_PROBE_READ_KERN))
6817                                 insn->imm = BPF_FUNC_probe_read_str;
6818                         break;
6819                 default:
6820                         break;
6821                 }
6822         }
6823         return 0;
6824 }
6825
6826 static int
6827 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
6828              char *license, __u32 kern_version, int *pfd)
6829 {
6830         struct bpf_prog_load_params load_attr = {};
6831         char *cp, errmsg[STRERR_BUFSIZE];
6832         size_t log_buf_size = 0;
6833         char *log_buf = NULL;
6834         int btf_fd, ret;
6835
6836         if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6837                 /*
6838                  * The program type must be set.  Most likely we couldn't find a proper
6839                  * section definition at load time, and thus we didn't infer the type.
6840                  */
6841                 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6842                         prog->name, prog->sec_name);
6843                 return -EINVAL;
6844         }
6845
6846         if (!insns || !insns_cnt)
6847                 return -EINVAL;
6848
6849         load_attr.prog_type = prog->type;
6850         /* old kernels might not support specifying expected_attach_type */
6851         if (!kernel_supports(FEAT_EXP_ATTACH_TYPE) && prog->sec_def &&
6852             prog->sec_def->is_exp_attach_type_optional)
6853                 load_attr.expected_attach_type = 0;
6854         else
6855                 load_attr.expected_attach_type = prog->expected_attach_type;
6856         if (kernel_supports(FEAT_PROG_NAME))
6857                 load_attr.name = prog->name;
6858         load_attr.insns = insns;
6859         load_attr.insn_cnt = insns_cnt;
6860         load_attr.license = license;
6861         load_attr.attach_btf_id = prog->attach_btf_id;
6862         if (prog->attach_prog_fd)
6863                 load_attr.attach_prog_fd = prog->attach_prog_fd;
6864         else
6865                 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6866         load_attr.attach_btf_id = prog->attach_btf_id;
6867         load_attr.kern_version = kern_version;
6868         load_attr.prog_ifindex = prog->prog_ifindex;
6869
6870         /* specify func_info/line_info only if kernel supports them */
6871         btf_fd = bpf_object__btf_fd(prog->obj);
6872         if (btf_fd >= 0 && kernel_supports(FEAT_BTF_FUNC)) {
6873                 load_attr.prog_btf_fd = btf_fd;
6874                 load_attr.func_info = prog->func_info;
6875                 load_attr.func_info_rec_size = prog->func_info_rec_size;
6876                 load_attr.func_info_cnt = prog->func_info_cnt;
6877                 load_attr.line_info = prog->line_info;
6878                 load_attr.line_info_rec_size = prog->line_info_rec_size;
6879                 load_attr.line_info_cnt = prog->line_info_cnt;
6880         }
6881         load_attr.log_level = prog->log_level;
6882         load_attr.prog_flags = prog->prog_flags;
6883
6884 retry_load:
6885         if (log_buf_size) {
6886                 log_buf = malloc(log_buf_size);
6887                 if (!log_buf)
6888                         return -ENOMEM;
6889
6890                 *log_buf = 0;
6891         }
6892
6893         load_attr.log_buf = log_buf;
6894         load_attr.log_buf_sz = log_buf_size;
6895         ret = libbpf__bpf_prog_load(&load_attr);
6896
6897         if (ret >= 0) {
6898                 if (log_buf && load_attr.log_level)
6899                         pr_debug("verifier log:\n%s", log_buf);
6900
6901                 if (prog->obj->rodata_map_idx >= 0 &&
6902                     kernel_supports(FEAT_PROG_BIND_MAP)) {
6903                         struct bpf_map *rodata_map =
6904                                 &prog->obj->maps[prog->obj->rodata_map_idx];
6905
6906                         if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) {
6907                                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6908                                 pr_warn("prog '%s': failed to bind .rodata map: %s\n",
6909                                         prog->name, cp);
6910                                 /* Don't fail hard if can't bind rodata. */
6911                         }
6912                 }
6913
6914                 *pfd = ret;
6915                 ret = 0;
6916                 goto out;
6917         }
6918
6919         if (!log_buf || errno == ENOSPC) {
6920                 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
6921                                    log_buf_size << 1);
6922
6923                 free(log_buf);
6924                 goto retry_load;
6925         }
6926         ret = errno ? -errno : -LIBBPF_ERRNO__LOAD;
6927         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6928         pr_warn("load bpf program failed: %s\n", cp);
6929         pr_perm_msg(ret);
6930
6931         if (log_buf && log_buf[0] != '\0') {
6932                 ret = -LIBBPF_ERRNO__VERIFY;
6933                 pr_warn("-- BEGIN DUMP LOG ---\n");
6934                 pr_warn("\n%s\n", log_buf);
6935                 pr_warn("-- END LOG --\n");
6936         } else if (load_attr.insn_cnt >= BPF_MAXINSNS) {
6937                 pr_warn("Program too large (%zu insns), at most %d insns\n",
6938                         load_attr.insn_cnt, BPF_MAXINSNS);
6939                 ret = -LIBBPF_ERRNO__PROG2BIG;
6940         } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
6941                 /* Wrong program type? */
6942                 int fd;
6943
6944                 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
6945                 load_attr.expected_attach_type = 0;
6946                 load_attr.log_buf = NULL;
6947                 load_attr.log_buf_sz = 0;
6948                 fd = libbpf__bpf_prog_load(&load_attr);
6949                 if (fd >= 0) {
6950                         close(fd);
6951                         ret = -LIBBPF_ERRNO__PROGTYPE;
6952                         goto out;
6953                 }
6954         }
6955
6956 out:
6957         free(log_buf);
6958         return ret;
6959 }
6960
6961 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id);
6962
6963 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
6964 {
6965         int err = 0, fd, i;
6966
6967         if (prog->obj->loaded) {
6968                 pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
6969                 return -EINVAL;
6970         }
6971
6972         if ((prog->type == BPF_PROG_TYPE_TRACING ||
6973              prog->type == BPF_PROG_TYPE_LSM ||
6974              prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
6975                 int btf_obj_fd = 0, btf_type_id = 0;
6976
6977                 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id);
6978                 if (err)
6979                         return err;
6980
6981                 prog->attach_btf_obj_fd = btf_obj_fd;
6982                 prog->attach_btf_id = btf_type_id;
6983         }
6984
6985         if (prog->instances.nr < 0 || !prog->instances.fds) {
6986                 if (prog->preprocessor) {
6987                         pr_warn("Internal error: can't load program '%s'\n",
6988                                 prog->name);
6989                         return -LIBBPF_ERRNO__INTERNAL;
6990                 }
6991
6992                 prog->instances.fds = malloc(sizeof(int));
6993                 if (!prog->instances.fds) {
6994                         pr_warn("Not enough memory for BPF fds\n");
6995                         return -ENOMEM;
6996                 }
6997                 prog->instances.nr = 1;
6998                 prog->instances.fds[0] = -1;
6999         }
7000
7001         if (!prog->preprocessor) {
7002                 if (prog->instances.nr != 1) {
7003                         pr_warn("prog '%s': inconsistent nr(%d) != 1\n",
7004                                 prog->name, prog->instances.nr);
7005                 }
7006                 err = load_program(prog, prog->insns, prog->insns_cnt,
7007                                    license, kern_ver, &fd);
7008                 if (!err)
7009                         prog->instances.fds[0] = fd;
7010                 goto out;
7011         }
7012
7013         for (i = 0; i < prog->instances.nr; i++) {
7014                 struct bpf_prog_prep_result result;
7015                 bpf_program_prep_t preprocessor = prog->preprocessor;
7016
7017                 memset(&result, 0, sizeof(result));
7018                 err = preprocessor(prog, i, prog->insns,
7019                                    prog->insns_cnt, &result);
7020                 if (err) {
7021                         pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
7022                                 i, prog->name);
7023                         goto out;
7024                 }
7025
7026                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
7027                         pr_debug("Skip loading the %dth instance of program '%s'\n",
7028                                  i, prog->name);
7029                         prog->instances.fds[i] = -1;
7030                         if (result.pfd)
7031                                 *result.pfd = -1;
7032                         continue;
7033                 }
7034
7035                 err = load_program(prog, result.new_insn_ptr,
7036                                    result.new_insn_cnt, license, kern_ver, &fd);
7037                 if (err) {
7038                         pr_warn("Loading the %dth instance of program '%s' failed\n",
7039                                 i, prog->name);
7040                         goto out;
7041                 }
7042
7043                 if (result.pfd)
7044                         *result.pfd = fd;
7045                 prog->instances.fds[i] = fd;
7046         }
7047 out:
7048         if (err)
7049                 pr_warn("failed to load program '%s'\n", prog->name);
7050         zfree(&prog->insns);
7051         prog->insns_cnt = 0;
7052         return err;
7053 }
7054
7055 static int
7056 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7057 {
7058         struct bpf_program *prog;
7059         size_t i;
7060         int err;
7061
7062         for (i = 0; i < obj->nr_programs; i++) {
7063                 prog = &obj->programs[i];
7064                 err = bpf_object__sanitize_prog(obj, prog);
7065                 if (err)
7066                         return err;
7067         }
7068
7069         for (i = 0; i < obj->nr_programs; i++) {
7070                 prog = &obj->programs[i];
7071                 if (prog_is_subprog(obj, prog))
7072                         continue;
7073                 if (!prog->load) {
7074                         pr_debug("prog '%s': skipped loading\n", prog->name);
7075                         continue;
7076                 }
7077                 prog->log_level |= log_level;
7078                 err = bpf_program__load(prog, obj->license, obj->kern_version);
7079                 if (err)
7080                         return err;
7081         }
7082         return 0;
7083 }
7084
7085 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7086
7087 static struct bpf_object *
7088 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7089                    const struct bpf_object_open_opts *opts)
7090 {
7091         const char *obj_name, *kconfig;
7092         struct bpf_program *prog;
7093         struct bpf_object *obj;
7094         char tmp_name[64];
7095         int err;
7096
7097         if (elf_version(EV_CURRENT) == EV_NONE) {
7098                 pr_warn("failed to init libelf for %s\n",
7099                         path ? : "(mem buf)");
7100                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7101         }
7102
7103         if (!OPTS_VALID(opts, bpf_object_open_opts))
7104                 return ERR_PTR(-EINVAL);
7105
7106         obj_name = OPTS_GET(opts, object_name, NULL);
7107         if (obj_buf) {
7108                 if (!obj_name) {
7109                         snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7110                                  (unsigned long)obj_buf,
7111                                  (unsigned long)obj_buf_sz);
7112                         obj_name = tmp_name;
7113                 }
7114                 path = obj_name;
7115                 pr_debug("loading object '%s' from buffer\n", obj_name);
7116         }
7117
7118         obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7119         if (IS_ERR(obj))
7120                 return obj;
7121
7122         kconfig = OPTS_GET(opts, kconfig, NULL);
7123         if (kconfig) {
7124                 obj->kconfig = strdup(kconfig);
7125                 if (!obj->kconfig)
7126                         return ERR_PTR(-ENOMEM);
7127         }
7128
7129         err = bpf_object__elf_init(obj);
7130         err = err ? : bpf_object__check_endianness(obj);
7131         err = err ? : bpf_object__elf_collect(obj);
7132         err = err ? : bpf_object__collect_externs(obj);
7133         err = err ? : bpf_object__finalize_btf(obj);
7134         err = err ? : bpf_object__init_maps(obj, opts);
7135         err = err ? : bpf_object__collect_relos(obj);
7136         if (err)
7137                 goto out;
7138         bpf_object__elf_finish(obj);
7139
7140         bpf_object__for_each_program(prog, obj) {
7141                 prog->sec_def = find_sec_def(prog->sec_name);
7142                 if (!prog->sec_def) {
7143                         /* couldn't guess, but user might manually specify */
7144                         pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7145                                 prog->name, prog->sec_name);
7146                         continue;
7147                 }
7148
7149                 if (prog->sec_def->is_sleepable)
7150                         prog->prog_flags |= BPF_F_SLEEPABLE;
7151                 bpf_program__set_type(prog, prog->sec_def->prog_type);
7152                 bpf_program__set_expected_attach_type(prog,
7153                                 prog->sec_def->expected_attach_type);
7154
7155                 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
7156                     prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
7157                         prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
7158         }
7159
7160         return obj;
7161 out:
7162         bpf_object__close(obj);
7163         return ERR_PTR(err);
7164 }
7165
7166 static struct bpf_object *
7167 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
7168 {
7169         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7170                 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
7171         );
7172
7173         /* param validation */
7174         if (!attr->file)
7175                 return NULL;
7176
7177         pr_debug("loading %s\n", attr->file);
7178         return __bpf_object__open(attr->file, NULL, 0, &opts);
7179 }
7180
7181 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
7182 {
7183         return __bpf_object__open_xattr(attr, 0);
7184 }
7185
7186 struct bpf_object *bpf_object__open(const char *path)
7187 {
7188         struct bpf_object_open_attr attr = {
7189                 .file           = path,
7190                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
7191         };
7192
7193         return bpf_object__open_xattr(&attr);
7194 }
7195
7196 struct bpf_object *
7197 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7198 {
7199         if (!path)
7200                 return ERR_PTR(-EINVAL);
7201
7202         pr_debug("loading %s\n", path);
7203
7204         return __bpf_object__open(path, NULL, 0, opts);
7205 }
7206
7207 struct bpf_object *
7208 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7209                      const struct bpf_object_open_opts *opts)
7210 {
7211         if (!obj_buf || obj_buf_sz == 0)
7212                 return ERR_PTR(-EINVAL);
7213
7214         return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
7215 }
7216
7217 struct bpf_object *
7218 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
7219                         const char *name)
7220 {
7221         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7222                 .object_name = name,
7223                 /* wrong default, but backwards-compatible */
7224                 .relaxed_maps = true,
7225         );
7226
7227         /* returning NULL is wrong, but backwards-compatible */
7228         if (!obj_buf || obj_buf_sz == 0)
7229                 return NULL;
7230
7231         return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
7232 }
7233
7234 int bpf_object__unload(struct bpf_object *obj)
7235 {
7236         size_t i;
7237
7238         if (!obj)
7239                 return -EINVAL;
7240
7241         for (i = 0; i < obj->nr_maps; i++) {
7242                 zclose(obj->maps[i].fd);
7243                 if (obj->maps[i].st_ops)
7244                         zfree(&obj->maps[i].st_ops->kern_vdata);
7245         }
7246
7247         for (i = 0; i < obj->nr_programs; i++)
7248                 bpf_program__unload(&obj->programs[i]);
7249
7250         return 0;
7251 }
7252
7253 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7254 {
7255         struct bpf_map *m;
7256
7257         bpf_object__for_each_map(m, obj) {
7258                 if (!bpf_map__is_internal(m))
7259                         continue;
7260                 if (!kernel_supports(FEAT_GLOBAL_DATA)) {
7261                         pr_warn("kernel doesn't support global data\n");
7262                         return -ENOTSUP;
7263                 }
7264                 if (!kernel_supports(FEAT_ARRAY_MMAP))
7265                         m->def.map_flags ^= BPF_F_MMAPABLE;
7266         }
7267
7268         return 0;
7269 }
7270
7271 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7272 {
7273         char sym_type, sym_name[500];
7274         unsigned long long sym_addr;
7275         struct extern_desc *ext;
7276         int ret, err = 0;
7277         FILE *f;
7278
7279         f = fopen("/proc/kallsyms", "r");
7280         if (!f) {
7281                 err = -errno;
7282                 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7283                 return err;
7284         }
7285
7286         while (true) {
7287                 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7288                              &sym_addr, &sym_type, sym_name);
7289                 if (ret == EOF && feof(f))
7290                         break;
7291                 if (ret != 3) {
7292                         pr_warn("failed to read kallsyms entry: %d\n", ret);
7293                         err = -EINVAL;
7294                         goto out;
7295                 }
7296
7297                 ext = find_extern_by_name(obj, sym_name);
7298                 if (!ext || ext->type != EXT_KSYM)
7299                         continue;
7300
7301                 if (ext->is_set && ext->ksym.addr != sym_addr) {
7302                         pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
7303                                 sym_name, ext->ksym.addr, sym_addr);
7304                         err = -EINVAL;
7305                         goto out;
7306                 }
7307                 if (!ext->is_set) {
7308                         ext->is_set = true;
7309                         ext->ksym.addr = sym_addr;
7310                         pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr);
7311                 }
7312         }
7313
7314 out:
7315         fclose(f);
7316         return err;
7317 }
7318
7319 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7320 {
7321         struct extern_desc *ext;
7322         int i, id;
7323
7324         for (i = 0; i < obj->nr_extern; i++) {
7325                 const struct btf_type *targ_var, *targ_type;
7326                 __u32 targ_type_id, local_type_id;
7327                 const char *targ_var_name;
7328                 int ret;
7329
7330                 ext = &obj->externs[i];
7331                 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7332                         continue;
7333
7334                 id = btf__find_by_name_kind(obj->btf_vmlinux, ext->name,
7335                                             BTF_KIND_VAR);
7336                 if (id <= 0) {
7337                         pr_warn("extern (ksym) '%s': failed to find BTF ID in vmlinux BTF.\n",
7338                                 ext->name);
7339                         return -ESRCH;
7340                 }
7341
7342                 /* find local type_id */
7343                 local_type_id = ext->ksym.type_id;
7344
7345                 /* find target type_id */
7346                 targ_var = btf__type_by_id(obj->btf_vmlinux, id);
7347                 targ_var_name = btf__name_by_offset(obj->btf_vmlinux,
7348                                                     targ_var->name_off);
7349                 targ_type = skip_mods_and_typedefs(obj->btf_vmlinux,
7350                                                    targ_var->type,
7351                                                    &targ_type_id);
7352
7353                 ret = bpf_core_types_are_compat(obj->btf, local_type_id,
7354                                                 obj->btf_vmlinux, targ_type_id);
7355                 if (ret <= 0) {
7356                         const struct btf_type *local_type;
7357                         const char *targ_name, *local_name;
7358
7359                         local_type = btf__type_by_id(obj->btf, local_type_id);
7360                         local_name = btf__name_by_offset(obj->btf,
7361                                                          local_type->name_off);
7362                         targ_name = btf__name_by_offset(obj->btf_vmlinux,
7363                                                         targ_type->name_off);
7364
7365                         pr_warn("extern (ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7366                                 ext->name, local_type_id,
7367                                 btf_kind_str(local_type), local_name, targ_type_id,
7368                                 btf_kind_str(targ_type), targ_name);
7369                         return -EINVAL;
7370                 }
7371
7372                 ext->is_set = true;
7373                 ext->ksym.vmlinux_btf_id = id;
7374                 pr_debug("extern (ksym) '%s': resolved to [%d] %s %s\n",
7375                          ext->name, id, btf_kind_str(targ_var), targ_var_name);
7376         }
7377         return 0;
7378 }
7379
7380 static int bpf_object__resolve_externs(struct bpf_object *obj,
7381                                        const char *extra_kconfig)
7382 {
7383         bool need_config = false, need_kallsyms = false;
7384         bool need_vmlinux_btf = false;
7385         struct extern_desc *ext;
7386         void *kcfg_data = NULL;
7387         int err, i;
7388
7389         if (obj->nr_extern == 0)
7390                 return 0;
7391
7392         if (obj->kconfig_map_idx >= 0)
7393                 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7394
7395         for (i = 0; i < obj->nr_extern; i++) {
7396                 ext = &obj->externs[i];
7397
7398                 if (ext->type == EXT_KCFG &&
7399                     strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7400                         void *ext_val = kcfg_data + ext->kcfg.data_off;
7401                         __u32 kver = get_kernel_version();
7402
7403                         if (!kver) {
7404                                 pr_warn("failed to get kernel version\n");
7405                                 return -EINVAL;
7406                         }
7407                         err = set_kcfg_value_num(ext, ext_val, kver);
7408                         if (err)
7409                                 return err;
7410                         pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver);
7411                 } else if (ext->type == EXT_KCFG &&
7412                            strncmp(ext->name, "CONFIG_", 7) == 0) {
7413                         need_config = true;
7414                 } else if (ext->type == EXT_KSYM) {
7415                         if (ext->ksym.type_id)
7416                                 need_vmlinux_btf = true;
7417                         else
7418                                 need_kallsyms = true;
7419                 } else {
7420                         pr_warn("unrecognized extern '%s'\n", ext->name);
7421                         return -EINVAL;
7422                 }
7423         }
7424         if (need_config && extra_kconfig) {
7425                 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7426                 if (err)
7427                         return -EINVAL;
7428                 need_config = false;
7429                 for (i = 0; i < obj->nr_extern; i++) {
7430                         ext = &obj->externs[i];
7431                         if (ext->type == EXT_KCFG && !ext->is_set) {
7432                                 need_config = true;
7433                                 break;
7434                         }
7435                 }
7436         }
7437         if (need_config) {
7438                 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7439                 if (err)
7440                         return -EINVAL;
7441         }
7442         if (need_kallsyms) {
7443                 err = bpf_object__read_kallsyms_file(obj);
7444                 if (err)
7445                         return -EINVAL;
7446         }
7447         if (need_vmlinux_btf) {
7448                 err = bpf_object__resolve_ksyms_btf_id(obj);
7449                 if (err)
7450                         return -EINVAL;
7451         }
7452         for (i = 0; i < obj->nr_extern; i++) {
7453                 ext = &obj->externs[i];
7454
7455                 if (!ext->is_set && !ext->is_weak) {
7456                         pr_warn("extern %s (strong) not resolved\n", ext->name);
7457                         return -ESRCH;
7458                 } else if (!ext->is_set) {
7459                         pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
7460                                  ext->name);
7461                 }
7462         }
7463
7464         return 0;
7465 }
7466
7467 int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
7468 {
7469         struct bpf_object *obj;
7470         int err, i;
7471
7472         if (!attr)
7473                 return -EINVAL;
7474         obj = attr->obj;
7475         if (!obj)
7476                 return -EINVAL;
7477
7478         if (obj->loaded) {
7479                 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7480                 return -EINVAL;
7481         }
7482
7483         err = bpf_object__probe_loading(obj);
7484         err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7485         err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7486         err = err ? : bpf_object__sanitize_and_load_btf(obj);
7487         err = err ? : bpf_object__sanitize_maps(obj);
7488         err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7489         err = err ? : bpf_object__create_maps(obj);
7490         err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
7491         err = err ? : bpf_object__load_progs(obj, attr->log_level);
7492
7493         /* clean up module BTFs */
7494         for (i = 0; i < obj->btf_module_cnt; i++) {
7495                 close(obj->btf_modules[i].fd);
7496                 btf__free(obj->btf_modules[i].btf);
7497                 free(obj->btf_modules[i].name);
7498         }
7499         free(obj->btf_modules);
7500
7501         /* clean up vmlinux BTF */
7502         btf__free(obj->btf_vmlinux);
7503         obj->btf_vmlinux = NULL;
7504
7505         obj->loaded = true; /* doesn't matter if successfully or not */
7506
7507         if (err)
7508                 goto out;
7509
7510         return 0;
7511 out:
7512         /* unpin any maps that were auto-pinned during load */
7513         for (i = 0; i < obj->nr_maps; i++)
7514                 if (obj->maps[i].pinned && !obj->maps[i].reused)
7515                         bpf_map__unpin(&obj->maps[i], NULL);
7516
7517         bpf_object__unload(obj);
7518         pr_warn("failed to load object '%s'\n", obj->path);
7519         return err;
7520 }
7521
7522 int bpf_object__load(struct bpf_object *obj)
7523 {
7524         struct bpf_object_load_attr attr = {
7525                 .obj = obj,
7526         };
7527
7528         return bpf_object__load_xattr(&attr);
7529 }
7530
7531 static int make_parent_dir(const char *path)
7532 {
7533         char *cp, errmsg[STRERR_BUFSIZE];
7534         char *dname, *dir;
7535         int err = 0;
7536
7537         dname = strdup(path);
7538         if (dname == NULL)
7539                 return -ENOMEM;
7540
7541         dir = dirname(dname);
7542         if (mkdir(dir, 0700) && errno != EEXIST)
7543                 err = -errno;
7544
7545         free(dname);
7546         if (err) {
7547                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7548                 pr_warn("failed to mkdir %s: %s\n", path, cp);
7549         }
7550         return err;
7551 }
7552
7553 static int check_path(const char *path)
7554 {
7555         char *cp, errmsg[STRERR_BUFSIZE];
7556         struct statfs st_fs;
7557         char *dname, *dir;
7558         int err = 0;
7559
7560         if (path == NULL)
7561                 return -EINVAL;
7562
7563         dname = strdup(path);
7564         if (dname == NULL)
7565                 return -ENOMEM;
7566
7567         dir = dirname(dname);
7568         if (statfs(dir, &st_fs)) {
7569                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7570                 pr_warn("failed to statfs %s: %s\n", dir, cp);
7571                 err = -errno;
7572         }
7573         free(dname);
7574
7575         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7576                 pr_warn("specified path %s is not on BPF FS\n", path);
7577                 err = -EINVAL;
7578         }
7579
7580         return err;
7581 }
7582
7583 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
7584                               int instance)
7585 {
7586         char *cp, errmsg[STRERR_BUFSIZE];
7587         int err;
7588
7589         err = make_parent_dir(path);
7590         if (err)
7591                 return err;
7592
7593         err = check_path(path);
7594         if (err)
7595                 return err;
7596
7597         if (prog == NULL) {
7598                 pr_warn("invalid program pointer\n");
7599                 return -EINVAL;
7600         }
7601
7602         if (instance < 0 || instance >= prog->instances.nr) {
7603                 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7604                         instance, prog->name, prog->instances.nr);
7605                 return -EINVAL;
7606         }
7607
7608         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
7609                 err = -errno;
7610                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7611                 pr_warn("failed to pin program: %s\n", cp);
7612                 return err;
7613         }
7614         pr_debug("pinned program '%s'\n", path);
7615
7616         return 0;
7617 }
7618
7619 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
7620                                 int instance)
7621 {
7622         int err;
7623
7624         err = check_path(path);
7625         if (err)
7626                 return err;
7627
7628         if (prog == NULL) {
7629                 pr_warn("invalid program pointer\n");
7630                 return -EINVAL;
7631         }
7632
7633         if (instance < 0 || instance >= prog->instances.nr) {
7634                 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7635                         instance, prog->name, prog->instances.nr);
7636                 return -EINVAL;
7637         }
7638
7639         err = unlink(path);
7640         if (err != 0)
7641                 return -errno;
7642         pr_debug("unpinned program '%s'\n", path);
7643
7644         return 0;
7645 }
7646
7647 int bpf_program__pin(struct bpf_program *prog, const char *path)
7648 {
7649         int i, err;
7650
7651         err = make_parent_dir(path);
7652         if (err)
7653                 return err;
7654
7655         err = check_path(path);
7656         if (err)
7657                 return err;
7658
7659         if (prog == NULL) {
7660                 pr_warn("invalid program pointer\n");
7661                 return -EINVAL;
7662         }
7663
7664         if (prog->instances.nr <= 0) {
7665                 pr_warn("no instances of prog %s to pin\n", prog->name);
7666                 return -EINVAL;
7667         }
7668
7669         if (prog->instances.nr == 1) {
7670                 /* don't create subdirs when pinning single instance */
7671                 return bpf_program__pin_instance(prog, path, 0);
7672         }
7673
7674         for (i = 0; i < prog->instances.nr; i++) {
7675                 char buf[PATH_MAX];
7676                 int len;
7677
7678                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7679                 if (len < 0) {
7680                         err = -EINVAL;
7681                         goto err_unpin;
7682                 } else if (len >= PATH_MAX) {
7683                         err = -ENAMETOOLONG;
7684                         goto err_unpin;
7685                 }
7686
7687                 err = bpf_program__pin_instance(prog, buf, i);
7688                 if (err)
7689                         goto err_unpin;
7690         }
7691
7692         return 0;
7693
7694 err_unpin:
7695         for (i = i - 1; i >= 0; i--) {
7696                 char buf[PATH_MAX];
7697                 int len;
7698
7699                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7700                 if (len < 0)
7701                         continue;
7702                 else if (len >= PATH_MAX)
7703                         continue;
7704
7705                 bpf_program__unpin_instance(prog, buf, i);
7706         }
7707
7708         rmdir(path);
7709
7710         return err;
7711 }
7712
7713 int bpf_program__unpin(struct bpf_program *prog, const char *path)
7714 {
7715         int i, err;
7716
7717         err = check_path(path);
7718         if (err)
7719                 return err;
7720
7721         if (prog == NULL) {
7722                 pr_warn("invalid program pointer\n");
7723                 return -EINVAL;
7724         }
7725
7726         if (prog->instances.nr <= 0) {
7727                 pr_warn("no instances of prog %s to pin\n", prog->name);
7728                 return -EINVAL;
7729         }
7730
7731         if (prog->instances.nr == 1) {
7732                 /* don't create subdirs when pinning single instance */
7733                 return bpf_program__unpin_instance(prog, path, 0);
7734         }
7735
7736         for (i = 0; i < prog->instances.nr; i++) {
7737                 char buf[PATH_MAX];
7738                 int len;
7739
7740                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7741                 if (len < 0)
7742                         return -EINVAL;
7743                 else if (len >= PATH_MAX)
7744                         return -ENAMETOOLONG;
7745
7746                 err = bpf_program__unpin_instance(prog, buf, i);
7747                 if (err)
7748                         return err;
7749         }
7750
7751         err = rmdir(path);
7752         if (err)
7753                 return -errno;
7754
7755         return 0;
7756 }
7757
7758 int bpf_map__pin(struct bpf_map *map, const char *path)
7759 {
7760         char *cp, errmsg[STRERR_BUFSIZE];
7761         int err;
7762
7763         if (map == NULL) {
7764                 pr_warn("invalid map pointer\n");
7765                 return -EINVAL;
7766         }
7767
7768         if (map->pin_path) {
7769                 if (path && strcmp(path, map->pin_path)) {
7770                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7771                                 bpf_map__name(map), map->pin_path, path);
7772                         return -EINVAL;
7773                 } else if (map->pinned) {
7774                         pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7775                                  bpf_map__name(map), map->pin_path);
7776                         return 0;
7777                 }
7778         } else {
7779                 if (!path) {
7780                         pr_warn("missing a path to pin map '%s' at\n",
7781                                 bpf_map__name(map));
7782                         return -EINVAL;
7783                 } else if (map->pinned) {
7784                         pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7785                         return -EEXIST;
7786                 }
7787
7788                 map->pin_path = strdup(path);
7789                 if (!map->pin_path) {
7790                         err = -errno;
7791                         goto out_err;
7792                 }
7793         }
7794
7795         err = make_parent_dir(map->pin_path);
7796         if (err)
7797                 return err;
7798
7799         err = check_path(map->pin_path);
7800         if (err)
7801                 return err;
7802
7803         if (bpf_obj_pin(map->fd, map->pin_path)) {
7804                 err = -errno;
7805                 goto out_err;
7806         }
7807
7808         map->pinned = true;
7809         pr_debug("pinned map '%s'\n", map->pin_path);
7810
7811         return 0;
7812
7813 out_err:
7814         cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7815         pr_warn("failed to pin map: %s\n", cp);
7816         return err;
7817 }
7818
7819 int bpf_map__unpin(struct bpf_map *map, const char *path)
7820 {
7821         int err;
7822
7823         if (map == NULL) {
7824                 pr_warn("invalid map pointer\n");
7825                 return -EINVAL;
7826         }
7827
7828         if (map->pin_path) {
7829                 if (path && strcmp(path, map->pin_path)) {
7830                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7831                                 bpf_map__name(map), map->pin_path, path);
7832                         return -EINVAL;
7833                 }
7834                 path = map->pin_path;
7835         } else if (!path) {
7836                 pr_warn("no path to unpin map '%s' from\n",
7837                         bpf_map__name(map));
7838                 return -EINVAL;
7839         }
7840
7841         err = check_path(path);
7842         if (err)
7843                 return err;
7844
7845         err = unlink(path);
7846         if (err != 0)
7847                 return -errno;
7848
7849         map->pinned = false;
7850         pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7851
7852         return 0;
7853 }
7854
7855 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7856 {
7857         char *new = NULL;
7858
7859         if (path) {
7860                 new = strdup(path);
7861                 if (!new)
7862                         return -errno;
7863         }
7864
7865         free(map->pin_path);
7866         map->pin_path = new;
7867         return 0;
7868 }
7869
7870 const char *bpf_map__get_pin_path(const struct bpf_map *map)
7871 {
7872         return map->pin_path;
7873 }
7874
7875 bool bpf_map__is_pinned(const struct bpf_map *map)
7876 {
7877         return map->pinned;
7878 }
7879
7880 static void sanitize_pin_path(char *s)
7881 {
7882         /* bpffs disallows periods in path names */
7883         while (*s) {
7884                 if (*s == '.')
7885                         *s = '_';
7886                 s++;
7887         }
7888 }
7889
7890 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
7891 {
7892         struct bpf_map *map;
7893         int err;
7894
7895         if (!obj)
7896                 return -ENOENT;
7897
7898         if (!obj->loaded) {
7899                 pr_warn("object not yet loaded; load it first\n");
7900                 return -ENOENT;
7901         }
7902
7903         bpf_object__for_each_map(map, obj) {
7904                 char *pin_path = NULL;
7905                 char buf[PATH_MAX];
7906
7907                 if (path) {
7908                         int len;
7909
7910                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
7911                                        bpf_map__name(map));
7912                         if (len < 0) {
7913                                 err = -EINVAL;
7914                                 goto err_unpin_maps;
7915                         } else if (len >= PATH_MAX) {
7916                                 err = -ENAMETOOLONG;
7917                                 goto err_unpin_maps;
7918                         }
7919                         sanitize_pin_path(buf);
7920                         pin_path = buf;
7921                 } else if (!map->pin_path) {
7922                         continue;
7923                 }
7924
7925                 err = bpf_map__pin(map, pin_path);
7926                 if (err)
7927                         goto err_unpin_maps;
7928         }
7929
7930         return 0;
7931
7932 err_unpin_maps:
7933         while ((map = bpf_map__prev(map, obj))) {
7934                 if (!map->pin_path)
7935                         continue;
7936
7937                 bpf_map__unpin(map, NULL);
7938         }
7939
7940         return err;
7941 }
7942
7943 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
7944 {
7945         struct bpf_map *map;
7946         int err;
7947
7948         if (!obj)
7949                 return -ENOENT;
7950
7951         bpf_object__for_each_map(map, obj) {
7952                 char *pin_path = NULL;
7953                 char buf[PATH_MAX];
7954
7955                 if (path) {
7956                         int len;
7957
7958                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
7959                                        bpf_map__name(map));
7960                         if (len < 0)
7961                                 return -EINVAL;
7962                         else if (len >= PATH_MAX)
7963                                 return -ENAMETOOLONG;
7964                         sanitize_pin_path(buf);
7965                         pin_path = buf;
7966                 } else if (!map->pin_path) {
7967                         continue;
7968                 }
7969
7970                 err = bpf_map__unpin(map, pin_path);
7971                 if (err)
7972                         return err;
7973         }
7974
7975         return 0;
7976 }
7977
7978 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
7979 {
7980         struct bpf_program *prog;
7981         int err;
7982
7983         if (!obj)
7984                 return -ENOENT;
7985
7986         if (!obj->loaded) {
7987                 pr_warn("object not yet loaded; load it first\n");
7988                 return -ENOENT;
7989         }
7990
7991         bpf_object__for_each_program(prog, obj) {
7992                 char buf[PATH_MAX];
7993                 int len;
7994
7995                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7996                                prog->pin_name);
7997                 if (len < 0) {
7998                         err = -EINVAL;
7999                         goto err_unpin_programs;
8000                 } else if (len >= PATH_MAX) {
8001                         err = -ENAMETOOLONG;
8002                         goto err_unpin_programs;
8003                 }
8004
8005                 err = bpf_program__pin(prog, buf);
8006                 if (err)
8007                         goto err_unpin_programs;
8008         }
8009
8010         return 0;
8011
8012 err_unpin_programs:
8013         while ((prog = bpf_program__prev(prog, obj))) {
8014                 char buf[PATH_MAX];
8015                 int len;
8016
8017                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8018                                prog->pin_name);
8019                 if (len < 0)
8020                         continue;
8021                 else if (len >= PATH_MAX)
8022                         continue;
8023
8024                 bpf_program__unpin(prog, buf);
8025         }
8026
8027         return err;
8028 }
8029
8030 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8031 {
8032         struct bpf_program *prog;
8033         int err;
8034
8035         if (!obj)
8036                 return -ENOENT;
8037
8038         bpf_object__for_each_program(prog, obj) {
8039                 char buf[PATH_MAX];
8040                 int len;
8041
8042                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8043                                prog->pin_name);
8044                 if (len < 0)
8045                         return -EINVAL;
8046                 else if (len >= PATH_MAX)
8047                         return -ENAMETOOLONG;
8048
8049                 err = bpf_program__unpin(prog, buf);
8050                 if (err)
8051                         return err;
8052         }
8053
8054         return 0;
8055 }
8056
8057 int bpf_object__pin(struct bpf_object *obj, const char *path)
8058 {
8059         int err;
8060
8061         err = bpf_object__pin_maps(obj, path);
8062         if (err)
8063                 return err;
8064
8065         err = bpf_object__pin_programs(obj, path);
8066         if (err) {
8067                 bpf_object__unpin_maps(obj, path);
8068                 return err;
8069         }
8070
8071         return 0;
8072 }
8073
8074 static void bpf_map__destroy(struct bpf_map *map)
8075 {
8076         if (map->clear_priv)
8077                 map->clear_priv(map, map->priv);
8078         map->priv = NULL;
8079         map->clear_priv = NULL;
8080
8081         if (map->inner_map) {
8082                 bpf_map__destroy(map->inner_map);
8083                 zfree(&map->inner_map);
8084         }
8085
8086         zfree(&map->init_slots);
8087         map->init_slots_sz = 0;
8088
8089         if (map->mmaped) {
8090                 munmap(map->mmaped, bpf_map_mmap_sz(map));
8091                 map->mmaped = NULL;
8092         }
8093
8094         if (map->st_ops) {
8095                 zfree(&map->st_ops->data);
8096                 zfree(&map->st_ops->progs);
8097                 zfree(&map->st_ops->kern_func_off);
8098                 zfree(&map->st_ops);
8099         }
8100
8101         zfree(&map->name);
8102         zfree(&map->pin_path);
8103
8104         if (map->fd >= 0)
8105                 zclose(map->fd);
8106 }
8107
8108 void bpf_object__close(struct bpf_object *obj)
8109 {
8110         size_t i;
8111
8112         if (IS_ERR_OR_NULL(obj))
8113                 return;
8114
8115         if (obj->clear_priv)
8116                 obj->clear_priv(obj, obj->priv);
8117
8118         bpf_object__elf_finish(obj);
8119         bpf_object__unload(obj);
8120         btf__free(obj->btf);
8121         btf_ext__free(obj->btf_ext);
8122
8123         for (i = 0; i < obj->nr_maps; i++)
8124                 bpf_map__destroy(&obj->maps[i]);
8125
8126         zfree(&obj->kconfig);
8127         zfree(&obj->externs);
8128         obj->nr_extern = 0;
8129
8130         zfree(&obj->maps);
8131         obj->nr_maps = 0;
8132
8133         if (obj->programs && obj->nr_programs) {
8134                 for (i = 0; i < obj->nr_programs; i++)
8135                         bpf_program__exit(&obj->programs[i]);
8136         }
8137         zfree(&obj->programs);
8138
8139         list_del(&obj->list);
8140         free(obj);
8141 }
8142
8143 struct bpf_object *
8144 bpf_object__next(struct bpf_object *prev)
8145 {
8146         struct bpf_object *next;
8147
8148         if (!prev)
8149                 next = list_first_entry(&bpf_objects_list,
8150                                         struct bpf_object,
8151                                         list);
8152         else
8153                 next = list_next_entry(prev, list);
8154
8155         /* Empty list is noticed here so don't need checking on entry. */
8156         if (&next->list == &bpf_objects_list)
8157                 return NULL;
8158
8159         return next;
8160 }
8161
8162 const char *bpf_object__name(const struct bpf_object *obj)
8163 {
8164         return obj ? obj->name : ERR_PTR(-EINVAL);
8165 }
8166
8167 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8168 {
8169         return obj ? obj->kern_version : 0;
8170 }
8171
8172 struct btf *bpf_object__btf(const struct bpf_object *obj)
8173 {
8174         return obj ? obj->btf : NULL;
8175 }
8176
8177 int bpf_object__btf_fd(const struct bpf_object *obj)
8178 {
8179         return obj->btf ? btf__fd(obj->btf) : -1;
8180 }
8181
8182 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
8183                          bpf_object_clear_priv_t clear_priv)
8184 {
8185         if (obj->priv && obj->clear_priv)
8186                 obj->clear_priv(obj, obj->priv);
8187
8188         obj->priv = priv;
8189         obj->clear_priv = clear_priv;
8190         return 0;
8191 }
8192
8193 void *bpf_object__priv(const struct bpf_object *obj)
8194 {
8195         return obj ? obj->priv : ERR_PTR(-EINVAL);
8196 }
8197
8198 static struct bpf_program *
8199 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8200                     bool forward)
8201 {
8202         size_t nr_programs = obj->nr_programs;
8203         ssize_t idx;
8204
8205         if (!nr_programs)
8206                 return NULL;
8207
8208         if (!p)
8209                 /* Iter from the beginning */
8210                 return forward ? &obj->programs[0] :
8211                         &obj->programs[nr_programs - 1];
8212
8213         if (p->obj != obj) {
8214                 pr_warn("error: program handler doesn't match object\n");
8215                 return NULL;
8216         }
8217
8218         idx = (p - obj->programs) + (forward ? 1 : -1);
8219         if (idx >= obj->nr_programs || idx < 0)
8220                 return NULL;
8221         return &obj->programs[idx];
8222 }
8223
8224 struct bpf_program *
8225 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
8226 {
8227         struct bpf_program *prog = prev;
8228
8229         do {
8230                 prog = __bpf_program__iter(prog, obj, true);
8231         } while (prog && prog_is_subprog(obj, prog));
8232
8233         return prog;
8234 }
8235
8236 struct bpf_program *
8237 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
8238 {
8239         struct bpf_program *prog = next;
8240
8241         do {
8242                 prog = __bpf_program__iter(prog, obj, false);
8243         } while (prog && prog_is_subprog(obj, prog));
8244
8245         return prog;
8246 }
8247
8248 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
8249                           bpf_program_clear_priv_t clear_priv)
8250 {
8251         if (prog->priv && prog->clear_priv)
8252                 prog->clear_priv(prog, prog->priv);
8253
8254         prog->priv = priv;
8255         prog->clear_priv = clear_priv;
8256         return 0;
8257 }
8258
8259 void *bpf_program__priv(const struct bpf_program *prog)
8260 {
8261         return prog ? prog->priv : ERR_PTR(-EINVAL);
8262 }
8263
8264 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8265 {
8266         prog->prog_ifindex = ifindex;
8267 }
8268
8269 const char *bpf_program__name(const struct bpf_program *prog)
8270 {
8271         return prog->name;
8272 }
8273
8274 const char *bpf_program__section_name(const struct bpf_program *prog)
8275 {
8276         return prog->sec_name;
8277 }
8278
8279 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
8280 {
8281         const char *title;
8282
8283         title = prog->sec_name;
8284         if (needs_copy) {
8285                 title = strdup(title);
8286                 if (!title) {
8287                         pr_warn("failed to strdup program title\n");
8288                         return ERR_PTR(-ENOMEM);
8289                 }
8290         }
8291
8292         return title;
8293 }
8294
8295 bool bpf_program__autoload(const struct bpf_program *prog)
8296 {
8297         return prog->load;
8298 }
8299
8300 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8301 {
8302         if (prog->obj->loaded)
8303                 return -EINVAL;
8304
8305         prog->load = autoload;
8306         return 0;
8307 }
8308
8309 int bpf_program__fd(const struct bpf_program *prog)
8310 {
8311         return bpf_program__nth_fd(prog, 0);
8312 }
8313
8314 size_t bpf_program__size(const struct bpf_program *prog)
8315 {
8316         return prog->insns_cnt * BPF_INSN_SZ;
8317 }
8318
8319 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
8320                           bpf_program_prep_t prep)
8321 {
8322         int *instances_fds;
8323
8324         if (nr_instances <= 0 || !prep)
8325                 return -EINVAL;
8326
8327         if (prog->instances.nr > 0 || prog->instances.fds) {
8328                 pr_warn("Can't set pre-processor after loading\n");
8329                 return -EINVAL;
8330         }
8331
8332         instances_fds = malloc(sizeof(int) * nr_instances);
8333         if (!instances_fds) {
8334                 pr_warn("alloc memory failed for fds\n");
8335                 return -ENOMEM;
8336         }
8337
8338         /* fill all fd with -1 */
8339         memset(instances_fds, -1, sizeof(int) * nr_instances);
8340
8341         prog->instances.nr = nr_instances;
8342         prog->instances.fds = instances_fds;
8343         prog->preprocessor = prep;
8344         return 0;
8345 }
8346
8347 int bpf_program__nth_fd(const struct bpf_program *prog, int n)
8348 {
8349         int fd;
8350
8351         if (!prog)
8352                 return -EINVAL;
8353
8354         if (n >= prog->instances.nr || n < 0) {
8355                 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
8356                         n, prog->name, prog->instances.nr);
8357                 return -EINVAL;
8358         }
8359
8360         fd = prog->instances.fds[n];
8361         if (fd < 0) {
8362                 pr_warn("%dth instance of program '%s' is invalid\n",
8363                         n, prog->name);
8364                 return -ENOENT;
8365         }
8366
8367         return fd;
8368 }
8369
8370 enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
8371 {
8372         return prog->type;
8373 }
8374
8375 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8376 {
8377         prog->type = type;
8378 }
8379
8380 static bool bpf_program__is_type(const struct bpf_program *prog,
8381                                  enum bpf_prog_type type)
8382 {
8383         return prog ? (prog->type == type) : false;
8384 }
8385
8386 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                           \
8387 int bpf_program__set_##NAME(struct bpf_program *prog)           \
8388 {                                                               \
8389         if (!prog)                                              \
8390                 return -EINVAL;                                 \
8391         bpf_program__set_type(prog, TYPE);                      \
8392         return 0;                                               \
8393 }                                                               \
8394                                                                 \
8395 bool bpf_program__is_##NAME(const struct bpf_program *prog)     \
8396 {                                                               \
8397         return bpf_program__is_type(prog, TYPE);                \
8398 }                                                               \
8399
8400 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
8401 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
8402 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
8403 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
8404 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
8405 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
8406 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
8407 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
8408 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
8409 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
8410 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
8411 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
8412 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
8413
8414 enum bpf_attach_type
8415 bpf_program__get_expected_attach_type(struct bpf_program *prog)
8416 {
8417         return prog->expected_attach_type;
8418 }
8419
8420 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
8421                                            enum bpf_attach_type type)
8422 {
8423         prog->expected_attach_type = type;
8424 }
8425
8426 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional,           \
8427                           attachable, attach_btf)                           \
8428         {                                                                   \
8429                 .sec = string,                                              \
8430                 .len = sizeof(string) - 1,                                  \
8431                 .prog_type = ptype,                                         \
8432                 .expected_attach_type = eatype,                             \
8433                 .is_exp_attach_type_optional = eatype_optional,             \
8434                 .is_attachable = attachable,                                \
8435                 .is_attach_btf = attach_btf,                                \
8436         }
8437
8438 /* Programs that can NOT be attached. */
8439 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
8440
8441 /* Programs that can be attached. */
8442 #define BPF_APROG_SEC(string, ptype, atype) \
8443         BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0)
8444
8445 /* Programs that must specify expected attach type at load time. */
8446 #define BPF_EAPROG_SEC(string, ptype, eatype) \
8447         BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0)
8448
8449 /* Programs that use BTF to identify attach point */
8450 #define BPF_PROG_BTF(string, ptype, eatype) \
8451         BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1)
8452
8453 /* Programs that can be attached but attach type can't be identified by section
8454  * name. Kept for backward compatibility.
8455  */
8456 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
8457
8458 #define SEC_DEF(sec_pfx, ptype, ...) {                                      \
8459         .sec = sec_pfx,                                                     \
8460         .len = sizeof(sec_pfx) - 1,                                         \
8461         .prog_type = BPF_PROG_TYPE_##ptype,                                 \
8462         __VA_ARGS__                                                         \
8463 }
8464
8465 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
8466                                       struct bpf_program *prog);
8467 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
8468                                   struct bpf_program *prog);
8469 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
8470                                       struct bpf_program *prog);
8471 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
8472                                      struct bpf_program *prog);
8473 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
8474                                    struct bpf_program *prog);
8475 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
8476                                     struct bpf_program *prog);
8477
8478 static const struct bpf_sec_def section_defs[] = {
8479         BPF_PROG_SEC("socket",                  BPF_PROG_TYPE_SOCKET_FILTER),
8480         BPF_PROG_SEC("sk_reuseport",            BPF_PROG_TYPE_SK_REUSEPORT),
8481         SEC_DEF("kprobe/", KPROBE,
8482                 .attach_fn = attach_kprobe),
8483         BPF_PROG_SEC("uprobe/",                 BPF_PROG_TYPE_KPROBE),
8484         SEC_DEF("kretprobe/", KPROBE,
8485                 .attach_fn = attach_kprobe),
8486         BPF_PROG_SEC("uretprobe/",              BPF_PROG_TYPE_KPROBE),
8487         BPF_PROG_SEC("classifier",              BPF_PROG_TYPE_SCHED_CLS),
8488         BPF_PROG_SEC("action",                  BPF_PROG_TYPE_SCHED_ACT),
8489         SEC_DEF("tracepoint/", TRACEPOINT,
8490                 .attach_fn = attach_tp),
8491         SEC_DEF("tp/", TRACEPOINT,
8492                 .attach_fn = attach_tp),
8493         SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
8494                 .attach_fn = attach_raw_tp),
8495         SEC_DEF("raw_tp/", RAW_TRACEPOINT,
8496                 .attach_fn = attach_raw_tp),
8497         SEC_DEF("tp_btf/", TRACING,
8498                 .expected_attach_type = BPF_TRACE_RAW_TP,
8499                 .is_attach_btf = true,
8500                 .attach_fn = attach_trace),
8501         SEC_DEF("fentry/", TRACING,
8502                 .expected_attach_type = BPF_TRACE_FENTRY,
8503                 .is_attach_btf = true,
8504                 .attach_fn = attach_trace),
8505         SEC_DEF("fmod_ret/", TRACING,
8506                 .expected_attach_type = BPF_MODIFY_RETURN,
8507                 .is_attach_btf = true,
8508                 .attach_fn = attach_trace),
8509         SEC_DEF("fexit/", TRACING,
8510                 .expected_attach_type = BPF_TRACE_FEXIT,
8511                 .is_attach_btf = true,
8512                 .attach_fn = attach_trace),
8513         SEC_DEF("fentry.s/", TRACING,
8514                 .expected_attach_type = BPF_TRACE_FENTRY,
8515                 .is_attach_btf = true,
8516                 .is_sleepable = true,
8517                 .attach_fn = attach_trace),
8518         SEC_DEF("fmod_ret.s/", TRACING,
8519                 .expected_attach_type = BPF_MODIFY_RETURN,
8520                 .is_attach_btf = true,
8521                 .is_sleepable = true,
8522                 .attach_fn = attach_trace),
8523         SEC_DEF("fexit.s/", TRACING,
8524                 .expected_attach_type = BPF_TRACE_FEXIT,
8525                 .is_attach_btf = true,
8526                 .is_sleepable = true,
8527                 .attach_fn = attach_trace),
8528         SEC_DEF("freplace/", EXT,
8529                 .is_attach_btf = true,
8530                 .attach_fn = attach_trace),
8531         SEC_DEF("lsm/", LSM,
8532                 .is_attach_btf = true,
8533                 .expected_attach_type = BPF_LSM_MAC,
8534                 .attach_fn = attach_lsm),
8535         SEC_DEF("lsm.s/", LSM,
8536                 .is_attach_btf = true,
8537                 .is_sleepable = true,
8538                 .expected_attach_type = BPF_LSM_MAC,
8539                 .attach_fn = attach_lsm),
8540         SEC_DEF("iter/", TRACING,
8541                 .expected_attach_type = BPF_TRACE_ITER,
8542                 .is_attach_btf = true,
8543                 .attach_fn = attach_iter),
8544         BPF_EAPROG_SEC("xdp_devmap/",           BPF_PROG_TYPE_XDP,
8545                                                 BPF_XDP_DEVMAP),
8546         BPF_EAPROG_SEC("xdp_cpumap/",           BPF_PROG_TYPE_XDP,
8547                                                 BPF_XDP_CPUMAP),
8548         BPF_APROG_SEC("xdp",                    BPF_PROG_TYPE_XDP,
8549                                                 BPF_XDP),
8550         BPF_PROG_SEC("perf_event",              BPF_PROG_TYPE_PERF_EVENT),
8551         BPF_PROG_SEC("lwt_in",                  BPF_PROG_TYPE_LWT_IN),
8552         BPF_PROG_SEC("lwt_out",                 BPF_PROG_TYPE_LWT_OUT),
8553         BPF_PROG_SEC("lwt_xmit",                BPF_PROG_TYPE_LWT_XMIT),
8554         BPF_PROG_SEC("lwt_seg6local",           BPF_PROG_TYPE_LWT_SEG6LOCAL),
8555         BPF_APROG_SEC("cgroup_skb/ingress",     BPF_PROG_TYPE_CGROUP_SKB,
8556                                                 BPF_CGROUP_INET_INGRESS),
8557         BPF_APROG_SEC("cgroup_skb/egress",      BPF_PROG_TYPE_CGROUP_SKB,
8558                                                 BPF_CGROUP_INET_EGRESS),
8559         BPF_APROG_COMPAT("cgroup/skb",          BPF_PROG_TYPE_CGROUP_SKB),
8560         BPF_EAPROG_SEC("cgroup/sock_create",    BPF_PROG_TYPE_CGROUP_SOCK,
8561                                                 BPF_CGROUP_INET_SOCK_CREATE),
8562         BPF_EAPROG_SEC("cgroup/sock_release",   BPF_PROG_TYPE_CGROUP_SOCK,
8563                                                 BPF_CGROUP_INET_SOCK_RELEASE),
8564         BPF_APROG_SEC("cgroup/sock",            BPF_PROG_TYPE_CGROUP_SOCK,
8565                                                 BPF_CGROUP_INET_SOCK_CREATE),
8566         BPF_EAPROG_SEC("cgroup/post_bind4",     BPF_PROG_TYPE_CGROUP_SOCK,
8567                                                 BPF_CGROUP_INET4_POST_BIND),
8568         BPF_EAPROG_SEC("cgroup/post_bind6",     BPF_PROG_TYPE_CGROUP_SOCK,
8569                                                 BPF_CGROUP_INET6_POST_BIND),
8570         BPF_APROG_SEC("cgroup/dev",             BPF_PROG_TYPE_CGROUP_DEVICE,
8571                                                 BPF_CGROUP_DEVICE),
8572         BPF_APROG_SEC("sockops",                BPF_PROG_TYPE_SOCK_OPS,
8573                                                 BPF_CGROUP_SOCK_OPS),
8574         BPF_APROG_SEC("sk_skb/stream_parser",   BPF_PROG_TYPE_SK_SKB,
8575                                                 BPF_SK_SKB_STREAM_PARSER),
8576         BPF_APROG_SEC("sk_skb/stream_verdict",  BPF_PROG_TYPE_SK_SKB,
8577                                                 BPF_SK_SKB_STREAM_VERDICT),
8578         BPF_APROG_COMPAT("sk_skb",              BPF_PROG_TYPE_SK_SKB),
8579         BPF_APROG_SEC("sk_msg",                 BPF_PROG_TYPE_SK_MSG,
8580                                                 BPF_SK_MSG_VERDICT),
8581         BPF_APROG_SEC("lirc_mode2",             BPF_PROG_TYPE_LIRC_MODE2,
8582                                                 BPF_LIRC_MODE2),
8583         BPF_APROG_SEC("flow_dissector",         BPF_PROG_TYPE_FLOW_DISSECTOR,
8584                                                 BPF_FLOW_DISSECTOR),
8585         BPF_EAPROG_SEC("cgroup/bind4",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8586                                                 BPF_CGROUP_INET4_BIND),
8587         BPF_EAPROG_SEC("cgroup/bind6",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8588                                                 BPF_CGROUP_INET6_BIND),
8589         BPF_EAPROG_SEC("cgroup/connect4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8590                                                 BPF_CGROUP_INET4_CONNECT),
8591         BPF_EAPROG_SEC("cgroup/connect6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8592                                                 BPF_CGROUP_INET6_CONNECT),
8593         BPF_EAPROG_SEC("cgroup/sendmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8594                                                 BPF_CGROUP_UDP4_SENDMSG),
8595         BPF_EAPROG_SEC("cgroup/sendmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8596                                                 BPF_CGROUP_UDP6_SENDMSG),
8597         BPF_EAPROG_SEC("cgroup/recvmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8598                                                 BPF_CGROUP_UDP4_RECVMSG),
8599         BPF_EAPROG_SEC("cgroup/recvmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8600                                                 BPF_CGROUP_UDP6_RECVMSG),
8601         BPF_EAPROG_SEC("cgroup/getpeername4",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8602                                                 BPF_CGROUP_INET4_GETPEERNAME),
8603         BPF_EAPROG_SEC("cgroup/getpeername6",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8604                                                 BPF_CGROUP_INET6_GETPEERNAME),
8605         BPF_EAPROG_SEC("cgroup/getsockname4",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8606                                                 BPF_CGROUP_INET4_GETSOCKNAME),
8607         BPF_EAPROG_SEC("cgroup/getsockname6",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8608                                                 BPF_CGROUP_INET6_GETSOCKNAME),
8609         BPF_EAPROG_SEC("cgroup/sysctl",         BPF_PROG_TYPE_CGROUP_SYSCTL,
8610                                                 BPF_CGROUP_SYSCTL),
8611         BPF_EAPROG_SEC("cgroup/getsockopt",     BPF_PROG_TYPE_CGROUP_SOCKOPT,
8612                                                 BPF_CGROUP_GETSOCKOPT),
8613         BPF_EAPROG_SEC("cgroup/setsockopt",     BPF_PROG_TYPE_CGROUP_SOCKOPT,
8614                                                 BPF_CGROUP_SETSOCKOPT),
8615         BPF_PROG_SEC("struct_ops",              BPF_PROG_TYPE_STRUCT_OPS),
8616         BPF_EAPROG_SEC("sk_lookup/",            BPF_PROG_TYPE_SK_LOOKUP,
8617                                                 BPF_SK_LOOKUP),
8618 };
8619
8620 #undef BPF_PROG_SEC_IMPL
8621 #undef BPF_PROG_SEC
8622 #undef BPF_APROG_SEC
8623 #undef BPF_EAPROG_SEC
8624 #undef BPF_APROG_COMPAT
8625 #undef SEC_DEF
8626
8627 #define MAX_TYPE_NAME_SIZE 32
8628
8629 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8630 {
8631         int i, n = ARRAY_SIZE(section_defs);
8632
8633         for (i = 0; i < n; i++) {
8634                 if (strncmp(sec_name,
8635                             section_defs[i].sec, section_defs[i].len))
8636                         continue;
8637                 return &section_defs[i];
8638         }
8639         return NULL;
8640 }
8641
8642 static char *libbpf_get_type_names(bool attach_type)
8643 {
8644         int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8645         char *buf;
8646
8647         buf = malloc(len);
8648         if (!buf)
8649                 return NULL;
8650
8651         buf[0] = '\0';
8652         /* Forge string buf with all available names */
8653         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8654                 if (attach_type && !section_defs[i].is_attachable)
8655                         continue;
8656
8657                 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8658                         free(buf);
8659                         return NULL;
8660                 }
8661                 strcat(buf, " ");
8662                 strcat(buf, section_defs[i].sec);
8663         }
8664
8665         return buf;
8666 }
8667
8668 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8669                              enum bpf_attach_type *expected_attach_type)
8670 {
8671         const struct bpf_sec_def *sec_def;
8672         char *type_names;
8673
8674         if (!name)
8675                 return -EINVAL;
8676
8677         sec_def = find_sec_def(name);
8678         if (sec_def) {
8679                 *prog_type = sec_def->prog_type;
8680                 *expected_attach_type = sec_def->expected_attach_type;
8681                 return 0;
8682         }
8683
8684         pr_debug("failed to guess program type from ELF section '%s'\n", name);
8685         type_names = libbpf_get_type_names(false);
8686         if (type_names != NULL) {
8687                 pr_debug("supported section(type) names are:%s\n", type_names);
8688                 free(type_names);
8689         }
8690
8691         return -ESRCH;
8692 }
8693
8694 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8695                                                      size_t offset)
8696 {
8697         struct bpf_map *map;
8698         size_t i;
8699
8700         for (i = 0; i < obj->nr_maps; i++) {
8701                 map = &obj->maps[i];
8702                 if (!bpf_map__is_struct_ops(map))
8703                         continue;
8704                 if (map->sec_offset <= offset &&
8705                     offset - map->sec_offset < map->def.value_size)
8706                         return map;
8707         }
8708
8709         return NULL;
8710 }
8711
8712 /* Collect the reloc from ELF and populate the st_ops->progs[] */
8713 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8714                                             GElf_Shdr *shdr, Elf_Data *data)
8715 {
8716         const struct btf_member *member;
8717         struct bpf_struct_ops *st_ops;
8718         struct bpf_program *prog;
8719         unsigned int shdr_idx;
8720         const struct btf *btf;
8721         struct bpf_map *map;
8722         Elf_Data *symbols;
8723         unsigned int moff, insn_idx;
8724         const char *name;
8725         __u32 member_idx;
8726         GElf_Sym sym;
8727         GElf_Rel rel;
8728         int i, nrels;
8729
8730         symbols = obj->efile.symbols;
8731         btf = obj->btf;
8732         nrels = shdr->sh_size / shdr->sh_entsize;
8733         for (i = 0; i < nrels; i++) {
8734                 if (!gelf_getrel(data, i, &rel)) {
8735                         pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8736                         return -LIBBPF_ERRNO__FORMAT;
8737                 }
8738
8739                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
8740                         pr_warn("struct_ops reloc: symbol %zx not found\n",
8741                                 (size_t)GELF_R_SYM(rel.r_info));
8742                         return -LIBBPF_ERRNO__FORMAT;
8743                 }
8744
8745                 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
8746                 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
8747                 if (!map) {
8748                         pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
8749                                 (size_t)rel.r_offset);
8750                         return -EINVAL;
8751                 }
8752
8753                 moff = rel.r_offset - map->sec_offset;
8754                 shdr_idx = sym.st_shndx;
8755                 st_ops = map->st_ops;
8756                 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8757                          map->name,
8758                          (long long)(rel.r_info >> 32),
8759                          (long long)sym.st_value,
8760                          shdr_idx, (size_t)rel.r_offset,
8761                          map->sec_offset, sym.st_name, name);
8762
8763                 if (shdr_idx >= SHN_LORESERVE) {
8764                         pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
8765                                 map->name, (size_t)rel.r_offset, shdr_idx);
8766                         return -LIBBPF_ERRNO__RELOC;
8767                 }
8768                 if (sym.st_value % BPF_INSN_SZ) {
8769                         pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8770                                 map->name, (unsigned long long)sym.st_value);
8771                         return -LIBBPF_ERRNO__FORMAT;
8772                 }
8773                 insn_idx = sym.st_value / BPF_INSN_SZ;
8774
8775                 member = find_member_by_offset(st_ops->type, moff * 8);
8776                 if (!member) {
8777                         pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8778                                 map->name, moff);
8779                         return -EINVAL;
8780                 }
8781                 member_idx = member - btf_members(st_ops->type);
8782                 name = btf__name_by_offset(btf, member->name_off);
8783
8784                 if (!resolve_func_ptr(btf, member->type, NULL)) {
8785                         pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8786                                 map->name, name);
8787                         return -EINVAL;
8788                 }
8789
8790                 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8791                 if (!prog) {
8792                         pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8793                                 map->name, shdr_idx, name);
8794                         return -EINVAL;
8795                 }
8796
8797                 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
8798                         const struct bpf_sec_def *sec_def;
8799
8800                         sec_def = find_sec_def(prog->sec_name);
8801                         if (sec_def &&
8802                             sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
8803                                 /* for pr_warn */
8804                                 prog->type = sec_def->prog_type;
8805                                 goto invalid_prog;
8806                         }
8807
8808                         prog->type = BPF_PROG_TYPE_STRUCT_OPS;
8809                         prog->attach_btf_id = st_ops->type_id;
8810                         prog->expected_attach_type = member_idx;
8811                 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
8812                            prog->attach_btf_id != st_ops->type_id ||
8813                            prog->expected_attach_type != member_idx) {
8814                         goto invalid_prog;
8815                 }
8816                 st_ops->progs[member_idx] = prog;
8817         }
8818
8819         return 0;
8820
8821 invalid_prog:
8822         pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8823                 map->name, prog->name, prog->sec_name, prog->type,
8824                 prog->attach_btf_id, prog->expected_attach_type, name);
8825         return -EINVAL;
8826 }
8827
8828 #define BTF_TRACE_PREFIX "btf_trace_"
8829 #define BTF_LSM_PREFIX "bpf_lsm_"
8830 #define BTF_ITER_PREFIX "bpf_iter_"
8831 #define BTF_MAX_NAME_SIZE 128
8832
8833 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
8834                                    const char *name, __u32 kind)
8835 {
8836         char btf_type_name[BTF_MAX_NAME_SIZE];
8837         int ret;
8838
8839         ret = snprintf(btf_type_name, sizeof(btf_type_name),
8840                        "%s%s", prefix, name);
8841         /* snprintf returns the number of characters written excluding the
8842          * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
8843          * indicates truncation.
8844          */
8845         if (ret < 0 || ret >= sizeof(btf_type_name))
8846                 return -ENAMETOOLONG;
8847         return btf__find_by_name_kind(btf, btf_type_name, kind);
8848 }
8849
8850 static inline int find_attach_btf_id(struct btf *btf, const char *name,
8851                                      enum bpf_attach_type attach_type)
8852 {
8853         int err;
8854
8855         if (attach_type == BPF_TRACE_RAW_TP)
8856                 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name,
8857                                               BTF_KIND_TYPEDEF);
8858         else if (attach_type == BPF_LSM_MAC)
8859                 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name,
8860                                               BTF_KIND_FUNC);
8861         else if (attach_type == BPF_TRACE_ITER)
8862                 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name,
8863                                               BTF_KIND_FUNC);
8864         else
8865                 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8866
8867         return err;
8868 }
8869
8870 int libbpf_find_vmlinux_btf_id(const char *name,
8871                                enum bpf_attach_type attach_type)
8872 {
8873         struct btf *btf;
8874         int err;
8875
8876         btf = libbpf_find_kernel_btf();
8877         if (IS_ERR(btf)) {
8878                 pr_warn("vmlinux BTF is not found\n");
8879                 return -EINVAL;
8880         }
8881
8882         err = find_attach_btf_id(btf, name, attach_type);
8883         if (err <= 0)
8884                 pr_warn("%s is not found in vmlinux BTF\n", name);
8885
8886         btf__free(btf);
8887         return err;
8888 }
8889
8890 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
8891 {
8892         struct bpf_prog_info_linear *info_linear;
8893         struct bpf_prog_info *info;
8894         struct btf *btf = NULL;
8895         int err = -EINVAL;
8896
8897         info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
8898         if (IS_ERR_OR_NULL(info_linear)) {
8899                 pr_warn("failed get_prog_info_linear for FD %d\n",
8900                         attach_prog_fd);
8901                 return -EINVAL;
8902         }
8903         info = &info_linear->info;
8904         if (!info->btf_id) {
8905                 pr_warn("The target program doesn't have BTF\n");
8906                 goto out;
8907         }
8908         if (btf__get_from_id(info->btf_id, &btf)) {
8909                 pr_warn("Failed to get BTF of the program\n");
8910                 goto out;
8911         }
8912         err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8913         btf__free(btf);
8914         if (err <= 0) {
8915                 pr_warn("%s is not found in prog's BTF\n", name);
8916                 goto out;
8917         }
8918 out:
8919         free(info_linear);
8920         return err;
8921 }
8922
8923 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
8924                               enum bpf_attach_type attach_type,
8925                               int *btf_obj_fd, int *btf_type_id)
8926 {
8927         int ret, i;
8928
8929         ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
8930         if (ret > 0) {
8931                 *btf_obj_fd = 0; /* vmlinux BTF */
8932                 *btf_type_id = ret;
8933                 return 0;
8934         }
8935         if (ret != -ENOENT)
8936                 return ret;
8937
8938         ret = load_module_btfs(obj);
8939         if (ret)
8940                 return ret;
8941
8942         for (i = 0; i < obj->btf_module_cnt; i++) {
8943                 const struct module_btf *mod = &obj->btf_modules[i];
8944
8945                 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
8946                 if (ret > 0) {
8947                         *btf_obj_fd = mod->fd;
8948                         *btf_type_id = ret;
8949                         return 0;
8950                 }
8951                 if (ret == -ENOENT)
8952                         continue;
8953
8954                 return ret;
8955         }
8956
8957         return -ESRCH;
8958 }
8959
8960 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id)
8961 {
8962         enum bpf_attach_type attach_type = prog->expected_attach_type;
8963         __u32 attach_prog_fd = prog->attach_prog_fd;
8964         const char *name = prog->sec_name, *attach_name;
8965         const struct bpf_sec_def *sec = NULL;
8966         int i, err;
8967
8968         if (!name)
8969                 return -EINVAL;
8970
8971         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8972                 if (!section_defs[i].is_attach_btf)
8973                         continue;
8974                 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
8975                         continue;
8976
8977                 sec = &section_defs[i];
8978                 break;
8979         }
8980
8981         if (!sec) {
8982                 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);
8983                 return -ESRCH;
8984         }
8985         attach_name = name + sec->len;
8986
8987         /* BPF program's BTF ID */
8988         if (attach_prog_fd) {
8989                 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
8990                 if (err < 0) {
8991                         pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
8992                                  attach_prog_fd, attach_name, err);
8993                         return err;
8994                 }
8995                 *btf_obj_fd = 0;
8996                 *btf_type_id = err;
8997                 return 0;
8998         }
8999
9000         /* kernel/module BTF ID */
9001         err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9002         if (err) {
9003                 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
9004                 return err;
9005         }
9006         return 0;
9007 }
9008
9009 int libbpf_attach_type_by_name(const char *name,
9010                                enum bpf_attach_type *attach_type)
9011 {
9012         char *type_names;
9013         int i;
9014
9015         if (!name)
9016                 return -EINVAL;
9017
9018         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
9019                 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
9020                         continue;
9021                 if (!section_defs[i].is_attachable)
9022                         return -EINVAL;
9023                 *attach_type = section_defs[i].expected_attach_type;
9024                 return 0;
9025         }
9026         pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9027         type_names = libbpf_get_type_names(true);
9028         if (type_names != NULL) {
9029                 pr_debug("attachable section(type) names are:%s\n", type_names);
9030                 free(type_names);
9031         }
9032
9033         return -EINVAL;
9034 }
9035
9036 int bpf_map__fd(const struct bpf_map *map)
9037 {
9038         return map ? map->fd : -EINVAL;
9039 }
9040
9041 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
9042 {
9043         return map ? &map->def : ERR_PTR(-EINVAL);
9044 }
9045
9046 const char *bpf_map__name(const struct bpf_map *map)
9047 {
9048         return map ? map->name : NULL;
9049 }
9050
9051 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9052 {
9053         return map->def.type;
9054 }
9055
9056 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9057 {
9058         if (map->fd >= 0)
9059                 return -EBUSY;
9060         map->def.type = type;
9061         return 0;
9062 }
9063
9064 __u32 bpf_map__map_flags(const struct bpf_map *map)
9065 {
9066         return map->def.map_flags;
9067 }
9068
9069 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9070 {
9071         if (map->fd >= 0)
9072                 return -EBUSY;
9073         map->def.map_flags = flags;
9074         return 0;
9075 }
9076
9077 __u32 bpf_map__numa_node(const struct bpf_map *map)
9078 {
9079         return map->numa_node;
9080 }
9081
9082 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9083 {
9084         if (map->fd >= 0)
9085                 return -EBUSY;
9086         map->numa_node = numa_node;
9087         return 0;
9088 }
9089
9090 __u32 bpf_map__key_size(const struct bpf_map *map)
9091 {
9092         return map->def.key_size;
9093 }
9094
9095 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9096 {
9097         if (map->fd >= 0)
9098                 return -EBUSY;
9099         map->def.key_size = size;
9100         return 0;
9101 }
9102
9103 __u32 bpf_map__value_size(const struct bpf_map *map)
9104 {
9105         return map->def.value_size;
9106 }
9107
9108 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9109 {
9110         if (map->fd >= 0)
9111                 return -EBUSY;
9112         map->def.value_size = size;
9113         return 0;
9114 }
9115
9116 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9117 {
9118         return map ? map->btf_key_type_id : 0;
9119 }
9120
9121 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9122 {
9123         return map ? map->btf_value_type_id : 0;
9124 }
9125
9126 int bpf_map__set_priv(struct bpf_map *map, void *priv,
9127                      bpf_map_clear_priv_t clear_priv)
9128 {
9129         if (!map)
9130                 return -EINVAL;
9131
9132         if (map->priv) {
9133                 if (map->clear_priv)
9134                         map->clear_priv(map, map->priv);
9135         }
9136
9137         map->priv = priv;
9138         map->clear_priv = clear_priv;
9139         return 0;
9140 }
9141
9142 void *bpf_map__priv(const struct bpf_map *map)
9143 {
9144         return map ? map->priv : ERR_PTR(-EINVAL);
9145 }
9146
9147 int bpf_map__set_initial_value(struct bpf_map *map,
9148                                const void *data, size_t size)
9149 {
9150         if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9151             size != map->def.value_size || map->fd >= 0)
9152                 return -EINVAL;
9153
9154         memcpy(map->mmaped, data, size);
9155         return 0;
9156 }
9157
9158 bool bpf_map__is_offload_neutral(const struct bpf_map *map)
9159 {
9160         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
9161 }
9162
9163 bool bpf_map__is_internal(const struct bpf_map *map)
9164 {
9165         return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9166 }
9167
9168 __u32 bpf_map__ifindex(const struct bpf_map *map)
9169 {
9170         return map->map_ifindex;
9171 }
9172
9173 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9174 {
9175         if (map->fd >= 0)
9176                 return -EBUSY;
9177         map->map_ifindex = ifindex;
9178         return 0;
9179 }
9180
9181 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9182 {
9183         if (!bpf_map_type__is_map_in_map(map->def.type)) {
9184                 pr_warn("error: unsupported map type\n");
9185                 return -EINVAL;
9186         }
9187         if (map->inner_map_fd != -1) {
9188                 pr_warn("error: inner_map_fd already specified\n");
9189                 return -EINVAL;
9190         }
9191         map->inner_map_fd = fd;
9192         return 0;
9193 }
9194
9195 static struct bpf_map *
9196 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9197 {
9198         ssize_t idx;
9199         struct bpf_map *s, *e;
9200
9201         if (!obj || !obj->maps)
9202                 return NULL;
9203
9204         s = obj->maps;
9205         e = obj->maps + obj->nr_maps;
9206
9207         if ((m < s) || (m >= e)) {
9208                 pr_warn("error in %s: map handler doesn't belong to object\n",
9209                          __func__);
9210                 return NULL;
9211         }
9212
9213         idx = (m - obj->maps) + i;
9214         if (idx >= obj->nr_maps || idx < 0)
9215                 return NULL;
9216         return &obj->maps[idx];
9217 }
9218
9219 struct bpf_map *
9220 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
9221 {
9222         if (prev == NULL)
9223                 return obj->maps;
9224
9225         return __bpf_map__iter(prev, obj, 1);
9226 }
9227
9228 struct bpf_map *
9229 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
9230 {
9231         if (next == NULL) {
9232                 if (!obj->nr_maps)
9233                         return NULL;
9234                 return obj->maps + obj->nr_maps - 1;
9235         }
9236
9237         return __bpf_map__iter(next, obj, -1);
9238 }
9239
9240 struct bpf_map *
9241 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9242 {
9243         struct bpf_map *pos;
9244
9245         bpf_object__for_each_map(pos, obj) {
9246                 if (pos->name && !strcmp(pos->name, name))
9247                         return pos;
9248         }
9249         return NULL;
9250 }
9251
9252 int
9253 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9254 {
9255         return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9256 }
9257
9258 struct bpf_map *
9259 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
9260 {
9261         return ERR_PTR(-ENOTSUP);
9262 }
9263
9264 long libbpf_get_error(const void *ptr)
9265 {
9266         return PTR_ERR_OR_ZERO(ptr);
9267 }
9268
9269 int bpf_prog_load(const char *file, enum bpf_prog_type type,
9270                   struct bpf_object **pobj, int *prog_fd)
9271 {
9272         struct bpf_prog_load_attr attr;
9273
9274         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
9275         attr.file = file;
9276         attr.prog_type = type;
9277         attr.expected_attach_type = 0;
9278
9279         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
9280 }
9281
9282 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
9283                         struct bpf_object **pobj, int *prog_fd)
9284 {
9285         struct bpf_object_open_attr open_attr = {};
9286         struct bpf_program *prog, *first_prog = NULL;
9287         struct bpf_object *obj;
9288         struct bpf_map *map;
9289         int err;
9290
9291         if (!attr)
9292                 return -EINVAL;
9293         if (!attr->file)
9294                 return -EINVAL;
9295
9296         open_attr.file = attr->file;
9297         open_attr.prog_type = attr->prog_type;
9298
9299         obj = bpf_object__open_xattr(&open_attr);
9300         if (IS_ERR_OR_NULL(obj))
9301                 return -ENOENT;
9302
9303         bpf_object__for_each_program(prog, obj) {
9304                 enum bpf_attach_type attach_type = attr->expected_attach_type;
9305                 /*
9306                  * to preserve backwards compatibility, bpf_prog_load treats
9307                  * attr->prog_type, if specified, as an override to whatever
9308                  * bpf_object__open guessed
9309                  */
9310                 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
9311                         bpf_program__set_type(prog, attr->prog_type);
9312                         bpf_program__set_expected_attach_type(prog,
9313                                                               attach_type);
9314                 }
9315                 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
9316                         /*
9317                          * we haven't guessed from section name and user
9318                          * didn't provide a fallback type, too bad...
9319                          */
9320                         bpf_object__close(obj);
9321                         return -EINVAL;
9322                 }
9323
9324                 prog->prog_ifindex = attr->ifindex;
9325                 prog->log_level = attr->log_level;
9326                 prog->prog_flags |= attr->prog_flags;
9327                 if (!first_prog)
9328                         first_prog = prog;
9329         }
9330
9331         bpf_object__for_each_map(map, obj) {
9332                 if (!bpf_map__is_offload_neutral(map))
9333                         map->map_ifindex = attr->ifindex;
9334         }
9335
9336         if (!first_prog) {
9337                 pr_warn("object file doesn't contain bpf program\n");
9338                 bpf_object__close(obj);
9339                 return -ENOENT;
9340         }
9341
9342         err = bpf_object__load(obj);
9343         if (err) {
9344                 bpf_object__close(obj);
9345                 return err;
9346         }
9347
9348         *pobj = obj;
9349         *prog_fd = bpf_program__fd(first_prog);
9350         return 0;
9351 }
9352
9353 struct bpf_link {
9354         int (*detach)(struct bpf_link *link);
9355         int (*destroy)(struct bpf_link *link);
9356         char *pin_path;         /* NULL, if not pinned */
9357         int fd;                 /* hook FD, -1 if not applicable */
9358         bool disconnected;
9359 };
9360
9361 /* Replace link's underlying BPF program with the new one */
9362 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9363 {
9364         return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9365 }
9366
9367 /* Release "ownership" of underlying BPF resource (typically, BPF program
9368  * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
9369  * link, when destructed through bpf_link__destroy() call won't attempt to
9370  * detach/unregisted that BPF resource. This is useful in situations where,
9371  * say, attached BPF program has to outlive userspace program that attached it
9372  * in the system. Depending on type of BPF program, though, there might be
9373  * additional steps (like pinning BPF program in BPF FS) necessary to ensure
9374  * exit of userspace program doesn't trigger automatic detachment and clean up
9375  * inside the kernel.
9376  */
9377 void bpf_link__disconnect(struct bpf_link *link)
9378 {
9379         link->disconnected = true;
9380 }
9381
9382 int bpf_link__destroy(struct bpf_link *link)
9383 {
9384         int err = 0;
9385
9386         if (IS_ERR_OR_NULL(link))
9387                 return 0;
9388
9389         if (!link->disconnected && link->detach)
9390                 err = link->detach(link);
9391         if (link->destroy)
9392                 link->destroy(link);
9393         if (link->pin_path)
9394                 free(link->pin_path);
9395         free(link);
9396
9397         return err;
9398 }
9399
9400 int bpf_link__fd(const struct bpf_link *link)
9401 {
9402         return link->fd;
9403 }
9404
9405 const char *bpf_link__pin_path(const struct bpf_link *link)
9406 {
9407         return link->pin_path;
9408 }
9409
9410 static int bpf_link__detach_fd(struct bpf_link *link)
9411 {
9412         return close(link->fd);
9413 }
9414
9415 struct bpf_link *bpf_link__open(const char *path)
9416 {
9417         struct bpf_link *link;
9418         int fd;
9419
9420         fd = bpf_obj_get(path);
9421         if (fd < 0) {
9422                 fd = -errno;
9423                 pr_warn("failed to open link at %s: %d\n", path, fd);
9424                 return ERR_PTR(fd);
9425         }
9426
9427         link = calloc(1, sizeof(*link));
9428         if (!link) {
9429                 close(fd);
9430                 return ERR_PTR(-ENOMEM);
9431         }
9432         link->detach = &bpf_link__detach_fd;
9433         link->fd = fd;
9434
9435         link->pin_path = strdup(path);
9436         if (!link->pin_path) {
9437                 bpf_link__destroy(link);
9438                 return ERR_PTR(-ENOMEM);
9439         }
9440
9441         return link;
9442 }
9443
9444 int bpf_link__detach(struct bpf_link *link)
9445 {
9446         return bpf_link_detach(link->fd) ? -errno : 0;
9447 }
9448
9449 int bpf_link__pin(struct bpf_link *link, const char *path)
9450 {
9451         int err;
9452
9453         if (link->pin_path)
9454                 return -EBUSY;
9455         err = make_parent_dir(path);
9456         if (err)
9457                 return err;
9458         err = check_path(path);
9459         if (err)
9460                 return err;
9461
9462         link->pin_path = strdup(path);
9463         if (!link->pin_path)
9464                 return -ENOMEM;
9465
9466         if (bpf_obj_pin(link->fd, link->pin_path)) {
9467                 err = -errno;
9468                 zfree(&link->pin_path);
9469                 return err;
9470         }
9471
9472         pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9473         return 0;
9474 }
9475
9476 int bpf_link__unpin(struct bpf_link *link)
9477 {
9478         int err;
9479
9480         if (!link->pin_path)
9481                 return -EINVAL;
9482
9483         err = unlink(link->pin_path);
9484         if (err != 0)
9485                 return -errno;
9486
9487         pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9488         zfree(&link->pin_path);
9489         return 0;
9490 }
9491
9492 static int bpf_link__detach_perf_event(struct bpf_link *link)
9493 {
9494         int err;
9495
9496         err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0);
9497         if (err)
9498                 err = -errno;
9499
9500         close(link->fd);
9501         return err;
9502 }
9503
9504 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
9505                                                 int pfd)
9506 {
9507         char errmsg[STRERR_BUFSIZE];
9508         struct bpf_link *link;
9509         int prog_fd, err;
9510
9511         if (pfd < 0) {
9512                 pr_warn("prog '%s': invalid perf event FD %d\n",
9513                         prog->name, pfd);
9514                 return ERR_PTR(-EINVAL);
9515         }
9516         prog_fd = bpf_program__fd(prog);
9517         if (prog_fd < 0) {
9518                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9519                         prog->name);
9520                 return ERR_PTR(-EINVAL);
9521         }
9522
9523         link = calloc(1, sizeof(*link));
9524         if (!link)
9525                 return ERR_PTR(-ENOMEM);
9526         link->detach = &bpf_link__detach_perf_event;
9527         link->fd = pfd;
9528
9529         if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9530                 err = -errno;
9531                 free(link);
9532                 pr_warn("prog '%s': failed to attach to pfd %d: %s\n",
9533                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9534                 if (err == -EPROTO)
9535                         pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9536                                 prog->name, pfd);
9537                 return ERR_PTR(err);
9538         }
9539         if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9540                 err = -errno;
9541                 free(link);
9542                 pr_warn("prog '%s': failed to enable pfd %d: %s\n",
9543                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9544                 return ERR_PTR(err);
9545         }
9546         return link;
9547 }
9548
9549 /*
9550  * this function is expected to parse integer in the range of [0, 2^31-1] from
9551  * given file using scanf format string fmt. If actual parsed value is
9552  * negative, the result might be indistinguishable from error
9553  */
9554 static int parse_uint_from_file(const char *file, const char *fmt)
9555 {
9556         char buf[STRERR_BUFSIZE];
9557         int err, ret;
9558         FILE *f;
9559
9560         f = fopen(file, "r");
9561         if (!f) {
9562                 err = -errno;
9563                 pr_debug("failed to open '%s': %s\n", file,
9564                          libbpf_strerror_r(err, buf, sizeof(buf)));
9565                 return err;
9566         }
9567         err = fscanf(f, fmt, &ret);
9568         if (err != 1) {
9569                 err = err == EOF ? -EIO : -errno;
9570                 pr_debug("failed to parse '%s': %s\n", file,
9571                         libbpf_strerror_r(err, buf, sizeof(buf)));
9572                 fclose(f);
9573                 return err;
9574         }
9575         fclose(f);
9576         return ret;
9577 }
9578
9579 static int determine_kprobe_perf_type(void)
9580 {
9581         const char *file = "/sys/bus/event_source/devices/kprobe/type";
9582
9583         return parse_uint_from_file(file, "%d\n");
9584 }
9585
9586 static int determine_uprobe_perf_type(void)
9587 {
9588         const char *file = "/sys/bus/event_source/devices/uprobe/type";
9589
9590         return parse_uint_from_file(file, "%d\n");
9591 }
9592
9593 static int determine_kprobe_retprobe_bit(void)
9594 {
9595         const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9596
9597         return parse_uint_from_file(file, "config:%d\n");
9598 }
9599
9600 static int determine_uprobe_retprobe_bit(void)
9601 {
9602         const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9603
9604         return parse_uint_from_file(file, "config:%d\n");
9605 }
9606
9607 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9608                                  uint64_t offset, int pid)
9609 {
9610         struct perf_event_attr attr = {};
9611         char errmsg[STRERR_BUFSIZE];
9612         int type, pfd, err;
9613
9614         type = uprobe ? determine_uprobe_perf_type()
9615                       : determine_kprobe_perf_type();
9616         if (type < 0) {
9617                 pr_warn("failed to determine %s perf type: %s\n",
9618                         uprobe ? "uprobe" : "kprobe",
9619                         libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9620                 return type;
9621         }
9622         if (retprobe) {
9623                 int bit = uprobe ? determine_uprobe_retprobe_bit()
9624                                  : determine_kprobe_retprobe_bit();
9625
9626                 if (bit < 0) {
9627                         pr_warn("failed to determine %s retprobe bit: %s\n",
9628                                 uprobe ? "uprobe" : "kprobe",
9629                                 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9630                         return bit;
9631                 }
9632                 attr.config |= 1 << bit;
9633         }
9634         attr.size = sizeof(attr);
9635         attr.type = type;
9636         attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
9637         attr.config2 = offset;           /* kprobe_addr or probe_offset */
9638
9639         /* pid filter is meaningful only for uprobes */
9640         pfd = syscall(__NR_perf_event_open, &attr,
9641                       pid < 0 ? -1 : pid /* pid */,
9642                       pid == -1 ? 0 : -1 /* cpu */,
9643                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9644         if (pfd < 0) {
9645                 err = -errno;
9646                 pr_warn("%s perf_event_open() failed: %s\n",
9647                         uprobe ? "uprobe" : "kprobe",
9648                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9649                 return err;
9650         }
9651         return pfd;
9652 }
9653
9654 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
9655                                             bool retprobe,
9656                                             const char *func_name)
9657 {
9658         char errmsg[STRERR_BUFSIZE];
9659         struct bpf_link *link;
9660         int pfd, err;
9661
9662         pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,
9663                                     0 /* offset */, -1 /* pid */);
9664         if (pfd < 0) {
9665                 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n",
9666                         prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9667                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9668                 return ERR_PTR(pfd);
9669         }
9670         link = bpf_program__attach_perf_event(prog, pfd);
9671         if (IS_ERR(link)) {
9672                 close(pfd);
9673                 err = PTR_ERR(link);
9674                 pr_warn("prog '%s': failed to attach to %s '%s': %s\n",
9675                         prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9676                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9677                 return link;
9678         }
9679         return link;
9680 }
9681
9682 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
9683                                       struct bpf_program *prog)
9684 {
9685         const char *func_name;
9686         bool retprobe;
9687
9688         func_name = prog->sec_name + sec->len;
9689         retprobe = strcmp(sec->sec, "kretprobe/") == 0;
9690
9691         return bpf_program__attach_kprobe(prog, retprobe, func_name);
9692 }
9693
9694 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
9695                                             bool retprobe, pid_t pid,
9696                                             const char *binary_path,
9697                                             size_t func_offset)
9698 {
9699         char errmsg[STRERR_BUFSIZE];
9700         struct bpf_link *link;
9701         int pfd, err;
9702
9703         pfd = perf_event_open_probe(true /* uprobe */, retprobe,
9704                                     binary_path, func_offset, pid);
9705         if (pfd < 0) {
9706                 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
9707                         prog->name, retprobe ? "uretprobe" : "uprobe",
9708                         binary_path, func_offset,
9709                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9710                 return ERR_PTR(pfd);
9711         }
9712         link = bpf_program__attach_perf_event(prog, pfd);
9713         if (IS_ERR(link)) {
9714                 close(pfd);
9715                 err = PTR_ERR(link);
9716                 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
9717                         prog->name, retprobe ? "uretprobe" : "uprobe",
9718                         binary_path, func_offset,
9719                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9720                 return link;
9721         }
9722         return link;
9723 }
9724
9725 static int determine_tracepoint_id(const char *tp_category,
9726                                    const char *tp_name)
9727 {
9728         char file[PATH_MAX];
9729         int ret;
9730
9731         ret = snprintf(file, sizeof(file),
9732                        "/sys/kernel/debug/tracing/events/%s/%s/id",
9733                        tp_category, tp_name);
9734         if (ret < 0)
9735                 return -errno;
9736         if (ret >= sizeof(file)) {
9737                 pr_debug("tracepoint %s/%s path is too long\n",
9738                          tp_category, tp_name);
9739                 return -E2BIG;
9740         }
9741         return parse_uint_from_file(file, "%d\n");
9742 }
9743
9744 static int perf_event_open_tracepoint(const char *tp_category,
9745                                       const char *tp_name)
9746 {
9747         struct perf_event_attr attr = {};
9748         char errmsg[STRERR_BUFSIZE];
9749         int tp_id, pfd, err;
9750
9751         tp_id = determine_tracepoint_id(tp_category, tp_name);
9752         if (tp_id < 0) {
9753                 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
9754                         tp_category, tp_name,
9755                         libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
9756                 return tp_id;
9757         }
9758
9759         attr.type = PERF_TYPE_TRACEPOINT;
9760         attr.size = sizeof(attr);
9761         attr.config = tp_id;
9762
9763         pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
9764                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9765         if (pfd < 0) {
9766                 err = -errno;
9767                 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
9768                         tp_category, tp_name,
9769                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9770                 return err;
9771         }
9772         return pfd;
9773 }
9774
9775 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
9776                                                 const char *tp_category,
9777                                                 const char *tp_name)
9778 {
9779         char errmsg[STRERR_BUFSIZE];
9780         struct bpf_link *link;
9781         int pfd, err;
9782
9783         pfd = perf_event_open_tracepoint(tp_category, tp_name);
9784         if (pfd < 0) {
9785                 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
9786                         prog->name, tp_category, tp_name,
9787                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9788                 return ERR_PTR(pfd);
9789         }
9790         link = bpf_program__attach_perf_event(prog, pfd);
9791         if (IS_ERR(link)) {
9792                 close(pfd);
9793                 err = PTR_ERR(link);
9794                 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
9795                         prog->name, tp_category, tp_name,
9796                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9797                 return link;
9798         }
9799         return link;
9800 }
9801
9802 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
9803                                   struct bpf_program *prog)
9804 {
9805         char *sec_name, *tp_cat, *tp_name;
9806         struct bpf_link *link;
9807
9808         sec_name = strdup(prog->sec_name);
9809         if (!sec_name)
9810                 return ERR_PTR(-ENOMEM);
9811
9812         /* extract "tp/<category>/<name>" */
9813         tp_cat = sec_name + sec->len;
9814         tp_name = strchr(tp_cat, '/');
9815         if (!tp_name) {
9816                 link = ERR_PTR(-EINVAL);
9817                 goto out;
9818         }
9819         *tp_name = '\0';
9820         tp_name++;
9821
9822         link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
9823 out:
9824         free(sec_name);
9825         return link;
9826 }
9827
9828 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
9829                                                     const char *tp_name)
9830 {
9831         char errmsg[STRERR_BUFSIZE];
9832         struct bpf_link *link;
9833         int prog_fd, pfd;
9834
9835         prog_fd = bpf_program__fd(prog);
9836         if (prog_fd < 0) {
9837                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9838                 return ERR_PTR(-EINVAL);
9839         }
9840
9841         link = calloc(1, sizeof(*link));
9842         if (!link)
9843                 return ERR_PTR(-ENOMEM);
9844         link->detach = &bpf_link__detach_fd;
9845
9846         pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
9847         if (pfd < 0) {
9848                 pfd = -errno;
9849                 free(link);
9850                 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
9851                         prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9852                 return ERR_PTR(pfd);
9853         }
9854         link->fd = pfd;
9855         return link;
9856 }
9857
9858 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
9859                                       struct bpf_program *prog)
9860 {
9861         const char *tp_name = prog->sec_name + sec->len;
9862
9863         return bpf_program__attach_raw_tracepoint(prog, tp_name);
9864 }
9865
9866 /* Common logic for all BPF program types that attach to a btf_id */
9867 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
9868 {
9869         char errmsg[STRERR_BUFSIZE];
9870         struct bpf_link *link;
9871         int prog_fd, pfd;
9872
9873         prog_fd = bpf_program__fd(prog);
9874         if (prog_fd < 0) {
9875                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9876                 return ERR_PTR(-EINVAL);
9877         }
9878
9879         link = calloc(1, sizeof(*link));
9880         if (!link)
9881                 return ERR_PTR(-ENOMEM);
9882         link->detach = &bpf_link__detach_fd;
9883
9884         pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
9885         if (pfd < 0) {
9886                 pfd = -errno;
9887                 free(link);
9888                 pr_warn("prog '%s': failed to attach: %s\n",
9889                         prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9890                 return ERR_PTR(pfd);
9891         }
9892         link->fd = pfd;
9893         return (struct bpf_link *)link;
9894 }
9895
9896 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
9897 {
9898         return bpf_program__attach_btf_id(prog);
9899 }
9900
9901 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
9902 {
9903         return bpf_program__attach_btf_id(prog);
9904 }
9905
9906 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
9907                                      struct bpf_program *prog)
9908 {
9909         return bpf_program__attach_trace(prog);
9910 }
9911
9912 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
9913                                    struct bpf_program *prog)
9914 {
9915         return bpf_program__attach_lsm(prog);
9916 }
9917
9918 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
9919                                     struct bpf_program *prog)
9920 {
9921         return bpf_program__attach_iter(prog, NULL);
9922 }
9923
9924 static struct bpf_link *
9925 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id,
9926                        const char *target_name)
9927 {
9928         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
9929                             .target_btf_id = btf_id);
9930         enum bpf_attach_type attach_type;
9931         char errmsg[STRERR_BUFSIZE];
9932         struct bpf_link *link;
9933         int prog_fd, link_fd;
9934
9935         prog_fd = bpf_program__fd(prog);
9936         if (prog_fd < 0) {
9937                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9938                 return ERR_PTR(-EINVAL);
9939         }
9940
9941         link = calloc(1, sizeof(*link));
9942         if (!link)
9943                 return ERR_PTR(-ENOMEM);
9944         link->detach = &bpf_link__detach_fd;
9945
9946         attach_type = bpf_program__get_expected_attach_type(prog);
9947         link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
9948         if (link_fd < 0) {
9949                 link_fd = -errno;
9950                 free(link);
9951                 pr_warn("prog '%s': failed to attach to %s: %s\n",
9952                         prog->name, target_name,
9953                         libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
9954                 return ERR_PTR(link_fd);
9955         }
9956         link->fd = link_fd;
9957         return link;
9958 }
9959
9960 struct bpf_link *
9961 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
9962 {
9963         return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
9964 }
9965
9966 struct bpf_link *
9967 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)
9968 {
9969         return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
9970 }
9971
9972 struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)
9973 {
9974         /* target_fd/target_ifindex use the same field in LINK_CREATE */
9975         return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
9976 }
9977
9978 struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog,
9979                                               int target_fd,
9980                                               const char *attach_func_name)
9981 {
9982         int btf_id;
9983
9984         if (!!target_fd != !!attach_func_name) {
9985                 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
9986                         prog->name);
9987                 return ERR_PTR(-EINVAL);
9988         }
9989
9990         if (prog->type != BPF_PROG_TYPE_EXT) {
9991                 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
9992                         prog->name);
9993                 return ERR_PTR(-EINVAL);
9994         }
9995
9996         if (target_fd) {
9997                 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
9998                 if (btf_id < 0)
9999                         return ERR_PTR(btf_id);
10000
10001                 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
10002         } else {
10003                 /* no target, so use raw_tracepoint_open for compatibility
10004                  * with old kernels
10005                  */
10006                 return bpf_program__attach_trace(prog);
10007         }
10008 }
10009
10010 struct bpf_link *
10011 bpf_program__attach_iter(struct bpf_program *prog,
10012                          const struct bpf_iter_attach_opts *opts)
10013 {
10014         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
10015         char errmsg[STRERR_BUFSIZE];
10016         struct bpf_link *link;
10017         int prog_fd, link_fd;
10018         __u32 target_fd = 0;
10019
10020         if (!OPTS_VALID(opts, bpf_iter_attach_opts))
10021                 return ERR_PTR(-EINVAL);
10022
10023         link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
10024         link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
10025
10026         prog_fd = bpf_program__fd(prog);
10027         if (prog_fd < 0) {
10028                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
10029                 return ERR_PTR(-EINVAL);
10030         }
10031
10032         link = calloc(1, sizeof(*link));
10033         if (!link)
10034                 return ERR_PTR(-ENOMEM);
10035         link->detach = &bpf_link__detach_fd;
10036
10037         link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
10038                                   &link_create_opts);
10039         if (link_fd < 0) {
10040                 link_fd = -errno;
10041                 free(link);
10042                 pr_warn("prog '%s': failed to attach to iterator: %s\n",
10043                         prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
10044                 return ERR_PTR(link_fd);
10045         }
10046         link->fd = link_fd;
10047         return link;
10048 }
10049
10050 struct bpf_link *bpf_program__attach(struct bpf_program *prog)
10051 {
10052         const struct bpf_sec_def *sec_def;
10053
10054         sec_def = find_sec_def(prog->sec_name);
10055         if (!sec_def || !sec_def->attach_fn)
10056                 return ERR_PTR(-ESRCH);
10057
10058         return sec_def->attach_fn(sec_def, prog);
10059 }
10060
10061 static int bpf_link__detach_struct_ops(struct bpf_link *link)
10062 {
10063         __u32 zero = 0;
10064
10065         if (bpf_map_delete_elem(link->fd, &zero))
10066                 return -errno;
10067
10068         return 0;
10069 }
10070
10071 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
10072 {
10073         struct bpf_struct_ops *st_ops;
10074         struct bpf_link *link;
10075         __u32 i, zero = 0;
10076         int err;
10077
10078         if (!bpf_map__is_struct_ops(map) || map->fd == -1)
10079                 return ERR_PTR(-EINVAL);
10080
10081         link = calloc(1, sizeof(*link));
10082         if (!link)
10083                 return ERR_PTR(-EINVAL);
10084
10085         st_ops = map->st_ops;
10086         for (i = 0; i < btf_vlen(st_ops->type); i++) {
10087                 struct bpf_program *prog = st_ops->progs[i];
10088                 void *kern_data;
10089                 int prog_fd;
10090
10091                 if (!prog)
10092                         continue;
10093
10094                 prog_fd = bpf_program__fd(prog);
10095                 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
10096                 *(unsigned long *)kern_data = prog_fd;
10097         }
10098
10099         err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
10100         if (err) {
10101                 err = -errno;
10102                 free(link);
10103                 return ERR_PTR(err);
10104         }
10105
10106         link->detach = bpf_link__detach_struct_ops;
10107         link->fd = map->fd;
10108
10109         return link;
10110 }
10111
10112 enum bpf_perf_event_ret
10113 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
10114                            void **copy_mem, size_t *copy_size,
10115                            bpf_perf_event_print_t fn, void *private_data)
10116 {
10117         struct perf_event_mmap_page *header = mmap_mem;
10118         __u64 data_head = ring_buffer_read_head(header);
10119         __u64 data_tail = header->data_tail;
10120         void *base = ((__u8 *)header) + page_size;
10121         int ret = LIBBPF_PERF_EVENT_CONT;
10122         struct perf_event_header *ehdr;
10123         size_t ehdr_size;
10124
10125         while (data_head != data_tail) {
10126                 ehdr = base + (data_tail & (mmap_size - 1));
10127                 ehdr_size = ehdr->size;
10128
10129                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
10130                         void *copy_start = ehdr;
10131                         size_t len_first = base + mmap_size - copy_start;
10132                         size_t len_secnd = ehdr_size - len_first;
10133
10134                         if (*copy_size < ehdr_size) {
10135                                 free(*copy_mem);
10136                                 *copy_mem = malloc(ehdr_size);
10137                                 if (!*copy_mem) {
10138                                         *copy_size = 0;
10139                                         ret = LIBBPF_PERF_EVENT_ERROR;
10140                                         break;
10141                                 }
10142                                 *copy_size = ehdr_size;
10143                         }
10144
10145                         memcpy(*copy_mem, copy_start, len_first);
10146                         memcpy(*copy_mem + len_first, base, len_secnd);
10147                         ehdr = *copy_mem;
10148                 }
10149
10150                 ret = fn(ehdr, private_data);
10151                 data_tail += ehdr_size;
10152                 if (ret != LIBBPF_PERF_EVENT_CONT)
10153                         break;
10154         }
10155
10156         ring_buffer_write_tail(header, data_tail);
10157         return ret;
10158 }
10159
10160 struct perf_buffer;
10161
10162 struct perf_buffer_params {
10163         struct perf_event_attr *attr;
10164         /* if event_cb is specified, it takes precendence */
10165         perf_buffer_event_fn event_cb;
10166         /* sample_cb and lost_cb are higher-level common-case callbacks */
10167         perf_buffer_sample_fn sample_cb;
10168         perf_buffer_lost_fn lost_cb;
10169         void *ctx;
10170         int cpu_cnt;
10171         int *cpus;
10172         int *map_keys;
10173 };
10174
10175 struct perf_cpu_buf {
10176         struct perf_buffer *pb;
10177         void *base; /* mmap()'ed memory */
10178         void *buf; /* for reconstructing segmented data */
10179         size_t buf_size;
10180         int fd;
10181         int cpu;
10182         int map_key;
10183 };
10184
10185 struct perf_buffer {
10186         perf_buffer_event_fn event_cb;
10187         perf_buffer_sample_fn sample_cb;
10188         perf_buffer_lost_fn lost_cb;
10189         void *ctx; /* passed into callbacks */
10190
10191         size_t page_size;
10192         size_t mmap_size;
10193         struct perf_cpu_buf **cpu_bufs;
10194         struct epoll_event *events;
10195         int cpu_cnt; /* number of allocated CPU buffers */
10196         int epoll_fd; /* perf event FD */
10197         int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
10198 };
10199
10200 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
10201                                       struct perf_cpu_buf *cpu_buf)
10202 {
10203         if (!cpu_buf)
10204                 return;
10205         if (cpu_buf->base &&
10206             munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
10207                 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
10208         if (cpu_buf->fd >= 0) {
10209                 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
10210                 close(cpu_buf->fd);
10211         }
10212         free(cpu_buf->buf);
10213         free(cpu_buf);
10214 }
10215
10216 void perf_buffer__free(struct perf_buffer *pb)
10217 {
10218         int i;
10219
10220         if (IS_ERR_OR_NULL(pb))
10221                 return;
10222         if (pb->cpu_bufs) {
10223                 for (i = 0; i < pb->cpu_cnt; i++) {
10224                         struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10225
10226                         if (!cpu_buf)
10227                                 continue;
10228
10229                         bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
10230                         perf_buffer__free_cpu_buf(pb, cpu_buf);
10231                 }
10232                 free(pb->cpu_bufs);
10233         }
10234         if (pb->epoll_fd >= 0)
10235                 close(pb->epoll_fd);
10236         free(pb->events);
10237         free(pb);
10238 }
10239
10240 static struct perf_cpu_buf *
10241 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
10242                           int cpu, int map_key)
10243 {
10244         struct perf_cpu_buf *cpu_buf;
10245         char msg[STRERR_BUFSIZE];
10246         int err;
10247
10248         cpu_buf = calloc(1, sizeof(*cpu_buf));
10249         if (!cpu_buf)
10250                 return ERR_PTR(-ENOMEM);
10251
10252         cpu_buf->pb = pb;
10253         cpu_buf->cpu = cpu;
10254         cpu_buf->map_key = map_key;
10255
10256         cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
10257                               -1, PERF_FLAG_FD_CLOEXEC);
10258         if (cpu_buf->fd < 0) {
10259                 err = -errno;
10260                 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
10261                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10262                 goto error;
10263         }
10264
10265         cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
10266                              PROT_READ | PROT_WRITE, MAP_SHARED,
10267                              cpu_buf->fd, 0);
10268         if (cpu_buf->base == MAP_FAILED) {
10269                 cpu_buf->base = NULL;
10270                 err = -errno;
10271                 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
10272                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10273                 goto error;
10274         }
10275
10276         if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
10277                 err = -errno;
10278                 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
10279                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10280                 goto error;
10281         }
10282
10283         return cpu_buf;
10284
10285 error:
10286         perf_buffer__free_cpu_buf(pb, cpu_buf);
10287         return (struct perf_cpu_buf *)ERR_PTR(err);
10288 }
10289
10290 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10291                                               struct perf_buffer_params *p);
10292
10293 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
10294                                      const struct perf_buffer_opts *opts)
10295 {
10296         struct perf_buffer_params p = {};
10297         struct perf_event_attr attr = { 0, };
10298
10299         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
10300         attr.type = PERF_TYPE_SOFTWARE;
10301         attr.sample_type = PERF_SAMPLE_RAW;
10302         attr.sample_period = 1;
10303         attr.wakeup_events = 1;
10304
10305         p.attr = &attr;
10306         p.sample_cb = opts ? opts->sample_cb : NULL;
10307         p.lost_cb = opts ? opts->lost_cb : NULL;
10308         p.ctx = opts ? opts->ctx : NULL;
10309
10310         return __perf_buffer__new(map_fd, page_cnt, &p);
10311 }
10312
10313 struct perf_buffer *
10314 perf_buffer__new_raw(int map_fd, size_t page_cnt,
10315                      const struct perf_buffer_raw_opts *opts)
10316 {
10317         struct perf_buffer_params p = {};
10318
10319         p.attr = opts->attr;
10320         p.event_cb = opts->event_cb;
10321         p.ctx = opts->ctx;
10322         p.cpu_cnt = opts->cpu_cnt;
10323         p.cpus = opts->cpus;
10324         p.map_keys = opts->map_keys;
10325
10326         return __perf_buffer__new(map_fd, page_cnt, &p);
10327 }
10328
10329 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10330                                               struct perf_buffer_params *p)
10331 {
10332         const char *online_cpus_file = "/sys/devices/system/cpu/online";
10333         struct bpf_map_info map;
10334         char msg[STRERR_BUFSIZE];
10335         struct perf_buffer *pb;
10336         bool *online = NULL;
10337         __u32 map_info_len;
10338         int err, i, j, n;
10339
10340         if (page_cnt & (page_cnt - 1)) {
10341                 pr_warn("page count should be power of two, but is %zu\n",
10342                         page_cnt);
10343                 return ERR_PTR(-EINVAL);
10344         }
10345
10346         /* best-effort sanity checks */
10347         memset(&map, 0, sizeof(map));
10348         map_info_len = sizeof(map);
10349         err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
10350         if (err) {
10351                 err = -errno;
10352                 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
10353                  * -EBADFD, -EFAULT, or -E2BIG on real error
10354                  */
10355                 if (err != -EINVAL) {
10356                         pr_warn("failed to get map info for map FD %d: %s\n",
10357                                 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
10358                         return ERR_PTR(err);
10359                 }
10360                 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
10361                          map_fd);
10362         } else {
10363                 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
10364                         pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
10365                                 map.name);
10366                         return ERR_PTR(-EINVAL);
10367                 }
10368         }
10369
10370         pb = calloc(1, sizeof(*pb));
10371         if (!pb)
10372                 return ERR_PTR(-ENOMEM);
10373
10374         pb->event_cb = p->event_cb;
10375         pb->sample_cb = p->sample_cb;
10376         pb->lost_cb = p->lost_cb;
10377         pb->ctx = p->ctx;
10378
10379         pb->page_size = getpagesize();
10380         pb->mmap_size = pb->page_size * page_cnt;
10381         pb->map_fd = map_fd;
10382
10383         pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
10384         if (pb->epoll_fd < 0) {
10385                 err = -errno;
10386                 pr_warn("failed to create epoll instance: %s\n",
10387                         libbpf_strerror_r(err, msg, sizeof(msg)));
10388                 goto error;
10389         }
10390
10391         if (p->cpu_cnt > 0) {
10392                 pb->cpu_cnt = p->cpu_cnt;
10393         } else {
10394                 pb->cpu_cnt = libbpf_num_possible_cpus();
10395                 if (pb->cpu_cnt < 0) {
10396                         err = pb->cpu_cnt;
10397                         goto error;
10398                 }
10399                 if (map.max_entries && map.max_entries < pb->cpu_cnt)
10400                         pb->cpu_cnt = map.max_entries;
10401         }
10402
10403         pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
10404         if (!pb->events) {
10405                 err = -ENOMEM;
10406                 pr_warn("failed to allocate events: out of memory\n");
10407                 goto error;
10408         }
10409         pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
10410         if (!pb->cpu_bufs) {
10411                 err = -ENOMEM;
10412                 pr_warn("failed to allocate buffers: out of memory\n");
10413                 goto error;
10414         }
10415
10416         err = parse_cpu_mask_file(online_cpus_file, &online, &n);
10417         if (err) {
10418                 pr_warn("failed to get online CPU mask: %d\n", err);
10419                 goto error;
10420         }
10421
10422         for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
10423                 struct perf_cpu_buf *cpu_buf;
10424                 int cpu, map_key;
10425
10426                 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
10427                 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
10428
10429                 /* in case user didn't explicitly requested particular CPUs to
10430                  * be attached to, skip offline/not present CPUs
10431                  */
10432                 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
10433                         continue;
10434
10435                 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
10436                 if (IS_ERR(cpu_buf)) {
10437                         err = PTR_ERR(cpu_buf);
10438                         goto error;
10439                 }
10440
10441                 pb->cpu_bufs[j] = cpu_buf;
10442
10443                 err = bpf_map_update_elem(pb->map_fd, &map_key,
10444                                           &cpu_buf->fd, 0);
10445                 if (err) {
10446                         err = -errno;
10447                         pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
10448                                 cpu, map_key, cpu_buf->fd,
10449                                 libbpf_strerror_r(err, msg, sizeof(msg)));
10450                         goto error;
10451                 }
10452
10453                 pb->events[j].events = EPOLLIN;
10454                 pb->events[j].data.ptr = cpu_buf;
10455                 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
10456                               &pb->events[j]) < 0) {
10457                         err = -errno;
10458                         pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
10459                                 cpu, cpu_buf->fd,
10460                                 libbpf_strerror_r(err, msg, sizeof(msg)));
10461                         goto error;
10462                 }
10463                 j++;
10464         }
10465         pb->cpu_cnt = j;
10466         free(online);
10467
10468         return pb;
10469
10470 error:
10471         free(online);
10472         if (pb)
10473                 perf_buffer__free(pb);
10474         return ERR_PTR(err);
10475 }
10476
10477 struct perf_sample_raw {
10478         struct perf_event_header header;
10479         uint32_t size;
10480         char data[];
10481 };
10482
10483 struct perf_sample_lost {
10484         struct perf_event_header header;
10485         uint64_t id;
10486         uint64_t lost;
10487         uint64_t sample_id;
10488 };
10489
10490 static enum bpf_perf_event_ret
10491 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
10492 {
10493         struct perf_cpu_buf *cpu_buf = ctx;
10494         struct perf_buffer *pb = cpu_buf->pb;
10495         void *data = e;
10496
10497         /* user wants full control over parsing perf event */
10498         if (pb->event_cb)
10499                 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
10500
10501         switch (e->type) {
10502         case PERF_RECORD_SAMPLE: {
10503                 struct perf_sample_raw *s = data;
10504
10505                 if (pb->sample_cb)
10506                         pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
10507                 break;
10508         }
10509         case PERF_RECORD_LOST: {
10510                 struct perf_sample_lost *s = data;
10511
10512                 if (pb->lost_cb)
10513                         pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
10514                 break;
10515         }
10516         default:
10517                 pr_warn("unknown perf sample type %d\n", e->type);
10518                 return LIBBPF_PERF_EVENT_ERROR;
10519         }
10520         return LIBBPF_PERF_EVENT_CONT;
10521 }
10522
10523 static int perf_buffer__process_records(struct perf_buffer *pb,
10524                                         struct perf_cpu_buf *cpu_buf)
10525 {
10526         enum bpf_perf_event_ret ret;
10527
10528         ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
10529                                          pb->page_size, &cpu_buf->buf,
10530                                          &cpu_buf->buf_size,
10531                                          perf_buffer__process_record, cpu_buf);
10532         if (ret != LIBBPF_PERF_EVENT_CONT)
10533                 return ret;
10534         return 0;
10535 }
10536
10537 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
10538 {
10539         return pb->epoll_fd;
10540 }
10541
10542 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
10543 {
10544         int i, cnt, err;
10545
10546         cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
10547         for (i = 0; i < cnt; i++) {
10548                 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
10549
10550                 err = perf_buffer__process_records(pb, cpu_buf);
10551                 if (err) {
10552                         pr_warn("error while processing records: %d\n", err);
10553                         return err;
10554                 }
10555         }
10556         return cnt < 0 ? -errno : cnt;
10557 }
10558
10559 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
10560  * manager.
10561  */
10562 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
10563 {
10564         return pb->cpu_cnt;
10565 }
10566
10567 /*
10568  * Return perf_event FD of a ring buffer in *buf_idx* slot of
10569  * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
10570  * select()/poll()/epoll() Linux syscalls.
10571  */
10572 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
10573 {
10574         struct perf_cpu_buf *cpu_buf;
10575
10576         if (buf_idx >= pb->cpu_cnt)
10577                 return -EINVAL;
10578
10579         cpu_buf = pb->cpu_bufs[buf_idx];
10580         if (!cpu_buf)
10581                 return -ENOENT;
10582
10583         return cpu_buf->fd;
10584 }
10585
10586 /*
10587  * Consume data from perf ring buffer corresponding to slot *buf_idx* in
10588  * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
10589  * consume, do nothing and return success.
10590  * Returns:
10591  *   - 0 on success;
10592  *   - <0 on failure.
10593  */
10594 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
10595 {
10596         struct perf_cpu_buf *cpu_buf;
10597
10598         if (buf_idx >= pb->cpu_cnt)
10599                 return -EINVAL;
10600
10601         cpu_buf = pb->cpu_bufs[buf_idx];
10602         if (!cpu_buf)
10603                 return -ENOENT;
10604
10605         return perf_buffer__process_records(pb, cpu_buf);
10606 }
10607
10608 int perf_buffer__consume(struct perf_buffer *pb)
10609 {
10610         int i, err;
10611
10612         for (i = 0; i < pb->cpu_cnt; i++) {
10613                 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10614
10615                 if (!cpu_buf)
10616                         continue;
10617
10618                 err = perf_buffer__process_records(pb, cpu_buf);
10619                 if (err) {
10620                         pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
10621                         return err;
10622                 }
10623         }
10624         return 0;
10625 }
10626
10627 struct bpf_prog_info_array_desc {
10628         int     array_offset;   /* e.g. offset of jited_prog_insns */
10629         int     count_offset;   /* e.g. offset of jited_prog_len */
10630         int     size_offset;    /* > 0: offset of rec size,
10631                                  * < 0: fix size of -size_offset
10632                                  */
10633 };
10634
10635 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
10636         [BPF_PROG_INFO_JITED_INSNS] = {
10637                 offsetof(struct bpf_prog_info, jited_prog_insns),
10638                 offsetof(struct bpf_prog_info, jited_prog_len),
10639                 -1,
10640         },
10641         [BPF_PROG_INFO_XLATED_INSNS] = {
10642                 offsetof(struct bpf_prog_info, xlated_prog_insns),
10643                 offsetof(struct bpf_prog_info, xlated_prog_len),
10644                 -1,
10645         },
10646         [BPF_PROG_INFO_MAP_IDS] = {
10647                 offsetof(struct bpf_prog_info, map_ids),
10648                 offsetof(struct bpf_prog_info, nr_map_ids),
10649                 -(int)sizeof(__u32),
10650         },
10651         [BPF_PROG_INFO_JITED_KSYMS] = {
10652                 offsetof(struct bpf_prog_info, jited_ksyms),
10653                 offsetof(struct bpf_prog_info, nr_jited_ksyms),
10654                 -(int)sizeof(__u64),
10655         },
10656         [BPF_PROG_INFO_JITED_FUNC_LENS] = {
10657                 offsetof(struct bpf_prog_info, jited_func_lens),
10658                 offsetof(struct bpf_prog_info, nr_jited_func_lens),
10659                 -(int)sizeof(__u32),
10660         },
10661         [BPF_PROG_INFO_FUNC_INFO] = {
10662                 offsetof(struct bpf_prog_info, func_info),
10663                 offsetof(struct bpf_prog_info, nr_func_info),
10664                 offsetof(struct bpf_prog_info, func_info_rec_size),
10665         },
10666         [BPF_PROG_INFO_LINE_INFO] = {
10667                 offsetof(struct bpf_prog_info, line_info),
10668                 offsetof(struct bpf_prog_info, nr_line_info),
10669                 offsetof(struct bpf_prog_info, line_info_rec_size),
10670         },
10671         [BPF_PROG_INFO_JITED_LINE_INFO] = {
10672                 offsetof(struct bpf_prog_info, jited_line_info),
10673                 offsetof(struct bpf_prog_info, nr_jited_line_info),
10674                 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
10675         },
10676         [BPF_PROG_INFO_PROG_TAGS] = {
10677                 offsetof(struct bpf_prog_info, prog_tags),
10678                 offsetof(struct bpf_prog_info, nr_prog_tags),
10679                 -(int)sizeof(__u8) * BPF_TAG_SIZE,
10680         },
10681
10682 };
10683
10684 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
10685                                            int offset)
10686 {
10687         __u32 *array = (__u32 *)info;
10688
10689         if (offset >= 0)
10690                 return array[offset / sizeof(__u32)];
10691         return -(int)offset;
10692 }
10693
10694 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
10695                                            int offset)
10696 {
10697         __u64 *array = (__u64 *)info;
10698
10699         if (offset >= 0)
10700                 return array[offset / sizeof(__u64)];
10701         return -(int)offset;
10702 }
10703
10704 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
10705                                          __u32 val)
10706 {
10707         __u32 *array = (__u32 *)info;
10708
10709         if (offset >= 0)
10710                 array[offset / sizeof(__u32)] = val;
10711 }
10712
10713 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
10714                                          __u64 val)
10715 {
10716         __u64 *array = (__u64 *)info;
10717
10718         if (offset >= 0)
10719                 array[offset / sizeof(__u64)] = val;
10720 }
10721
10722 struct bpf_prog_info_linear *
10723 bpf_program__get_prog_info_linear(int fd, __u64 arrays)
10724 {
10725         struct bpf_prog_info_linear *info_linear;
10726         struct bpf_prog_info info = {};
10727         __u32 info_len = sizeof(info);
10728         __u32 data_len = 0;
10729         int i, err;
10730         void *ptr;
10731
10732         if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
10733                 return ERR_PTR(-EINVAL);
10734
10735         /* step 1: get array dimensions */
10736         err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
10737         if (err) {
10738                 pr_debug("can't get prog info: %s", strerror(errno));
10739                 return ERR_PTR(-EFAULT);
10740         }
10741
10742         /* step 2: calculate total size of all arrays */
10743         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10744                 bool include_array = (arrays & (1UL << i)) > 0;
10745                 struct bpf_prog_info_array_desc *desc;
10746                 __u32 count, size;
10747
10748                 desc = bpf_prog_info_array_desc + i;
10749
10750                 /* kernel is too old to support this field */
10751                 if (info_len < desc->array_offset + sizeof(__u32) ||
10752                     info_len < desc->count_offset + sizeof(__u32) ||
10753                     (desc->size_offset > 0 && info_len < desc->size_offset))
10754                         include_array = false;
10755
10756                 if (!include_array) {
10757                         arrays &= ~(1UL << i);  /* clear the bit */
10758                         continue;
10759                 }
10760
10761                 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10762                 size  = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10763
10764                 data_len += count * size;
10765         }
10766
10767         /* step 3: allocate continuous memory */
10768         data_len = roundup(data_len, sizeof(__u64));
10769         info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
10770         if (!info_linear)
10771                 return ERR_PTR(-ENOMEM);
10772
10773         /* step 4: fill data to info_linear->info */
10774         info_linear->arrays = arrays;
10775         memset(&info_linear->info, 0, sizeof(info));
10776         ptr = info_linear->data;
10777
10778         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10779                 struct bpf_prog_info_array_desc *desc;
10780                 __u32 count, size;
10781
10782                 if ((arrays & (1UL << i)) == 0)
10783                         continue;
10784
10785                 desc  = bpf_prog_info_array_desc + i;
10786                 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10787                 size  = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10788                 bpf_prog_info_set_offset_u32(&info_linear->info,
10789                                              desc->count_offset, count);
10790                 bpf_prog_info_set_offset_u32(&info_linear->info,
10791                                              desc->size_offset, size);
10792                 bpf_prog_info_set_offset_u64(&info_linear->info,
10793                                              desc->array_offset,
10794                                              ptr_to_u64(ptr));
10795                 ptr += count * size;
10796         }
10797
10798         /* step 5: call syscall again to get required arrays */
10799         err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
10800         if (err) {
10801                 pr_debug("can't get prog info: %s", strerror(errno));
10802                 free(info_linear);
10803                 return ERR_PTR(-EFAULT);
10804         }
10805
10806         /* step 6: verify the data */
10807         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10808                 struct bpf_prog_info_array_desc *desc;
10809                 __u32 v1, v2;
10810
10811                 if ((arrays & (1UL << i)) == 0)
10812                         continue;
10813
10814                 desc = bpf_prog_info_array_desc + i;
10815                 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10816                 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10817                                                    desc->count_offset);
10818                 if (v1 != v2)
10819                         pr_warn("%s: mismatch in element count\n", __func__);
10820
10821                 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10822                 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10823                                                    desc->size_offset);
10824                 if (v1 != v2)
10825                         pr_warn("%s: mismatch in rec size\n", __func__);
10826         }
10827
10828         /* step 7: update info_len and data_len */
10829         info_linear->info_len = sizeof(struct bpf_prog_info);
10830         info_linear->data_len = data_len;
10831
10832         return info_linear;
10833 }
10834
10835 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
10836 {
10837         int i;
10838
10839         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10840                 struct bpf_prog_info_array_desc *desc;
10841                 __u64 addr, offs;
10842
10843                 if ((info_linear->arrays & (1UL << i)) == 0)
10844                         continue;
10845
10846                 desc = bpf_prog_info_array_desc + i;
10847                 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
10848                                                      desc->array_offset);
10849                 offs = addr - ptr_to_u64(info_linear->data);
10850                 bpf_prog_info_set_offset_u64(&info_linear->info,
10851                                              desc->array_offset, offs);
10852         }
10853 }
10854
10855 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
10856 {
10857         int i;
10858
10859         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10860                 struct bpf_prog_info_array_desc *desc;
10861                 __u64 addr, offs;
10862
10863                 if ((info_linear->arrays & (1UL << i)) == 0)
10864                         continue;
10865
10866                 desc = bpf_prog_info_array_desc + i;
10867                 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
10868                                                      desc->array_offset);
10869                 addr = offs + ptr_to_u64(info_linear->data);
10870                 bpf_prog_info_set_offset_u64(&info_linear->info,
10871                                              desc->array_offset, addr);
10872         }
10873 }
10874
10875 int bpf_program__set_attach_target(struct bpf_program *prog,
10876                                    int attach_prog_fd,
10877                                    const char *attach_func_name)
10878 {
10879         int btf_obj_fd = 0, btf_id = 0, err;
10880
10881         if (!prog || attach_prog_fd < 0 || !attach_func_name)
10882                 return -EINVAL;
10883
10884         if (prog->obj->loaded)
10885                 return -EINVAL;
10886
10887         if (attach_prog_fd) {
10888                 btf_id = libbpf_find_prog_btf_id(attach_func_name,
10889                                                  attach_prog_fd);
10890                 if (btf_id < 0)
10891                         return btf_id;
10892         } else {
10893                 /* load btf_vmlinux, if not yet */
10894                 err = bpf_object__load_vmlinux_btf(prog->obj, true);
10895                 if (err)
10896                         return err;
10897                 err = find_kernel_btf_id(prog->obj, attach_func_name,
10898                                          prog->expected_attach_type,
10899                                          &btf_obj_fd, &btf_id);
10900                 if (err)
10901                         return err;
10902         }
10903
10904         prog->attach_btf_id = btf_id;
10905         prog->attach_btf_obj_fd = btf_obj_fd;
10906         prog->attach_prog_fd = attach_prog_fd;
10907         return 0;
10908 }
10909
10910 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
10911 {
10912         int err = 0, n, len, start, end = -1;
10913         bool *tmp;
10914
10915         *mask = NULL;
10916         *mask_sz = 0;
10917
10918         /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
10919         while (*s) {
10920                 if (*s == ',' || *s == '\n') {
10921                         s++;
10922                         continue;
10923                 }
10924                 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
10925                 if (n <= 0 || n > 2) {
10926                         pr_warn("Failed to get CPU range %s: %d\n", s, n);
10927                         err = -EINVAL;
10928                         goto cleanup;
10929                 } else if (n == 1) {
10930                         end = start;
10931                 }
10932                 if (start < 0 || start > end) {
10933                         pr_warn("Invalid CPU range [%d,%d] in %s\n",
10934                                 start, end, s);
10935                         err = -EINVAL;
10936                         goto cleanup;
10937                 }
10938                 tmp = realloc(*mask, end + 1);
10939                 if (!tmp) {
10940                         err = -ENOMEM;
10941                         goto cleanup;
10942                 }
10943                 *mask = tmp;
10944                 memset(tmp + *mask_sz, 0, start - *mask_sz);
10945                 memset(tmp + start, 1, end - start + 1);
10946                 *mask_sz = end + 1;
10947                 s += len;
10948         }
10949         if (!*mask_sz) {
10950                 pr_warn("Empty CPU range\n");
10951                 return -EINVAL;
10952         }
10953         return 0;
10954 cleanup:
10955         free(*mask);
10956         *mask = NULL;
10957         return err;
10958 }
10959
10960 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
10961 {
10962         int fd, err = 0, len;
10963         char buf[128];
10964
10965         fd = open(fcpu, O_RDONLY);
10966         if (fd < 0) {
10967                 err = -errno;
10968                 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
10969                 return err;
10970         }
10971         len = read(fd, buf, sizeof(buf));
10972         close(fd);
10973         if (len <= 0) {
10974                 err = len ? -errno : -EINVAL;
10975                 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
10976                 return err;
10977         }
10978         if (len >= sizeof(buf)) {
10979                 pr_warn("CPU mask is too big in file %s\n", fcpu);
10980                 return -E2BIG;
10981         }
10982         buf[len] = '\0';
10983
10984         return parse_cpu_mask_str(buf, mask, mask_sz);
10985 }
10986
10987 int libbpf_num_possible_cpus(void)
10988 {
10989         static const char *fcpu = "/sys/devices/system/cpu/possible";
10990         static int cpus;
10991         int err, n, i, tmp_cpus;
10992         bool *mask;
10993
10994         tmp_cpus = READ_ONCE(cpus);
10995         if (tmp_cpus > 0)
10996                 return tmp_cpus;
10997
10998         err = parse_cpu_mask_file(fcpu, &mask, &n);
10999         if (err)
11000                 return err;
11001
11002         tmp_cpus = 0;
11003         for (i = 0; i < n; i++) {
11004                 if (mask[i])
11005                         tmp_cpus++;
11006         }
11007         free(mask);
11008
11009         WRITE_ONCE(cpus, tmp_cpus);
11010         return tmp_cpus;
11011 }
11012
11013 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
11014                               const struct bpf_object_open_opts *opts)
11015 {
11016         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
11017                 .object_name = s->name,
11018         );
11019         struct bpf_object *obj;
11020         int i;
11021
11022         /* Attempt to preserve opts->object_name, unless overriden by user
11023          * explicitly. Overwriting object name for skeletons is discouraged,
11024          * as it breaks global data maps, because they contain object name
11025          * prefix as their own map name prefix. When skeleton is generated,
11026          * bpftool is making an assumption that this name will stay the same.
11027          */
11028         if (opts) {
11029                 memcpy(&skel_opts, opts, sizeof(*opts));
11030                 if (!opts->object_name)
11031                         skel_opts.object_name = s->name;
11032         }
11033
11034         obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
11035         if (IS_ERR(obj)) {
11036                 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n",
11037                         s->name, PTR_ERR(obj));
11038                 return PTR_ERR(obj);
11039         }
11040
11041         *s->obj = obj;
11042
11043         for (i = 0; i < s->map_cnt; i++) {
11044                 struct bpf_map **map = s->maps[i].map;
11045                 const char *name = s->maps[i].name;
11046                 void **mmaped = s->maps[i].mmaped;
11047
11048                 *map = bpf_object__find_map_by_name(obj, name);
11049                 if (!*map) {
11050                         pr_warn("failed to find skeleton map '%s'\n", name);
11051                         return -ESRCH;
11052                 }
11053
11054                 /* externs shouldn't be pre-setup from user code */
11055                 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
11056                         *mmaped = (*map)->mmaped;
11057         }
11058
11059         for (i = 0; i < s->prog_cnt; i++) {
11060                 struct bpf_program **prog = s->progs[i].prog;
11061                 const char *name = s->progs[i].name;
11062
11063                 *prog = bpf_object__find_program_by_name(obj, name);
11064                 if (!*prog) {
11065                         pr_warn("failed to find skeleton program '%s'\n", name);
11066                         return -ESRCH;
11067                 }
11068         }
11069
11070         return 0;
11071 }
11072
11073 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
11074 {
11075         int i, err;
11076
11077         err = bpf_object__load(*s->obj);
11078         if (err) {
11079                 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
11080                 return err;
11081         }
11082
11083         for (i = 0; i < s->map_cnt; i++) {
11084                 struct bpf_map *map = *s->maps[i].map;
11085                 size_t mmap_sz = bpf_map_mmap_sz(map);
11086                 int prot, map_fd = bpf_map__fd(map);
11087                 void **mmaped = s->maps[i].mmaped;
11088
11089                 if (!mmaped)
11090                         continue;
11091
11092                 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
11093                         *mmaped = NULL;
11094                         continue;
11095                 }
11096
11097                 if (map->def.map_flags & BPF_F_RDONLY_PROG)
11098                         prot = PROT_READ;
11099                 else
11100                         prot = PROT_READ | PROT_WRITE;
11101
11102                 /* Remap anonymous mmap()-ed "map initialization image" as
11103                  * a BPF map-backed mmap()-ed memory, but preserving the same
11104                  * memory address. This will cause kernel to change process'
11105                  * page table to point to a different piece of kernel memory,
11106                  * but from userspace point of view memory address (and its
11107                  * contents, being identical at this point) will stay the
11108                  * same. This mapping will be released by bpf_object__close()
11109                  * as per normal clean up procedure, so we don't need to worry
11110                  * about it from skeleton's clean up perspective.
11111                  */
11112                 *mmaped = mmap(map->mmaped, mmap_sz, prot,
11113                                 MAP_SHARED | MAP_FIXED, map_fd, 0);
11114                 if (*mmaped == MAP_FAILED) {
11115                         err = -errno;
11116                         *mmaped = NULL;
11117                         pr_warn("failed to re-mmap() map '%s': %d\n",
11118                                  bpf_map__name(map), err);
11119                         return err;
11120                 }
11121         }
11122
11123         return 0;
11124 }
11125
11126 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
11127 {
11128         int i;
11129
11130         for (i = 0; i < s->prog_cnt; i++) {
11131                 struct bpf_program *prog = *s->progs[i].prog;
11132                 struct bpf_link **link = s->progs[i].link;
11133                 const struct bpf_sec_def *sec_def;
11134
11135                 if (!prog->load)
11136                         continue;
11137
11138                 sec_def = find_sec_def(prog->sec_name);
11139                 if (!sec_def || !sec_def->attach_fn)
11140                         continue;
11141
11142                 *link = sec_def->attach_fn(sec_def, prog);
11143                 if (IS_ERR(*link)) {
11144                         pr_warn("failed to auto-attach program '%s': %ld\n",
11145                                 bpf_program__name(prog), PTR_ERR(*link));
11146                         return PTR_ERR(*link);
11147                 }
11148         }
11149
11150         return 0;
11151 }
11152
11153 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
11154 {
11155         int i;
11156
11157         for (i = 0; i < s->prog_cnt; i++) {
11158                 struct bpf_link **link = s->progs[i].link;
11159
11160                 bpf_link__destroy(*link);
11161                 *link = NULL;
11162         }
11163 }
11164
11165 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
11166 {
11167         if (s->progs)
11168                 bpf_object__detach_skeleton(s);
11169         if (s->obj)
11170                 bpf_object__close(*s->obj);
11171         free(s->maps);
11172         free(s->progs);
11173         free(s);
11174 }