tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  * Copyright (C) 2019 Isovalent, Inc.
  11  */
  12
  13 #ifndef _GNU_SOURCE
  14 #define _GNU_SOURCE
  15 #endif
  16 #include <stdlib.h>
  17 #include <stdio.h>
  18 #include <stdarg.h>
  19 #include <libgen.h>
  20 #include <inttypes.h>
  21 #include <limits.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <endian.h>
  25 #include <fcntl.h>
  26 #include <errno.h>
  27 #include <ctype.h>
  28 #include <asm/unistd.h>
  29 #include <linux/err.h>
  30 #include <linux/kernel.h>
  31 #include <linux/bpf.h>
  32 #include <linux/btf.h>
  33 #include <linux/filter.h>
  34 #include <linux/list.h>
  35 #include <linux/limits.h>
  36 #include <linux/perf_event.h>
  37 #include <linux/ring_buffer.h>
  38 #include <linux/version.h>
  39 #include <sys/epoll.h>
  40 #include <sys/ioctl.h>
  41 #include <sys/mman.h>
  42 #include <sys/stat.h>
  43 #include <sys/types.h>
  44 #include <sys/vfs.h>
  45 #include <sys/utsname.h>
  46 #include <sys/resource.h>
  47 #include <libelf.h>
  48 #include <gelf.h>
  49 #include <zlib.h>
  50
  51 #include "libbpf.h"
  52 #include "bpf.h"
  53 #include "btf.h"
  54 #include "str_error.h"
  55 #include "libbpf_internal.h"
  56 #include "hashmap.h"
  57
  58 #ifndef EM_BPF
  59 #define EM_BPF 247
  60 #endif
  61
  62 #ifndef BPF_FS_MAGIC
  63 #define BPF_FS_MAGIC            0xcafe4a11
  64 #endif
  65
  66 #define BPF_INSN_SZ (sizeof(struct bpf_insn))
  67
  68 /* vsprintf() in __base_pr() uses nonliteral format string. It may break
  69  * compilation if user enables corresponding warning. Disable it explicitly.
  70  */
  71 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
  72
  73 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  74
  75 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
  76 static const struct btf_type *
  77 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
  78
  79 static int __base_pr(enum libbpf_print_level level, const char *format,
  80                      va_list args)
  81 {
  82         if (level == LIBBPF_DEBUG)
  83                 return 0;
  84
  85         return vfprintf(stderr, format, args);
  86 }
  87
  88 static libbpf_print_fn_t __libbpf_pr = __base_pr;
  89
  90 libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
  91 {
  92         libbpf_print_fn_t old_print_fn = __libbpf_pr;
  93
  94         __libbpf_pr = fn;
  95         return old_print_fn;
  96 }
  97
  98 __printf(2, 3)
  99 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
 100 {
 101         va_list args;
 102
 103         if (!__libbpf_pr)
 104                 return;
 105
 106         va_start(args, format);
 107         __libbpf_pr(level, format, args);
 108         va_end(args);
 109 }
 110
 111 static void pr_perm_msg(int err)
 112 {
 113         struct rlimit limit;
 114         char buf[100];
 115
 116         if (err != -EPERM || geteuid() != 0)
 117                 return;
 118
 119         err = getrlimit(RLIMIT_MEMLOCK, &limit);
 120         if (err)
 121                 return;
 122
 123         if (limit.rlim_cur == RLIM_INFINITY)
 124                 return;
 125
 126         if (limit.rlim_cur < 1024)
 127                 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
 128         else if (limit.rlim_cur < 1024*1024)
 129                 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
 130         else
 131                 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
 132
 133         pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
 134                 buf);
 135 }
 136
 137 #define STRERR_BUFSIZE  128
 138
 139 /* Copied from tools/perf/util/util.h */
 140 #ifndef zfree
 141 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 142 #endif
 143
 144 #ifndef zclose
 145 # define zclose(fd) ({                  \
 146         int ___err = 0;                 \
 147         if ((fd) >= 0)                  \
 148                 ___err = close((fd));   \
 149         fd = -1;                        \
 150         ___err; })
 151 #endif
 152
 153 static inline __u64 ptr_to_u64(const void *ptr)
 154 {
 155         return (__u64) (unsigned long) ptr;
 156 }
 157
 158 enum kern_feature_id {
 159         /* v4.14: kernel support for program & map names. */
 160         FEAT_PROG_NAME,
 161         /* v5.2: kernel support for global data sections. */
 162         FEAT_GLOBAL_DATA,
 163         /* BTF support */
 164         FEAT_BTF,
 165         /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
 166         FEAT_BTF_FUNC,
 167         /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
 168         FEAT_BTF_DATASEC,
 169         /* BTF_FUNC_GLOBAL is supported */
 170         FEAT_BTF_GLOBAL_FUNC,
 171         /* BPF_F_MMAPABLE is supported for arrays */
 172         FEAT_ARRAY_MMAP,
 173         /* kernel support for expected_attach_type in BPF_PROG_LOAD */
 174         FEAT_EXP_ATTACH_TYPE,
 175         /* bpf_probe_read_{kernel,user}[_str] helpers */
 176         FEAT_PROBE_READ_KERN,
 177         /* BPF_PROG_BIND_MAP is supported */
 178         FEAT_PROG_BIND_MAP,
 179         /* Kernel support for module BTFs */
 180         FEAT_MODULE_BTF,
 181         __FEAT_CNT,
 182 };
 183
 184 static bool kernel_supports(enum kern_feature_id feat_id);
 185
 186 enum reloc_type {
 187         RELO_LD64,
 188         RELO_CALL,
 189         RELO_DATA,
 190         RELO_EXTERN,
 191 };
 192
 193 struct reloc_desc {
 194         enum reloc_type type;
 195         int insn_idx;
 196         int map_idx;
 197         int sym_off;
 198         bool processed;
 199 };
 200
 201 struct bpf_sec_def;
 202
 203 typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
 204                                         struct bpf_program *prog);
 205
 206 struct bpf_sec_def {
 207         const char *sec;
 208         size_t len;
 209         enum bpf_prog_type prog_type;
 210         enum bpf_attach_type expected_attach_type;
 211         bool is_exp_attach_type_optional;
 212         bool is_attachable;
 213         bool is_attach_btf;
 214         bool is_sleepable;
 215         attach_fn_t attach_fn;
 216 };
 217
 218 /*
 219  * bpf_prog should be a better name but it has been used in
 220  * linux/filter.h.
 221  */
 222 struct bpf_program {
 223         const struct bpf_sec_def *sec_def;
 224         char *sec_name;
 225         size_t sec_idx;
 226         /* this program's instruction offset (in number of instructions)
 227          * within its containing ELF section
 228          */
 229         size_t sec_insn_off;
 230         /* number of original instructions in ELF section belonging to this
 231          * program, not taking into account subprogram instructions possible
 232          * appended later during relocation
 233          */
 234         size_t sec_insn_cnt;
 235         /* Offset (in number of instructions) of the start of instruction
 236          * belonging to this BPF program  within its containing main BPF
 237          * program. For the entry-point (main) BPF program, this is always
 238          * zero. For a sub-program, this gets reset before each of main BPF
 239          * programs are processed and relocated and is used to determined
 240          * whether sub-program was already appended to the main program, and
 241          * if yes, at which instruction offset.
 242          */
 243         size_t sub_insn_off;
 244
 245         char *name;
 246         /* sec_name with / replaced by _; makes recursive pinning
 247          * in bpf_object__pin_programs easier
 248          */
 249         char *pin_name;
 250
 251         /* instructions that belong to BPF program; insns[0] is located at
 252          * sec_insn_off instruction within its ELF section in ELF file, so
 253          * when mapping ELF file instruction index to the local instruction,
 254          * one needs to subtract sec_insn_off; and vice versa.
 255          */
 256         struct bpf_insn *insns;
 257         /* actual number of instruction in this BPF program's image; for
 258          * entry-point BPF programs this includes the size of main program
 259          * itself plus all the used sub-programs, appended at the end
 260          */
 261         size_t insns_cnt;
 262
 263         struct reloc_desc *reloc_desc;
 264         int nr_reloc;
 265         int log_level;
 266
 267         struct {
 268                 int nr;
 269                 int *fds;
 270         } instances;
 271         bpf_program_prep_t preprocessor;
 272
 273         struct bpf_object *obj;
 274         void *priv;
 275         bpf_program_clear_priv_t clear_priv;
 276
 277         bool load;
 278         enum bpf_prog_type type;
 279         enum bpf_attach_type expected_attach_type;
 280         int prog_ifindex;
 281         __u32 attach_btf_obj_fd;
 282         __u32 attach_btf_id;
 283         __u32 attach_prog_fd;
 284         void *func_info;
 285         __u32 func_info_rec_size;
 286         __u32 func_info_cnt;
 287
 288         void *line_info;
 289         __u32 line_info_rec_size;
 290         __u32 line_info_cnt;
 291         __u32 prog_flags;
 292 };
 293
 294 struct bpf_struct_ops {
 295         const char *tname;
 296         const struct btf_type *type;
 297         struct bpf_program **progs;
 298         __u32 *kern_func_off;
 299         /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
 300         void *data;
 301         /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
 302          *      btf_vmlinux's format.
 303          * struct bpf_struct_ops_tcp_congestion_ops {
 304          *      [... some other kernel fields ...]
 305          *      struct tcp_congestion_ops data;
 306          * }
 307          * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
 308          * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
 309          * from "data".
 310          */
 311         void *kern_vdata;
 312         __u32 type_id;
 313 };
 314
 315 #define DATA_SEC ".data"
 316 #define BSS_SEC ".bss"
 317 #define RODATA_SEC ".rodata"
 318 #define KCONFIG_SEC ".kconfig"
 319 #define KSYMS_SEC ".ksyms"
 320 #define STRUCT_OPS_SEC ".struct_ops"
 321
 322 enum libbpf_map_type {
 323         LIBBPF_MAP_UNSPEC,
 324         LIBBPF_MAP_DATA,
 325         LIBBPF_MAP_BSS,
 326         LIBBPF_MAP_RODATA,
 327         LIBBPF_MAP_KCONFIG,
 328 };
 329
 330 static const char * const libbpf_type_to_btf_name[] = {
 331         [LIBBPF_MAP_DATA]       = DATA_SEC,
 332         [LIBBPF_MAP_BSS]        = BSS_SEC,
 333         [LIBBPF_MAP_RODATA]     = RODATA_SEC,
 334         [LIBBPF_MAP_KCONFIG]    = KCONFIG_SEC,
 335 };
 336
 337 struct bpf_map {
 338         char *name;
 339         int fd;
 340         int sec_idx;
 341         size_t sec_offset;
 342         int map_ifindex;
 343         int inner_map_fd;
 344         struct bpf_map_def def;
 345         __u32 numa_node;
 346         __u32 btf_var_idx;
 347         __u32 btf_key_type_id;
 348         __u32 btf_value_type_id;
 349         __u32 btf_vmlinux_value_type_id;
 350         void *priv;
 351         bpf_map_clear_priv_t clear_priv;
 352         enum libbpf_map_type libbpf_type;
 353         void *mmaped;
 354         struct bpf_struct_ops *st_ops;
 355         struct bpf_map *inner_map;
 356         void **init_slots;
 357         int init_slots_sz;
 358         char *pin_path;
 359         bool pinned;
 360         bool reused;
 361 };
 362
 363 enum extern_type {
 364         EXT_UNKNOWN,
 365         EXT_KCFG,
 366         EXT_KSYM,
 367 };
 368
 369 enum kcfg_type {
 370         KCFG_UNKNOWN,
 371         KCFG_CHAR,
 372         KCFG_BOOL,
 373         KCFG_INT,
 374         KCFG_TRISTATE,
 375         KCFG_CHAR_ARR,
 376 };
 377
 378 struct extern_desc {
 379         enum extern_type type;
 380         int sym_idx;
 381         int btf_id;
 382         int sec_btf_id;
 383         const char *name;
 384         bool is_set;
 385         bool is_weak;
 386         union {
 387                 struct {
 388                         enum kcfg_type type;
 389                         int sz;
 390                         int align;
 391                         int data_off;
 392                         bool is_signed;
 393                 } kcfg;
 394                 struct {
 395                         unsigned long long addr;
 396
 397                         /* target btf_id of the corresponding kernel var. */
 398                         int vmlinux_btf_id;
 399
 400                         /* local btf_id of the ksym extern's type. */
 401                         __u32 type_id;
 402                 } ksym;
 403         };
 404 };
 405
 406 static LIST_HEAD(bpf_objects_list);
 407
 408 struct module_btf {
 409         struct btf *btf;
 410         char *name;
 411         __u32 id;
 412         int fd;
 413 };
 414
 415 struct bpf_object {
 416         char name[BPF_OBJ_NAME_LEN];
 417         char license[64];
 418         __u32 kern_version;
 419
 420         struct bpf_program *programs;
 421         size_t nr_programs;
 422         struct bpf_map *maps;
 423         size_t nr_maps;
 424         size_t maps_cap;
 425
 426         char *kconfig;
 427         struct extern_desc *externs;
 428         int nr_extern;
 429         int kconfig_map_idx;
 430         int rodata_map_idx;
 431
 432         bool loaded;
 433         bool has_subcalls;
 434
 435         /*
 436          * Information when doing elf related work. Only valid if fd
 437          * is valid.
 438          */
 439         struct {
 440                 int fd;
 441                 const void *obj_buf;
 442                 size_t obj_buf_sz;
 443                 Elf *elf;
 444                 GElf_Ehdr ehdr;
 445                 Elf_Data *symbols;
 446                 Elf_Data *data;
 447                 Elf_Data *rodata;
 448                 Elf_Data *bss;
 449                 Elf_Data *st_ops_data;
 450                 size_t shstrndx; /* section index for section name strings */
 451                 size_t strtabidx;
 452                 struct {
 453                         GElf_Shdr shdr;
 454                         Elf_Data *data;
 455                 } *reloc_sects;
 456                 int nr_reloc_sects;
 457                 int maps_shndx;
 458                 int btf_maps_shndx;
 459                 __u32 btf_maps_sec_btf_id;
 460                 int text_shndx;
 461                 int symbols_shndx;
 462                 int data_shndx;
 463                 int rodata_shndx;
 464                 int bss_shndx;
 465                 int st_ops_shndx;
 466         } efile;
 467         /*
 468          * All loaded bpf_object is linked in a list, which is
 469          * hidden to caller. bpf_objects__<func> handlers deal with
 470          * all objects.
 471          */
 472         struct list_head list;
 473
 474         struct btf *btf;
 475         struct btf_ext *btf_ext;
 476
 477         /* Parse and load BTF vmlinux if any of the programs in the object need
 478          * it at load time.
 479          */
 480         struct btf *btf_vmlinux;
 481         /* vmlinux BTF override for CO-RE relocations */
 482         struct btf *btf_vmlinux_override;
 483         /* Lazily initialized kernel module BTFs */
 484         struct module_btf *btf_modules;
 485         bool btf_modules_loaded;
 486         size_t btf_module_cnt;
 487         size_t btf_module_cap;
 488
 489         void *priv;
 490         bpf_object_clear_priv_t clear_priv;
 491
 492         char path[];
 493 };
 494 #define obj_elf_valid(o)        ((o)->efile.elf)
 495
 496 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
 497 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
 498 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
 499 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
 500 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr);
 501 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
 502 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
 503 static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
 504                               size_t off, __u32 sym_type, GElf_Sym *sym);
 505
 506 void bpf_program__unload(struct bpf_program *prog)
 507 {
 508         int i;
 509
 510         if (!prog)
 511                 return;
 512
 513         /*
 514          * If the object is opened but the program was never loaded,
 515          * it is possible that prog->instances.nr == -1.
 516          */
 517         if (prog->instances.nr > 0) {
 518                 for (i = 0; i < prog->instances.nr; i++)
 519                         zclose(prog->instances.fds[i]);
 520         } else if (prog->instances.nr != -1) {
 521                 pr_warn("Internal error: instances.nr is %d\n",
 522                         prog->instances.nr);
 523         }
 524
 525         prog->instances.nr = -1;
 526         zfree(&prog->instances.fds);
 527
 528         zfree(&prog->func_info);
 529         zfree(&prog->line_info);
 530 }
 531
 532 static void bpf_program__exit(struct bpf_program *prog)
 533 {
 534         if (!prog)
 535                 return;
 536
 537         if (prog->clear_priv)
 538                 prog->clear_priv(prog, prog->priv);
 539
 540         prog->priv = NULL;
 541         prog->clear_priv = NULL;
 542
 543         bpf_program__unload(prog);
 544         zfree(&prog->name);
 545         zfree(&prog->sec_name);
 546         zfree(&prog->pin_name);
 547         zfree(&prog->insns);
 548         zfree(&prog->reloc_desc);
 549
 550         prog->nr_reloc = 0;
 551         prog->insns_cnt = 0;
 552         prog->sec_idx = -1;
 553 }
 554
 555 static char *__bpf_program__pin_name(struct bpf_program *prog)
 556 {
 557         char *name, *p;
 558
 559         name = p = strdup(prog->sec_name);
 560         while ((p = strchr(p, '/')))
 561                 *p = '_';
 562
 563         return name;
 564 }
 565
 566 static bool insn_is_subprog_call(const struct bpf_insn *insn)
 567 {
 568         return BPF_CLASS(insn->code) == BPF_JMP &&
 569                BPF_OP(insn->code) == BPF_CALL &&
 570                BPF_SRC(insn->code) == BPF_K &&
 571                insn->src_reg == BPF_PSEUDO_CALL &&
 572                insn->dst_reg == 0 &&
 573                insn->off == 0;
 574 }
 575
 576 static int
 577 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
 578                       const char *name, size_t sec_idx, const char *sec_name,
 579                       size_t sec_off, void *insn_data, size_t insn_data_sz)
 580 {
 581         if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
 582                 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
 583                         sec_name, name, sec_off, insn_data_sz);
 584                 return -EINVAL;
 585         }
 586
 587         memset(prog, 0, sizeof(*prog));
 588         prog->obj = obj;
 589
 590         prog->sec_idx = sec_idx;
 591         prog->sec_insn_off = sec_off / BPF_INSN_SZ;
 592         prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
 593         /* insns_cnt can later be increased by appending used subprograms */
 594         prog->insns_cnt = prog->sec_insn_cnt;
 595
 596         prog->type = BPF_PROG_TYPE_UNSPEC;
 597         prog->load = true;
 598
 599         prog->instances.fds = NULL;
 600         prog->instances.nr = -1;
 601
 602         prog->sec_name = strdup(sec_name);
 603         if (!prog->sec_name)
 604                 goto errout;
 605
 606         prog->name = strdup(name);
 607         if (!prog->name)
 608                 goto errout;
 609
 610         prog->pin_name = __bpf_program__pin_name(prog);
 611         if (!prog->pin_name)
 612                 goto errout;
 613
 614         prog->insns = malloc(insn_data_sz);
 615         if (!prog->insns)
 616                 goto errout;
 617         memcpy(prog->insns, insn_data, insn_data_sz);
 618
 619         return 0;
 620 errout:
 621         pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
 622         bpf_program__exit(prog);
 623         return -ENOMEM;
 624 }
 625
 626 static int
 627 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
 628                          const char *sec_name, int sec_idx)
 629 {
 630         struct bpf_program *prog, *progs;
 631         void *data = sec_data->d_buf;
 632         size_t sec_sz = sec_data->d_size, sec_off, prog_sz;
 633         int nr_progs, err;
 634         const char *name;
 635         GElf_Sym sym;
 636
 637         progs = obj->programs;
 638         nr_progs = obj->nr_programs;
 639         sec_off = 0;
 640
 641         while (sec_off < sec_sz) {
 642                 if (elf_sym_by_sec_off(obj, sec_idx, sec_off, STT_FUNC, &sym)) {
 643                         pr_warn("sec '%s': failed to find program symbol at offset %zu\n",
 644                                 sec_name, sec_off);
 645                         return -LIBBPF_ERRNO__FORMAT;
 646                 }
 647
 648                 prog_sz = sym.st_size;
 649
 650                 name = elf_sym_str(obj, sym.st_name);
 651                 if (!name) {
 652                         pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
 653                                 sec_name, sec_off);
 654                         return -LIBBPF_ERRNO__FORMAT;
 655                 }
 656
 657                 if (sec_off + prog_sz > sec_sz) {
 658                         pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
 659                                 sec_name, sec_off);
 660                         return -LIBBPF_ERRNO__FORMAT;
 661                 }
 662
 663                 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
 664                          sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
 665
 666                 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
 667                 if (!progs) {
 668                         /*
 669                          * In this case the original obj->programs
 670                          * is still valid, so don't need special treat for
 671                          * bpf_close_object().
 672                          */
 673                         pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
 674                                 sec_name, name);
 675                         return -ENOMEM;
 676                 }
 677                 obj->programs = progs;
 678
 679                 prog = &progs[nr_progs];
 680
 681                 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
 682                                             sec_off, data + sec_off, prog_sz);
 683                 if (err)
 684                         return err;
 685
 686                 nr_progs++;
 687                 obj->nr_programs = nr_progs;
 688
 689                 sec_off += prog_sz;
 690         }
 691
 692         return 0;
 693 }
 694
 695 static __u32 get_kernel_version(void)
 696 {
 697         __u32 major, minor, patch;
 698         struct utsname info;
 699
 700         uname(&info);
 701         if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
 702                 return 0;
 703         return KERNEL_VERSION(major, minor, patch);
 704 }
 705
 706 static const struct btf_member *
 707 find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
 708 {
 709         struct btf_member *m;
 710         int i;
 711
 712         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 713                 if (btf_member_bit_offset(t, i) == bit_offset)
 714                         return m;
 715         }
 716
 717         return NULL;
 718 }
 719
 720 static const struct btf_member *
 721 find_member_by_name(const struct btf *btf, const struct btf_type *t,
 722                     const char *name)
 723 {
 724         struct btf_member *m;
 725         int i;
 726
 727         for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
 728                 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
 729                         return m;
 730         }
 731
 732         return NULL;
 733 }
 734
 735 #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
 736 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
 737                                    const char *name, __u32 kind);
 738
 739 static int
 740 find_struct_ops_kern_types(const struct btf *btf, const char *tname,
 741                            const struct btf_type **type, __u32 *type_id,
 742                            const struct btf_type **vtype, __u32 *vtype_id,
 743                            const struct btf_member **data_member)
 744 {
 745         const struct btf_type *kern_type, *kern_vtype;
 746         const struct btf_member *kern_data_member;
 747         __s32 kern_vtype_id, kern_type_id;
 748         __u32 i;
 749
 750         kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
 751         if (kern_type_id < 0) {
 752                 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
 753                         tname);
 754                 return kern_type_id;
 755         }
 756         kern_type = btf__type_by_id(btf, kern_type_id);
 757
 758         /* Find the corresponding "map_value" type that will be used
 759          * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
 760          * find "struct bpf_struct_ops_tcp_congestion_ops" from the
 761          * btf_vmlinux.
 762          */
 763         kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
 764                                                 tname, BTF_KIND_STRUCT);
 765         if (kern_vtype_id < 0) {
 766                 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
 767                         STRUCT_OPS_VALUE_PREFIX, tname);
 768                 return kern_vtype_id;
 769         }
 770         kern_vtype = btf__type_by_id(btf, kern_vtype_id);
 771
 772         /* Find "struct tcp_congestion_ops" from
 773          * struct bpf_struct_ops_tcp_congestion_ops {
 774          *      [ ... ]
 775          *      struct tcp_congestion_ops data;
 776          * }
 777          */
 778         kern_data_member = btf_members(kern_vtype);
 779         for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
 780                 if (kern_data_member->type == kern_type_id)
 781                         break;
 782         }
 783         if (i == btf_vlen(kern_vtype)) {
 784                 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
 785                         tname, STRUCT_OPS_VALUE_PREFIX, tname);
 786                 return -EINVAL;
 787         }
 788
 789         *type = kern_type;
 790         *type_id = kern_type_id;
 791         *vtype = kern_vtype;
 792         *vtype_id = kern_vtype_id;
 793         *data_member = kern_data_member;
 794
 795         return 0;
 796 }
 797
 798 static bool bpf_map__is_struct_ops(const struct bpf_map *map)
 799 {
 800         return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
 801 }
 802
 803 /* Init the map's fields that depend on kern_btf */
 804 static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
 805                                          const struct btf *btf,
 806                                          const struct btf *kern_btf)
 807 {
 808         const struct btf_member *member, *kern_member, *kern_data_member;
 809         const struct btf_type *type, *kern_type, *kern_vtype;
 810         __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
 811         struct bpf_struct_ops *st_ops;
 812         void *data, *kern_data;
 813         const char *tname;
 814         int err;
 815
 816         st_ops = map->st_ops;
 817         type = st_ops->type;
 818         tname = st_ops->tname;
 819         err = find_struct_ops_kern_types(kern_btf, tname,
 820                                          &kern_type, &kern_type_id,
 821                                          &kern_vtype, &kern_vtype_id,
 822                                          &kern_data_member);
 823         if (err)
 824                 return err;
 825
 826         pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
 827                  map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
 828
 829         map->def.value_size = kern_vtype->size;
 830         map->btf_vmlinux_value_type_id = kern_vtype_id;
 831
 832         st_ops->kern_vdata = calloc(1, kern_vtype->size);
 833         if (!st_ops->kern_vdata)
 834                 return -ENOMEM;
 835
 836         data = st_ops->data;
 837         kern_data_off = kern_data_member->offset / 8;
 838         kern_data = st_ops->kern_vdata + kern_data_off;
 839
 840         member = btf_members(type);
 841         for (i = 0; i < btf_vlen(type); i++, member++) {
 842                 const struct btf_type *mtype, *kern_mtype;
 843                 __u32 mtype_id, kern_mtype_id;
 844                 void *mdata, *kern_mdata;
 845                 __s64 msize, kern_msize;
 846                 __u32 moff, kern_moff;
 847                 __u32 kern_member_idx;
 848                 const char *mname;
 849
 850                 mname = btf__name_by_offset(btf, member->name_off);
 851                 kern_member = find_member_by_name(kern_btf, kern_type, mname);
 852                 if (!kern_member) {
 853                         pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
 854                                 map->name, mname);
 855                         return -ENOTSUP;
 856                 }
 857
 858                 kern_member_idx = kern_member - btf_members(kern_type);
 859                 if (btf_member_bitfield_size(type, i) ||
 860                     btf_member_bitfield_size(kern_type, kern_member_idx)) {
 861                         pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
 862                                 map->name, mname);
 863                         return -ENOTSUP;
 864                 }
 865
 866                 moff = member->offset / 8;
 867                 kern_moff = kern_member->offset / 8;
 868
 869                 mdata = data + moff;
 870                 kern_mdata = kern_data + kern_moff;
 871
 872                 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
 873                 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
 874                                                     &kern_mtype_id);
 875                 if (BTF_INFO_KIND(mtype->info) !=
 876                     BTF_INFO_KIND(kern_mtype->info)) {
 877                         pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
 878                                 map->name, mname, BTF_INFO_KIND(mtype->info),
 879                                 BTF_INFO_KIND(kern_mtype->info));
 880                         return -ENOTSUP;
 881                 }
 882
 883                 if (btf_is_ptr(mtype)) {
 884                         struct bpf_program *prog;
 885
 886                         mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
 887                         kern_mtype = skip_mods_and_typedefs(kern_btf,
 888                                                             kern_mtype->type,
 889                                                             &kern_mtype_id);
 890                         if (!btf_is_func_proto(mtype) ||
 891                             !btf_is_func_proto(kern_mtype)) {
 892                                 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
 893                                         map->name, mname);
 894                                 return -ENOTSUP;
 895                         }
 896
 897                         prog = st_ops->progs[i];
 898                         if (!prog) {
 899                                 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
 900                                          map->name, mname);
 901                                 continue;
 902                         }
 903
 904                         prog->attach_btf_id = kern_type_id;
 905                         prog->expected_attach_type = kern_member_idx;
 906
 907                         st_ops->kern_func_off[i] = kern_data_off + kern_moff;
 908
 909                         pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
 910                                  map->name, mname, prog->name, moff,
 911                                  kern_moff);
 912
 913                         continue;
 914                 }
 915
 916                 msize = btf__resolve_size(btf, mtype_id);
 917                 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
 918                 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
 919                         pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
 920                                 map->name, mname, (ssize_t)msize,
 921                                 (ssize_t)kern_msize);
 922                         return -ENOTSUP;
 923                 }
 924
 925                 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
 926                          map->name, mname, (unsigned int)msize,
 927                          moff, kern_moff);
 928                 memcpy(kern_mdata, mdata, msize);
 929         }
 930
 931         return 0;
 932 }
 933
 934 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
 935 {
 936         struct bpf_map *map;
 937         size_t i;
 938         int err;
 939
 940         for (i = 0; i < obj->nr_maps; i++) {
 941                 map = &obj->maps[i];
 942
 943                 if (!bpf_map__is_struct_ops(map))
 944                         continue;
 945
 946                 err = bpf_map__init_kern_struct_ops(map, obj->btf,
 947                                                     obj->btf_vmlinux);
 948                 if (err)
 949                         return err;
 950         }
 951
 952         return 0;
 953 }
 954
 955 static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
 956 {
 957         const struct btf_type *type, *datasec;
 958         const struct btf_var_secinfo *vsi;
 959         struct bpf_struct_ops *st_ops;
 960         const char *tname, *var_name;
 961         __s32 type_id, datasec_id;
 962         const struct btf *btf;
 963         struct bpf_map *map;
 964         __u32 i;
 965
 966         if (obj->efile.st_ops_shndx == -1)
 967                 return 0;
 968
 969         btf = obj->btf;
 970         datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
 971                                             BTF_KIND_DATASEC);
 972         if (datasec_id < 0) {
 973                 pr_warn("struct_ops init: DATASEC %s not found\n",
 974                         STRUCT_OPS_SEC);
 975                 return -EINVAL;
 976         }
 977
 978         datasec = btf__type_by_id(btf, datasec_id);
 979         vsi = btf_var_secinfos(datasec);
 980         for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
 981                 type = btf__type_by_id(obj->btf, vsi->type);
 982                 var_name = btf__name_by_offset(obj->btf, type->name_off);
 983
 984                 type_id = btf__resolve_type(obj->btf, vsi->type);
 985                 if (type_id < 0) {
 986                         pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
 987                                 vsi->type, STRUCT_OPS_SEC);
 988                         return -EINVAL;
 989                 }
 990
 991                 type = btf__type_by_id(obj->btf, type_id);
 992                 tname = btf__name_by_offset(obj->btf, type->name_off);
 993                 if (!tname[0]) {
 994                         pr_warn("struct_ops init: anonymous type is not supported\n");
 995                         return -ENOTSUP;
 996                 }
 997                 if (!btf_is_struct(type)) {
 998                         pr_warn("struct_ops init: %s is not a struct\n", tname);
 999                         return -EINVAL;
1000                 }
1001
1002                 map = bpf_object__add_map(obj);
1003                 if (IS_ERR(map))
1004                         return PTR_ERR(map);
1005
1006                 map->sec_idx = obj->efile.st_ops_shndx;
1007                 map->sec_offset = vsi->offset;
1008                 map->name = strdup(var_name);
1009                 if (!map->name)
1010                         return -ENOMEM;
1011
1012                 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1013                 map->def.key_size = sizeof(int);
1014                 map->def.value_size = type->size;
1015                 map->def.max_entries = 1;
1016
1017                 map->st_ops = calloc(1, sizeof(*map->st_ops));
1018                 if (!map->st_ops)
1019                         return -ENOMEM;
1020                 st_ops = map->st_ops;
1021                 st_ops->data = malloc(type->size);
1022                 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1023                 st_ops->kern_func_off = malloc(btf_vlen(type) *
1024                                                sizeof(*st_ops->kern_func_off));
1025                 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1026                         return -ENOMEM;
1027
1028                 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1029                         pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1030                                 var_name, STRUCT_OPS_SEC);
1031                         return -EINVAL;
1032                 }
1033
1034                 memcpy(st_ops->data,
1035                        obj->efile.st_ops_data->d_buf + vsi->offset,
1036                        type->size);
1037                 st_ops->tname = tname;
1038                 st_ops->type = type;
1039                 st_ops->type_id = type_id;
1040
1041                 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1042                          tname, type_id, var_name, vsi->offset);
1043         }
1044
1045         return 0;
1046 }
1047
1048 static struct bpf_object *bpf_object__new(const char *path,
1049                                           const void *obj_buf,
1050                                           size_t obj_buf_sz,
1051                                           const char *obj_name)
1052 {
1053         struct bpf_object *obj;
1054         char *end;
1055
1056         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1057         if (!obj) {
1058                 pr_warn("alloc memory failed for %s\n", path);
1059                 return ERR_PTR(-ENOMEM);
1060         }
1061
1062         strcpy(obj->path, path);
1063         if (obj_name) {
1064                 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
1065                 obj->name[sizeof(obj->name) - 1] = 0;
1066         } else {
1067                 /* Using basename() GNU version which doesn't modify arg. */
1068                 strncpy(obj->name, basename((void *)path),
1069                         sizeof(obj->name) - 1);
1070                 end = strchr(obj->name, '.');
1071                 if (end)
1072                         *end = 0;
1073         }
1074
1075         obj->efile.fd = -1;
1076         /*
1077          * Caller of this function should also call
1078          * bpf_object__elf_finish() after data collection to return
1079          * obj_buf to user. If not, we should duplicate the buffer to
1080          * avoid user freeing them before elf finish.
1081          */
1082         obj->efile.obj_buf = obj_buf;
1083         obj->efile.obj_buf_sz = obj_buf_sz;
1084         obj->efile.maps_shndx = -1;
1085         obj->efile.btf_maps_shndx = -1;
1086         obj->efile.data_shndx = -1;
1087         obj->efile.rodata_shndx = -1;
1088         obj->efile.bss_shndx = -1;
1089         obj->efile.st_ops_shndx = -1;
1090         obj->kconfig_map_idx = -1;
1091         obj->rodata_map_idx = -1;
1092
1093         obj->kern_version = get_kernel_version();
1094         obj->loaded = false;
1095
1096         INIT_LIST_HEAD(&obj->list);
1097         list_add(&obj->list, &bpf_objects_list);
1098         return obj;
1099 }
1100
1101 static void bpf_object__elf_finish(struct bpf_object *obj)
1102 {
1103         if (!obj_elf_valid(obj))
1104                 return;
1105
1106         if (obj->efile.elf) {
1107                 elf_end(obj->efile.elf);
1108                 obj->efile.elf = NULL;
1109         }
1110         obj->efile.symbols = NULL;
1111         obj->efile.data = NULL;
1112         obj->efile.rodata = NULL;
1113         obj->efile.bss = NULL;
1114         obj->efile.st_ops_data = NULL;
1115
1116         zfree(&obj->efile.reloc_sects);
1117         obj->efile.nr_reloc_sects = 0;
1118         zclose(obj->efile.fd);
1119         obj->efile.obj_buf = NULL;
1120         obj->efile.obj_buf_sz = 0;
1121 }
1122
1123 /* if libelf is old and doesn't support mmap(), fall back to read() */
1124 #ifndef ELF_C_READ_MMAP
1125 #define ELF_C_READ_MMAP ELF_C_READ
1126 #endif
1127
1128 static int bpf_object__elf_init(struct bpf_object *obj)
1129 {
1130         int err = 0;
1131         GElf_Ehdr *ep;
1132
1133         if (obj_elf_valid(obj)) {
1134                 pr_warn("elf: init internal error\n");
1135                 return -LIBBPF_ERRNO__LIBELF;
1136         }
1137
1138         if (obj->efile.obj_buf_sz > 0) {
1139                 /*
1140                  * obj_buf should have been validated by
1141                  * bpf_object__open_buffer().
1142                  */
1143                 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1144                                             obj->efile.obj_buf_sz);
1145         } else {
1146                 obj->efile.fd = open(obj->path, O_RDONLY);
1147                 if (obj->efile.fd < 0) {
1148                         char errmsg[STRERR_BUFSIZE], *cp;
1149
1150                         err = -errno;
1151                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1152                         pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1153                         return err;
1154                 }
1155
1156                 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1157         }
1158
1159         if (!obj->efile.elf) {
1160                 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1161                 err = -LIBBPF_ERRNO__LIBELF;
1162                 goto errout;
1163         }
1164
1165         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1166                 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1167                 err = -LIBBPF_ERRNO__FORMAT;
1168                 goto errout;
1169         }
1170         ep = &obj->efile.ehdr;
1171
1172         if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) {
1173                 pr_warn("elf: failed to get section names section index for %s: %s\n",
1174                         obj->path, elf_errmsg(-1));
1175                 err = -LIBBPF_ERRNO__FORMAT;
1176                 goto errout;
1177         }
1178
1179         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1180         if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) {
1181                 pr_warn("elf: failed to get section names strings from %s: %s\n",
1182                         obj->path, elf_errmsg(-1));
1183                 return -LIBBPF_ERRNO__FORMAT;
1184         }
1185
1186         /* Old LLVM set e_machine to EM_NONE */
1187         if (ep->e_type != ET_REL ||
1188             (ep->e_machine && ep->e_machine != EM_BPF)) {
1189                 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1190                 err = -LIBBPF_ERRNO__FORMAT;
1191                 goto errout;
1192         }
1193
1194         return 0;
1195 errout:
1196         bpf_object__elf_finish(obj);
1197         return err;
1198 }
1199
1200 static int bpf_object__check_endianness(struct bpf_object *obj)
1201 {
1202 #if __BYTE_ORDER == __LITTLE_ENDIAN
1203         if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1204                 return 0;
1205 #elif __BYTE_ORDER == __BIG_ENDIAN
1206         if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1207                 return 0;
1208 #else
1209 # error "Unrecognized __BYTE_ORDER__"
1210 #endif
1211         pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1212         return -LIBBPF_ERRNO__ENDIAN;
1213 }
1214
1215 static int
1216 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1217 {
1218         memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1219         pr_debug("license of %s is %s\n", obj->path, obj->license);
1220         return 0;
1221 }
1222
1223 static int
1224 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1225 {
1226         __u32 kver;
1227
1228         if (size != sizeof(kver)) {
1229                 pr_warn("invalid kver section in %s\n", obj->path);
1230                 return -LIBBPF_ERRNO__FORMAT;
1231         }
1232         memcpy(&kver, data, sizeof(kver));
1233         obj->kern_version = kver;
1234         pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1235         return 0;
1236 }
1237
1238 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1239 {
1240         if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1241             type == BPF_MAP_TYPE_HASH_OF_MAPS)
1242                 return true;
1243         return false;
1244 }
1245
1246 int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1247                              __u32 *size)
1248 {
1249         int ret = -ENOENT;
1250
1251         *size = 0;
1252         if (!name) {
1253                 return -EINVAL;
1254         } else if (!strcmp(name, DATA_SEC)) {
1255                 if (obj->efile.data)
1256                         *size = obj->efile.data->d_size;
1257         } else if (!strcmp(name, BSS_SEC)) {
1258                 if (obj->efile.bss)
1259                         *size = obj->efile.bss->d_size;
1260         } else if (!strcmp(name, RODATA_SEC)) {
1261                 if (obj->efile.rodata)
1262                         *size = obj->efile.rodata->d_size;
1263         } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1264                 if (obj->efile.st_ops_data)
1265                         *size = obj->efile.st_ops_data->d_size;
1266         } else {
1267                 Elf_Scn *scn = elf_sec_by_name(obj, name);
1268                 Elf_Data *data = elf_sec_data(obj, scn);
1269
1270                 if (data) {
1271                         ret = 0; /* found it */
1272                         *size = data->d_size;
1273                 }
1274         }
1275
1276         return *size ? 0 : ret;
1277 }
1278
1279 int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1280                                 __u32 *off)
1281 {
1282         Elf_Data *symbols = obj->efile.symbols;
1283         const char *sname;
1284         size_t si;
1285
1286         if (!name || !off)
1287                 return -EINVAL;
1288
1289         for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1290                 GElf_Sym sym;
1291
1292                 if (!gelf_getsym(symbols, si, &sym))
1293                         continue;
1294                 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1295                     GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1296                         continue;
1297
1298                 sname = elf_sym_str(obj, sym.st_name);
1299                 if (!sname) {
1300                         pr_warn("failed to get sym name string for var %s\n",
1301                                 name);
1302                         return -EIO;
1303                 }
1304                 if (strcmp(name, sname) == 0) {
1305                         *off = sym.st_value;
1306                         return 0;
1307                 }
1308         }
1309
1310         return -ENOENT;
1311 }
1312
1313 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1314 {
1315         struct bpf_map *new_maps;
1316         size_t new_cap;
1317         int i;
1318
1319         if (obj->nr_maps < obj->maps_cap)
1320                 return &obj->maps[obj->nr_maps++];
1321
1322         new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1323         new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps));
1324         if (!new_maps) {
1325                 pr_warn("alloc maps for object failed\n");
1326                 return ERR_PTR(-ENOMEM);
1327         }
1328
1329         obj->maps_cap = new_cap;
1330         obj->maps = new_maps;
1331
1332         /* zero out new maps */
1333         memset(obj->maps + obj->nr_maps, 0,
1334                (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1335         /*
1336          * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
1337          * when failure (zclose won't close negative fd)).
1338          */
1339         for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1340                 obj->maps[i].fd = -1;
1341                 obj->maps[i].inner_map_fd = -1;
1342         }
1343
1344         return &obj->maps[obj->nr_maps++];
1345 }
1346
1347 static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1348 {
1349         long page_sz = sysconf(_SC_PAGE_SIZE);
1350         size_t map_sz;
1351
1352         map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1353         map_sz = roundup(map_sz, page_sz);
1354         return map_sz;
1355 }
1356
1357 static char *internal_map_name(struct bpf_object *obj,
1358                                enum libbpf_map_type type)
1359 {
1360         char map_name[BPF_OBJ_NAME_LEN], *p;
1361         const char *sfx = libbpf_type_to_btf_name[type];
1362         int sfx_len = max((size_t)7, strlen(sfx));
1363         int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1364                           strlen(obj->name));
1365
1366         snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1367                  sfx_len, libbpf_type_to_btf_name[type]);
1368
1369         /* sanitise map name to characters allowed by kernel */
1370         for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1371                 if (!isalnum(*p) && *p != '_' && *p != '.')
1372                         *p = '_';
1373
1374         return strdup(map_name);
1375 }
1376
1377 static int
1378 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1379                               int sec_idx, void *data, size_t data_sz)
1380 {
1381         struct bpf_map_def *def;
1382         struct bpf_map *map;
1383         int err;
1384
1385         map = bpf_object__add_map(obj);
1386         if (IS_ERR(map))
1387                 return PTR_ERR(map);
1388
1389         map->libbpf_type = type;
1390         map->sec_idx = sec_idx;
1391         map->sec_offset = 0;
1392         map->name = internal_map_name(obj, type);
1393         if (!map->name) {
1394                 pr_warn("failed to alloc map name\n");
1395                 return -ENOMEM;
1396         }
1397
1398         def = &map->def;
1399         def->type = BPF_MAP_TYPE_ARRAY;
1400         def->key_size = sizeof(int);
1401         def->value_size = data_sz;
1402         def->max_entries = 1;
1403         def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1404                          ? BPF_F_RDONLY_PROG : 0;
1405         def->map_flags |= BPF_F_MMAPABLE;
1406
1407         pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1408                  map->name, map->sec_idx, map->sec_offset, def->map_flags);
1409
1410         map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1411                            MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1412         if (map->mmaped == MAP_FAILED) {
1413                 err = -errno;
1414                 map->mmaped = NULL;
1415                 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1416                         map->name, err);
1417                 zfree(&map->name);
1418                 return err;
1419         }
1420
1421         if (data)
1422                 memcpy(map->mmaped, data, data_sz);
1423
1424         pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1425         return 0;
1426 }
1427
1428 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1429 {
1430         int err;
1431
1432         /*
1433          * Populate obj->maps with libbpf internal maps.
1434          */
1435         if (obj->efile.data_shndx >= 0) {
1436                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1437                                                     obj->efile.data_shndx,
1438                                                     obj->efile.data->d_buf,
1439                                                     obj->efile.data->d_size);
1440                 if (err)
1441                         return err;
1442         }
1443         if (obj->efile.rodata_shndx >= 0) {
1444                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1445                                                     obj->efile.rodata_shndx,
1446                                                     obj->efile.rodata->d_buf,
1447                                                     obj->efile.rodata->d_size);
1448                 if (err)
1449                         return err;
1450
1451                 obj->rodata_map_idx = obj->nr_maps - 1;
1452         }
1453         if (obj->efile.bss_shndx >= 0) {
1454                 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1455                                                     obj->efile.bss_shndx,
1456                                                     NULL,
1457                                                     obj->efile.bss->d_size);
1458                 if (err)
1459                         return err;
1460         }
1461         return 0;
1462 }
1463
1464
1465 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1466                                                const void *name)
1467 {
1468         int i;
1469
1470         for (i = 0; i < obj->nr_extern; i++) {
1471                 if (strcmp(obj->externs[i].name, name) == 0)
1472                         return &obj->externs[i];
1473         }
1474         return NULL;
1475 }
1476
1477 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1478                               char value)
1479 {
1480         switch (ext->kcfg.type) {
1481         case KCFG_BOOL:
1482                 if (value == 'm') {
1483                         pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
1484                                 ext->name, value);
1485                         return -EINVAL;
1486                 }
1487                 *(bool *)ext_val = value == 'y' ? true : false;
1488                 break;
1489         case KCFG_TRISTATE:
1490                 if (value == 'y')
1491                         *(enum libbpf_tristate *)ext_val = TRI_YES;
1492                 else if (value == 'm')
1493                         *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1494                 else /* value == 'n' */
1495                         *(enum libbpf_tristate *)ext_val = TRI_NO;
1496                 break;
1497         case KCFG_CHAR:
1498                 *(char *)ext_val = value;
1499                 break;
1500         case KCFG_UNKNOWN:
1501         case KCFG_INT:
1502         case KCFG_CHAR_ARR:
1503         default:
1504                 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
1505                         ext->name, value);
1506                 return -EINVAL;
1507         }
1508         ext->is_set = true;
1509         return 0;
1510 }
1511
1512 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1513                               const char *value)
1514 {
1515         size_t len;
1516
1517         if (ext->kcfg.type != KCFG_CHAR_ARR) {
1518                 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
1519                 return -EINVAL;
1520         }
1521
1522         len = strlen(value);
1523         if (value[len - 1] != '"') {
1524                 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1525                         ext->name, value);
1526                 return -EINVAL;
1527         }
1528
1529         /* strip quotes */
1530         len -= 2;
1531         if (len >= ext->kcfg.sz) {
1532                 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1533                         ext->name, value, len, ext->kcfg.sz - 1);
1534                 len = ext->kcfg.sz - 1;
1535         }
1536         memcpy(ext_val, value + 1, len);
1537         ext_val[len] = '\0';
1538         ext->is_set = true;
1539         return 0;
1540 }
1541
1542 static int parse_u64(const char *value, __u64 *res)
1543 {
1544         char *value_end;
1545         int err;
1546
1547         errno = 0;
1548         *res = strtoull(value, &value_end, 0);
1549         if (errno) {
1550                 err = -errno;
1551                 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1552                 return err;
1553         }
1554         if (*value_end) {
1555                 pr_warn("failed to parse '%s' as integer completely\n", value);
1556                 return -EINVAL;
1557         }
1558         return 0;
1559 }
1560
1561 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1562 {
1563         int bit_sz = ext->kcfg.sz * 8;
1564
1565         if (ext->kcfg.sz == 8)
1566                 return true;
1567
1568         /* Validate that value stored in u64 fits in integer of `ext->sz`
1569          * bytes size without any loss of information. If the target integer
1570          * is signed, we rely on the following limits of integer type of
1571          * Y bits and subsequent transformation:
1572          *
1573          *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
1574          *            0 <= X + 2^(Y-1) <= 2^Y - 1
1575          *            0 <= X + 2^(Y-1) <  2^Y
1576          *
1577          *  For unsigned target integer, check that all the (64 - Y) bits are
1578          *  zero.
1579          */
1580         if (ext->kcfg.is_signed)
1581                 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1582         else
1583                 return (v >> bit_sz) == 0;
1584 }
1585
1586 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1587                               __u64 value)
1588 {
1589         if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1590                 pr_warn("extern (kcfg) %s=%llu should be integer\n",
1591                         ext->name, (unsigned long long)value);
1592                 return -EINVAL;
1593         }
1594         if (!is_kcfg_value_in_range(ext, value)) {
1595                 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
1596                         ext->name, (unsigned long long)value, ext->kcfg.sz);
1597                 return -ERANGE;
1598         }
1599         switch (ext->kcfg.sz) {
1600                 case 1: *(__u8 *)ext_val = value; break;
1601                 case 2: *(__u16 *)ext_val = value; break;
1602                 case 4: *(__u32 *)ext_val = value; break;
1603                 case 8: *(__u64 *)ext_val = value; break;
1604                 default:
1605                         return -EINVAL;
1606         }
1607         ext->is_set = true;
1608         return 0;
1609 }
1610
1611 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1612                                             char *buf, void *data)
1613 {
1614         struct extern_desc *ext;
1615         char *sep, *value;
1616         int len, err = 0;
1617         void *ext_val;
1618         __u64 num;
1619
1620         if (strncmp(buf, "CONFIG_", 7))
1621                 return 0;
1622
1623         sep = strchr(buf, '=');
1624         if (!sep) {
1625                 pr_warn("failed to parse '%s': no separator\n", buf);
1626                 return -EINVAL;
1627         }
1628
1629         /* Trim ending '\n' */
1630         len = strlen(buf);
1631         if (buf[len - 1] == '\n')
1632                 buf[len - 1] = '\0';
1633         /* Split on '=' and ensure that a value is present. */
1634         *sep = '\0';
1635         if (!sep[1]) {
1636                 *sep = '=';
1637                 pr_warn("failed to parse '%s': no value\n", buf);
1638                 return -EINVAL;
1639         }
1640
1641         ext = find_extern_by_name(obj, buf);
1642         if (!ext || ext->is_set)
1643                 return 0;
1644
1645         ext_val = data + ext->kcfg.data_off;
1646         value = sep + 1;
1647
1648         switch (*value) {
1649         case 'y': case 'n': case 'm':
1650                 err = set_kcfg_value_tri(ext, ext_val, *value);
1651                 break;
1652         case '"':
1653                 err = set_kcfg_value_str(ext, ext_val, value);
1654                 break;
1655         default:
1656                 /* assume integer */
1657                 err = parse_u64(value, &num);
1658                 if (err) {
1659                         pr_warn("extern (kcfg) %s=%s should be integer\n",
1660                                 ext->name, value);
1661                         return err;
1662                 }
1663                 err = set_kcfg_value_num(ext, ext_val, num);
1664                 break;
1665         }
1666         if (err)
1667                 return err;
1668         pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
1669         return 0;
1670 }
1671
1672 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1673 {
1674         char buf[PATH_MAX];
1675         struct utsname uts;
1676         int len, err = 0;
1677         gzFile file;
1678
1679         uname(&uts);
1680         len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1681         if (len < 0)
1682                 return -EINVAL;
1683         else if (len >= PATH_MAX)
1684                 return -ENAMETOOLONG;
1685
1686         /* gzopen also accepts uncompressed files. */
1687         file = gzopen(buf, "r");
1688         if (!file)
1689                 file = gzopen("/proc/config.gz", "r");
1690
1691         if (!file) {
1692                 pr_warn("failed to open system Kconfig\n");
1693                 return -ENOENT;
1694         }
1695
1696         while (gzgets(file, buf, sizeof(buf))) {
1697                 err = bpf_object__process_kconfig_line(obj, buf, data);
1698                 if (err) {
1699                         pr_warn("error parsing system Kconfig line '%s': %d\n",
1700                                 buf, err);
1701                         goto out;
1702                 }
1703         }
1704
1705 out:
1706         gzclose(file);
1707         return err;
1708 }
1709
1710 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1711                                         const char *config, void *data)
1712 {
1713         char buf[PATH_MAX];
1714         int err = 0;
1715         FILE *file;
1716
1717         file = fmemopen((void *)config, strlen(config), "r");
1718         if (!file) {
1719                 err = -errno;
1720                 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1721                 return err;
1722         }
1723
1724         while (fgets(buf, sizeof(buf), file)) {
1725                 err = bpf_object__process_kconfig_line(obj, buf, data);
1726                 if (err) {
1727                         pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1728                                 buf, err);
1729                         break;
1730                 }
1731         }
1732
1733         fclose(file);
1734         return err;
1735 }
1736
1737 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1738 {
1739         struct extern_desc *last_ext = NULL, *ext;
1740         size_t map_sz;
1741         int i, err;
1742
1743         for (i = 0; i < obj->nr_extern; i++) {
1744                 ext = &obj->externs[i];
1745                 if (ext->type == EXT_KCFG)
1746                         last_ext = ext;
1747         }
1748
1749         if (!last_ext)
1750                 return 0;
1751
1752         map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1753         err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1754                                             obj->efile.symbols_shndx,
1755                                             NULL, map_sz);
1756         if (err)
1757                 return err;
1758
1759         obj->kconfig_map_idx = obj->nr_maps - 1;
1760
1761         return 0;
1762 }
1763
1764 static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1765 {
1766         Elf_Data *symbols = obj->efile.symbols;
1767         int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1768         Elf_Data *data = NULL;
1769         Elf_Scn *scn;
1770
1771         if (obj->efile.maps_shndx < 0)
1772                 return 0;
1773
1774         if (!symbols)
1775                 return -EINVAL;
1776
1777
1778         scn = elf_sec_by_idx(obj, obj->efile.maps_shndx);
1779         data = elf_sec_data(obj, scn);
1780         if (!scn || !data) {
1781                 pr_warn("elf: failed to get legacy map definitions for %s\n",
1782                         obj->path);
1783                 return -EINVAL;
1784         }
1785
1786         /*
1787          * Count number of maps. Each map has a name.
1788          * Array of maps is not supported: only the first element is
1789          * considered.
1790          *
1791          * TODO: Detect array of map and report error.
1792          */
1793         nr_syms = symbols->d_size / sizeof(GElf_Sym);
1794         for (i = 0; i < nr_syms; i++) {
1795                 GElf_Sym sym;
1796
1797                 if (!gelf_getsym(symbols, i, &sym))
1798                         continue;
1799                 if (sym.st_shndx != obj->efile.maps_shndx)
1800                         continue;
1801                 nr_maps++;
1802         }
1803         /* Assume equally sized map definitions */
1804         pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n",
1805                  nr_maps, data->d_size, obj->path);
1806
1807         if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1808                 pr_warn("elf: unable to determine legacy map definition size in %s\n",
1809                         obj->path);
1810                 return -EINVAL;
1811         }
1812         map_def_sz = data->d_size / nr_maps;
1813
1814         /* Fill obj->maps using data in "maps" section.  */
1815         for (i = 0; i < nr_syms; i++) {
1816                 GElf_Sym sym;
1817                 const char *map_name;
1818                 struct bpf_map_def *def;
1819                 struct bpf_map *map;
1820
1821                 if (!gelf_getsym(symbols, i, &sym))
1822                         continue;
1823                 if (sym.st_shndx != obj->efile.maps_shndx)
1824                         continue;
1825
1826                 map = bpf_object__add_map(obj);
1827                 if (IS_ERR(map))
1828                         return PTR_ERR(map);
1829
1830                 map_name = elf_sym_str(obj, sym.st_name);
1831                 if (!map_name) {
1832                         pr_warn("failed to get map #%d name sym string for obj %s\n",
1833                                 i, obj->path);
1834                         return -LIBBPF_ERRNO__FORMAT;
1835                 }
1836
1837                 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1838                 map->sec_idx = sym.st_shndx;
1839                 map->sec_offset = sym.st_value;
1840                 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1841                          map_name, map->sec_idx, map->sec_offset);
1842                 if (sym.st_value + map_def_sz > data->d_size) {
1843                         pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1844                                 obj->path, map_name);
1845                         return -EINVAL;
1846                 }
1847
1848                 map->name = strdup(map_name);
1849                 if (!map->name) {
1850                         pr_warn("failed to alloc map name\n");
1851                         return -ENOMEM;
1852                 }
1853                 pr_debug("map %d is \"%s\"\n", i, map->name);
1854                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1855                 /*
1856                  * If the definition of the map in the object file fits in
1857                  * bpf_map_def, copy it.  Any extra fields in our version
1858                  * of bpf_map_def will default to zero as a result of the
1859                  * calloc above.
1860                  */
1861                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1862                         memcpy(&map->def, def, map_def_sz);
1863                 } else {
1864                         /*
1865                          * Here the map structure being read is bigger than what
1866                          * we expect, truncate if the excess bits are all zero.
1867                          * If they are not zero, reject this map as
1868                          * incompatible.
1869                          */
1870                         char *b;
1871
1872                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
1873                              b < ((char *)def) + map_def_sz; b++) {
1874                                 if (*b != 0) {
1875                                         pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1876                                                 obj->path, map_name);
1877                                         if (strict)
1878                                                 return -EINVAL;
1879                                 }
1880                         }
1881                         memcpy(&map->def, def, sizeof(struct bpf_map_def));
1882                 }
1883         }
1884         return 0;
1885 }
1886
1887 static const struct btf_type *
1888 skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1889 {
1890         const struct btf_type *t = btf__type_by_id(btf, id);
1891
1892         if (res_id)
1893                 *res_id = id;
1894
1895         while (btf_is_mod(t) || btf_is_typedef(t)) {
1896                 if (res_id)
1897                         *res_id = t->type;
1898                 t = btf__type_by_id(btf, t->type);
1899         }
1900
1901         return t;
1902 }
1903
1904 static const struct btf_type *
1905 resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1906 {
1907         const struct btf_type *t;
1908
1909         t = skip_mods_and_typedefs(btf, id, NULL);
1910         if (!btf_is_ptr(t))
1911                 return NULL;
1912
1913         t = skip_mods_and_typedefs(btf, t->type, res_id);
1914
1915         return btf_is_func_proto(t) ? t : NULL;
1916 }
1917
1918 static const char *btf_kind_str(const struct btf_type *t)
1919 {
1920         switch (btf_kind(t)) {
1921         case BTF_KIND_UNKN: return "void";
1922         case BTF_KIND_INT: return "int";
1923         case BTF_KIND_PTR: return "ptr";
1924         case BTF_KIND_ARRAY: return "array";
1925         case BTF_KIND_STRUCT: return "struct";
1926         case BTF_KIND_UNION: return "union";
1927         case BTF_KIND_ENUM: return "enum";
1928         case BTF_KIND_FWD: return "fwd";
1929         case BTF_KIND_TYPEDEF: return "typedef";
1930         case BTF_KIND_VOLATILE: return "volatile";
1931         case BTF_KIND_CONST: return "const";
1932         case BTF_KIND_RESTRICT: return "restrict";
1933         case BTF_KIND_FUNC: return "func";
1934         case BTF_KIND_FUNC_PROTO: return "func_proto";
1935         case BTF_KIND_VAR: return "var";
1936         case BTF_KIND_DATASEC: return "datasec";
1937         default: return "unknown";
1938         }
1939 }
1940
1941 /*
1942  * Fetch integer attribute of BTF map definition. Such attributes are
1943  * represented using a pointer to an array, in which dimensionality of array
1944  * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
1945  * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
1946  * type definition, while using only sizeof(void *) space in ELF data section.
1947  */
1948 static bool get_map_field_int(const char *map_name, const struct btf *btf,
1949                               const struct btf_member *m, __u32 *res)
1950 {
1951         const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1952         const char *name = btf__name_by_offset(btf, m->name_off);
1953         const struct btf_array *arr_info;
1954         const struct btf_type *arr_t;
1955
1956         if (!btf_is_ptr(t)) {
1957                 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
1958                         map_name, name, btf_kind_str(t));
1959                 return false;
1960         }
1961
1962         arr_t = btf__type_by_id(btf, t->type);
1963         if (!arr_t) {
1964                 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1965                         map_name, name, t->type);
1966                 return false;
1967         }
1968         if (!btf_is_array(arr_t)) {
1969                 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
1970                         map_name, name, btf_kind_str(arr_t));
1971                 return false;
1972         }
1973         arr_info = btf_array(arr_t);
1974         *res = arr_info->nelems;
1975         return true;
1976 }
1977
1978 static int build_map_pin_path(struct bpf_map *map, const char *path)
1979 {
1980         char buf[PATH_MAX];
1981         int len;
1982
1983         if (!path)
1984                 path = "/sys/fs/bpf";
1985
1986         len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
1987         if (len < 0)
1988                 return -EINVAL;
1989         else if (len >= PATH_MAX)
1990                 return -ENAMETOOLONG;
1991
1992         return bpf_map__set_pin_path(map, buf);
1993 }
1994
1995
1996 static int parse_btf_map_def(struct bpf_object *obj,
1997                              struct bpf_map *map,
1998                              const struct btf_type *def,
1999                              bool strict, bool is_inner,
2000                              const char *pin_root_path)
2001 {
2002         const struct btf_type *t;
2003         const struct btf_member *m;
2004         int vlen, i;
2005
2006         vlen = btf_vlen(def);
2007         m = btf_members(def);
2008         for (i = 0; i < vlen; i++, m++) {
2009                 const char *name = btf__name_by_offset(obj->btf, m->name_off);
2010
2011                 if (!name) {
2012                         pr_warn("map '%s': invalid field #%d.\n", map->name, i);
2013                         return -EINVAL;
2014                 }
2015                 if (strcmp(name, "type") == 0) {
2016                         if (!get_map_field_int(map->name, obj->btf, m,
2017                                                &map->def.type))
2018                                 return -EINVAL;
2019                         pr_debug("map '%s': found type = %u.\n",
2020                                  map->name, map->def.type);
2021                 } else if (strcmp(name, "max_entries") == 0) {
2022                         if (!get_map_field_int(map->name, obj->btf, m,
2023                                                &map->def.max_entries))
2024                                 return -EINVAL;
2025                         pr_debug("map '%s': found max_entries = %u.\n",
2026                                  map->name, map->def.max_entries);
2027                 } else if (strcmp(name, "map_flags") == 0) {
2028                         if (!get_map_field_int(map->name, obj->btf, m,
2029                                                &map->def.map_flags))
2030                                 return -EINVAL;
2031                         pr_debug("map '%s': found map_flags = %u.\n",
2032                                  map->name, map->def.map_flags);
2033                 } else if (strcmp(name, "numa_node") == 0) {
2034                         if (!get_map_field_int(map->name, obj->btf, m, &map->numa_node))
2035                                 return -EINVAL;
2036                         pr_debug("map '%s': found numa_node = %u.\n", map->name, map->numa_node);
2037                 } else if (strcmp(name, "key_size") == 0) {
2038                         __u32 sz;
2039
2040                         if (!get_map_field_int(map->name, obj->btf, m, &sz))
2041                                 return -EINVAL;
2042                         pr_debug("map '%s': found key_size = %u.\n",
2043                                  map->name, sz);
2044                         if (map->def.key_size && map->def.key_size != sz) {
2045                                 pr_warn("map '%s': conflicting key size %u != %u.\n",
2046                                         map->name, map->def.key_size, sz);
2047                                 return -EINVAL;
2048                         }
2049                         map->def.key_size = sz;
2050                 } else if (strcmp(name, "key") == 0) {
2051                         __s64 sz;
2052
2053                         t = btf__type_by_id(obj->btf, m->type);
2054                         if (!t) {
2055                                 pr_warn("map '%s': key type [%d] not found.\n",
2056                                         map->name, m->type);
2057                                 return -EINVAL;
2058                         }
2059                         if (!btf_is_ptr(t)) {
2060                                 pr_warn("map '%s': key spec is not PTR: %s.\n",
2061                                         map->name, btf_kind_str(t));
2062                                 return -EINVAL;
2063                         }
2064                         sz = btf__resolve_size(obj->btf, t->type);
2065                         if (sz < 0) {
2066                                 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2067                                         map->name, t->type, (ssize_t)sz);
2068                                 return sz;
2069                         }
2070                         pr_debug("map '%s': found key [%u], sz = %zd.\n",
2071                                  map->name, t->type, (ssize_t)sz);
2072                         if (map->def.key_size && map->def.key_size != sz) {
2073                                 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2074                                         map->name, map->def.key_size, (ssize_t)sz);
2075                                 return -EINVAL;
2076                         }
2077                         map->def.key_size = sz;
2078                         map->btf_key_type_id = t->type;
2079                 } else if (strcmp(name, "value_size") == 0) {
2080                         __u32 sz;
2081
2082                         if (!get_map_field_int(map->name, obj->btf, m, &sz))
2083                                 return -EINVAL;
2084                         pr_debug("map '%s': found value_size = %u.\n",
2085                                  map->name, sz);
2086                         if (map->def.value_size && map->def.value_size != sz) {
2087                                 pr_warn("map '%s': conflicting value size %u != %u.\n",
2088                                         map->name, map->def.value_size, sz);
2089                                 return -EINVAL;
2090                         }
2091                         map->def.value_size = sz;
2092                 } else if (strcmp(name, "value") == 0) {
2093                         __s64 sz;
2094
2095                         t = btf__type_by_id(obj->btf, m->type);
2096                         if (!t) {
2097                                 pr_warn("map '%s': value type [%d] not found.\n",
2098                                         map->name, m->type);
2099                                 return -EINVAL;
2100                         }
2101                         if (!btf_is_ptr(t)) {
2102                                 pr_warn("map '%s': value spec is not PTR: %s.\n",
2103                                         map->name, btf_kind_str(t));
2104                                 return -EINVAL;
2105                         }
2106                         sz = btf__resolve_size(obj->btf, t->type);
2107                         if (sz < 0) {
2108                                 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2109                                         map->name, t->type, (ssize_t)sz);
2110                                 return sz;
2111                         }
2112                         pr_debug("map '%s': found value [%u], sz = %zd.\n",
2113                                  map->name, t->type, (ssize_t)sz);
2114                         if (map->def.value_size && map->def.value_size != sz) {
2115                                 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2116                                         map->name, map->def.value_size, (ssize_t)sz);
2117                                 return -EINVAL;
2118                         }
2119                         map->def.value_size = sz;
2120                         map->btf_value_type_id = t->type;
2121                 }
2122                 else if (strcmp(name, "values") == 0) {
2123                         int err;
2124
2125                         if (is_inner) {
2126                                 pr_warn("map '%s': multi-level inner maps not supported.\n",
2127                                         map->name);
2128                                 return -ENOTSUP;
2129                         }
2130                         if (i != vlen - 1) {
2131                                 pr_warn("map '%s': '%s' member should be last.\n",
2132                                         map->name, name);
2133                                 return -EINVAL;
2134                         }
2135                         if (!bpf_map_type__is_map_in_map(map->def.type)) {
2136                                 pr_warn("map '%s': should be map-in-map.\n",
2137                                         map->name);
2138                                 return -ENOTSUP;
2139                         }
2140                         if (map->def.value_size && map->def.value_size != 4) {
2141                                 pr_warn("map '%s': conflicting value size %u != 4.\n",
2142                                         map->name, map->def.value_size);
2143                                 return -EINVAL;
2144                         }
2145                         map->def.value_size = 4;
2146                         t = btf__type_by_id(obj->btf, m->type);
2147                         if (!t) {
2148                                 pr_warn("map '%s': map-in-map inner type [%d] not found.\n",
2149                                         map->name, m->type);
2150                                 return -EINVAL;
2151                         }
2152                         if (!btf_is_array(t) || btf_array(t)->nelems) {
2153                                 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",
2154                                         map->name);
2155                                 return -EINVAL;
2156                         }
2157                         t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type,
2158                                                    NULL);
2159                         if (!btf_is_ptr(t)) {
2160                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2161                                         map->name, btf_kind_str(t));
2162                                 return -EINVAL;
2163                         }
2164                         t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
2165                         if (!btf_is_struct(t)) {
2166                                 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2167                                         map->name, btf_kind_str(t));
2168                                 return -EINVAL;
2169                         }
2170
2171                         map->inner_map = calloc(1, sizeof(*map->inner_map));
2172                         if (!map->inner_map)
2173                                 return -ENOMEM;
2174                         map->inner_map->sec_idx = obj->efile.btf_maps_shndx;
2175                         map->inner_map->name = malloc(strlen(map->name) +
2176                                                       sizeof(".inner") + 1);
2177                         if (!map->inner_map->name)
2178                                 return -ENOMEM;
2179                         sprintf(map->inner_map->name, "%s.inner", map->name);
2180
2181                         err = parse_btf_map_def(obj, map->inner_map, t, strict,
2182                                                 true /* is_inner */, NULL);
2183                         if (err)
2184                                 return err;
2185                 } else if (strcmp(name, "pinning") == 0) {
2186                         __u32 val;
2187                         int err;
2188
2189                         if (is_inner) {
2190                                 pr_debug("map '%s': inner def can't be pinned.\n",
2191                                          map->name);
2192                                 return -EINVAL;
2193                         }
2194                         if (!get_map_field_int(map->name, obj->btf, m, &val))
2195                                 return -EINVAL;
2196                         pr_debug("map '%s': found pinning = %u.\n",
2197                                  map->name, val);
2198
2199                         if (val != LIBBPF_PIN_NONE &&
2200                             val != LIBBPF_PIN_BY_NAME) {
2201                                 pr_warn("map '%s': invalid pinning value %u.\n",
2202                                         map->name, val);
2203                                 return -EINVAL;
2204                         }
2205                         if (val == LIBBPF_PIN_BY_NAME) {
2206                                 err = build_map_pin_path(map, pin_root_path);
2207                                 if (err) {
2208                                         pr_warn("map '%s': couldn't build pin path.\n",
2209                                                 map->name);
2210                                         return err;
2211                                 }
2212                         }
2213                 } else {
2214                         if (strict) {
2215                                 pr_warn("map '%s': unknown field '%s'.\n",
2216                                         map->name, name);
2217                                 return -ENOTSUP;
2218                         }
2219                         pr_debug("map '%s': ignoring unknown field '%s'.\n",
2220                                  map->name, name);
2221                 }
2222         }
2223
2224         if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
2225                 pr_warn("map '%s': map type isn't specified.\n", map->name);
2226                 return -EINVAL;
2227         }
2228
2229         return 0;
2230 }
2231
2232 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2233                                          const struct btf_type *sec,
2234                                          int var_idx, int sec_idx,
2235                                          const Elf_Data *data, bool strict,
2236                                          const char *pin_root_path)
2237 {
2238         const struct btf_type *var, *def;
2239         const struct btf_var_secinfo *vi;
2240         const struct btf_var *var_extra;
2241         const char *map_name;
2242         struct bpf_map *map;
2243
2244         vi = btf_var_secinfos(sec) + var_idx;
2245         var = btf__type_by_id(obj->btf, vi->type);
2246         var_extra = btf_var(var);
2247         map_name = btf__name_by_offset(obj->btf, var->name_off);
2248
2249         if (map_name == NULL || map_name[0] == '\0') {
2250                 pr_warn("map #%d: empty name.\n", var_idx);
2251                 return -EINVAL;
2252         }
2253         if ((__u64)vi->offset + vi->size > data->d_size) {
2254                 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2255                 return -EINVAL;
2256         }
2257         if (!btf_is_var(var)) {
2258                 pr_warn("map '%s': unexpected var kind %s.\n",
2259                         map_name, btf_kind_str(var));
2260                 return -EINVAL;
2261         }
2262         if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2263             var_extra->linkage != BTF_VAR_STATIC) {
2264                 pr_warn("map '%s': unsupported var linkage %u.\n",
2265                         map_name, var_extra->linkage);
2266                 return -EOPNOTSUPP;
2267         }
2268
2269         def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2270         if (!btf_is_struct(def)) {
2271                 pr_warn("map '%s': unexpected def kind %s.\n",
2272                         map_name, btf_kind_str(var));
2273                 return -EINVAL;
2274         }
2275         if (def->size > vi->size) {
2276                 pr_warn("map '%s': invalid def size.\n", map_name);
2277                 return -EINVAL;
2278         }
2279
2280         map = bpf_object__add_map(obj);
2281         if (IS_ERR(map))
2282                 return PTR_ERR(map);
2283         map->name = strdup(map_name);
2284         if (!map->name) {
2285                 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2286                 return -ENOMEM;
2287         }
2288         map->libbpf_type = LIBBPF_MAP_UNSPEC;
2289         map->def.type = BPF_MAP_TYPE_UNSPEC;
2290         map->sec_idx = sec_idx;
2291         map->sec_offset = vi->offset;
2292         map->btf_var_idx = var_idx;
2293         pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2294                  map_name, map->sec_idx, map->sec_offset);
2295
2296         return parse_btf_map_def(obj, map, def, strict, false, pin_root_path);
2297 }
2298
2299 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2300                                           const char *pin_root_path)
2301 {
2302         const struct btf_type *sec = NULL;
2303         int nr_types, i, vlen, err;
2304         const struct btf_type *t;
2305         const char *name;
2306         Elf_Data *data;
2307         Elf_Scn *scn;
2308
2309         if (obj->efile.btf_maps_shndx < 0)
2310                 return 0;
2311
2312         scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2313         data = elf_sec_data(obj, scn);
2314         if (!scn || !data) {
2315                 pr_warn("elf: failed to get %s map definitions for %s\n",
2316                         MAPS_ELF_SEC, obj->path);
2317                 return -EINVAL;
2318         }
2319
2320         nr_types = btf__get_nr_types(obj->btf);
2321         for (i = 1; i <= nr_types; i++) {
2322                 t = btf__type_by_id(obj->btf, i);
2323                 if (!btf_is_datasec(t))
2324                         continue;
2325                 name = btf__name_by_offset(obj->btf, t->name_off);
2326                 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2327                         sec = t;
2328                         obj->efile.btf_maps_sec_btf_id = i;
2329                         break;
2330                 }
2331         }
2332
2333         if (!sec) {
2334                 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2335                 return -ENOENT;
2336         }
2337
2338         vlen = btf_vlen(sec);
2339         for (i = 0; i < vlen; i++) {
2340                 err = bpf_object__init_user_btf_map(obj, sec, i,
2341                                                     obj->efile.btf_maps_shndx,
2342                                                     data, strict,
2343                                                     pin_root_path);
2344                 if (err)
2345                         return err;
2346         }
2347
2348         return 0;
2349 }
2350
2351 static int bpf_object__init_maps(struct bpf_object *obj,
2352                                  const struct bpf_object_open_opts *opts)
2353 {
2354         const char *pin_root_path;
2355         bool strict;
2356         int err;
2357
2358         strict = !OPTS_GET(opts, relaxed_maps, false);
2359         pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2360
2361         err = bpf_object__init_user_maps(obj, strict);
2362         err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2363         err = err ?: bpf_object__init_global_data_maps(obj);
2364         err = err ?: bpf_object__init_kconfig_map(obj);
2365         err = err ?: bpf_object__init_struct_ops_maps(obj);
2366         if (err)
2367                 return err;
2368
2369         return 0;
2370 }
2371
2372 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2373 {
2374         GElf_Shdr sh;
2375
2376         if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh))
2377                 return false;
2378
2379         return sh.sh_flags & SHF_EXECINSTR;
2380 }
2381
2382 static bool btf_needs_sanitization(struct bpf_object *obj)
2383 {
2384         bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2385         bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2386         bool has_func = kernel_supports(FEAT_BTF_FUNC);
2387
2388         return !has_func || !has_datasec || !has_func_global;
2389 }
2390
2391 static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2392 {
2393         bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2394         bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2395         bool has_func = kernel_supports(FEAT_BTF_FUNC);
2396         struct btf_type *t;
2397         int i, j, vlen;
2398
2399         for (i = 1; i <= btf__get_nr_types(btf); i++) {
2400                 t = (struct btf_type *)btf__type_by_id(btf, i);
2401
2402                 if (!has_datasec && btf_is_var(t)) {
2403                         /* replace VAR with INT */
2404                         t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2405                         /*
2406                          * using size = 1 is the safest choice, 4 will be too
2407                          * big and cause kernel BTF validation failure if
2408                          * original variable took less than 4 bytes
2409                          */
2410                         t->size = 1;
2411                         *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2412                 } else if (!has_datasec && btf_is_datasec(t)) {
2413                         /* replace DATASEC with STRUCT */
2414                         const struct btf_var_secinfo *v = btf_var_secinfos(t);
2415                         struct btf_member *m = btf_members(t);
2416                         struct btf_type *vt;
2417                         char *name;
2418
2419                         name = (char *)btf__name_by_offset(btf, t->name_off);
2420                         while (*name) {
2421                                 if (*name == '.')
2422                                         *name = '_';
2423                                 name++;
2424                         }
2425
2426                         vlen = btf_vlen(t);
2427                         t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2428                         for (j = 0; j < vlen; j++, v++, m++) {
2429                                 /* order of field assignments is important */
2430                                 m->offset = v->offset * 8;
2431                                 m->type = v->type;
2432                                 /* preserve variable name as member name */
2433                                 vt = (void *)btf__type_by_id(btf, v->type);
2434                                 m->name_off = vt->name_off;
2435                         }
2436                 } else if (!has_func && btf_is_func_proto(t)) {
2437                         /* replace FUNC_PROTO with ENUM */
2438                         vlen = btf_vlen(t);
2439                         t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2440                         t->size = sizeof(__u32); /* kernel enforced */
2441                 } else if (!has_func && btf_is_func(t)) {
2442                         /* replace FUNC with TYPEDEF */
2443                         t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2444                 } else if (!has_func_global && btf_is_func(t)) {
2445                         /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2446                         t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2447                 }
2448         }
2449 }
2450
2451 static bool libbpf_needs_btf(const struct bpf_object *obj)
2452 {
2453         return obj->efile.btf_maps_shndx >= 0 ||
2454                obj->efile.st_ops_shndx >= 0 ||
2455                obj->nr_extern > 0;
2456 }
2457
2458 static bool kernel_needs_btf(const struct bpf_object *obj)
2459 {
2460         return obj->efile.st_ops_shndx >= 0;
2461 }
2462
2463 static int bpf_object__init_btf(struct bpf_object *obj,
2464                                 Elf_Data *btf_data,
2465                                 Elf_Data *btf_ext_data)
2466 {
2467         int err = -ENOENT;
2468
2469         if (btf_data) {
2470                 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2471                 if (IS_ERR(obj->btf)) {
2472                         err = PTR_ERR(obj->btf);
2473                         obj->btf = NULL;
2474                         pr_warn("Error loading ELF section %s: %d.\n",
2475                                 BTF_ELF_SEC, err);
2476                         goto out;
2477                 }
2478                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2479                 btf__set_pointer_size(obj->btf, 8);
2480                 err = 0;
2481         }
2482         if (btf_ext_data) {
2483                 if (!obj->btf) {
2484                         pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2485                                  BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2486                         goto out;
2487                 }
2488                 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
2489                                             btf_ext_data->d_size);
2490                 if (IS_ERR(obj->btf_ext)) {
2491                         pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
2492                                 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
2493                         obj->btf_ext = NULL;
2494                         goto out;
2495                 }
2496         }
2497 out:
2498         if (err && libbpf_needs_btf(obj)) {
2499                 pr_warn("BTF is required, but is missing or corrupted.\n");
2500                 return err;
2501         }
2502         return 0;
2503 }
2504
2505 static int bpf_object__finalize_btf(struct bpf_object *obj)
2506 {
2507         int err;
2508
2509         if (!obj->btf)
2510                 return 0;
2511
2512         err = btf__finalize_data(obj, obj->btf);
2513         if (err) {
2514                 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2515                 return err;
2516         }
2517
2518         return 0;
2519 }
2520
2521 static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog)
2522 {
2523         if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2524             prog->type == BPF_PROG_TYPE_LSM)
2525                 return true;
2526
2527         /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
2528          * also need vmlinux BTF
2529          */
2530         if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2531                 return true;
2532
2533         return false;
2534 }
2535
2536 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj)
2537 {
2538         bool need_vmlinux_btf = false;
2539         struct bpf_program *prog;
2540         int i, err;
2541
2542         /* CO-RE relocations need kernel BTF */
2543         if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
2544                 need_vmlinux_btf = true;
2545
2546         /* Support for typed ksyms needs kernel BTF */
2547         for (i = 0; i < obj->nr_extern; i++) {
2548                 const struct extern_desc *ext;
2549
2550                 ext = &obj->externs[i];
2551                 if (ext->type == EXT_KSYM && ext->ksym.type_id) {
2552                         need_vmlinux_btf = true;
2553                         break;
2554                 }
2555         }
2556
2557         bpf_object__for_each_program(prog, obj) {
2558                 if (!prog->load)
2559                         continue;
2560                 if (libbpf_prog_needs_vmlinux_btf(prog)) {
2561                         need_vmlinux_btf = true;
2562                         break;
2563                 }
2564         }
2565
2566         if (!need_vmlinux_btf)
2567                 return 0;
2568
2569         obj->btf_vmlinux = libbpf_find_kernel_btf();
2570         if (IS_ERR(obj->btf_vmlinux)) {
2571                 err = PTR_ERR(obj->btf_vmlinux);
2572                 pr_warn("Error loading vmlinux BTF: %d\n", err);
2573                 obj->btf_vmlinux = NULL;
2574                 return err;
2575         }
2576         return 0;
2577 }
2578
2579 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2580 {
2581         struct btf *kern_btf = obj->btf;
2582         bool btf_mandatory, sanitize;
2583         int err = 0;
2584
2585         if (!obj->btf)
2586                 return 0;
2587
2588         if (!kernel_supports(FEAT_BTF)) {
2589                 if (kernel_needs_btf(obj)) {
2590                         err = -EOPNOTSUPP;
2591                         goto report;
2592                 }
2593                 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
2594                 return 0;
2595         }
2596
2597         sanitize = btf_needs_sanitization(obj);
2598         if (sanitize) {
2599                 const void *raw_data;
2600                 __u32 sz;
2601
2602                 /* clone BTF to sanitize a copy and leave the original intact */
2603                 raw_data = btf__get_raw_data(obj->btf, &sz);
2604                 kern_btf = btf__new(raw_data, sz);
2605                 if (IS_ERR(kern_btf))
2606                         return PTR_ERR(kern_btf);
2607
2608                 /* enforce 8-byte pointers for BPF-targeted BTFs */
2609                 btf__set_pointer_size(obj->btf, 8);
2610                 bpf_object__sanitize_btf(obj, kern_btf);
2611         }
2612
2613         err = btf__load(kern_btf);
2614         if (sanitize) {
2615                 if (!err) {
2616                         /* move fd to libbpf's BTF */
2617                         btf__set_fd(obj->btf, btf__fd(kern_btf));
2618                         btf__set_fd(kern_btf, -1);
2619                 }
2620                 btf__free(kern_btf);
2621         }
2622 report:
2623         if (err) {
2624                 btf_mandatory = kernel_needs_btf(obj);
2625                 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
2626                         btf_mandatory ? "BTF is mandatory, can't proceed."
2627                                       : "BTF is optional, ignoring.");
2628                 if (!btf_mandatory)
2629                         err = 0;
2630         }
2631         return err;
2632 }
2633
2634 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
2635 {
2636         const char *name;
2637
2638         name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
2639         if (!name) {
2640                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2641                         off, obj->path, elf_errmsg(-1));
2642                 return NULL;
2643         }
2644
2645         return name;
2646 }
2647
2648 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
2649 {
2650         const char *name;
2651
2652         name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
2653         if (!name) {
2654                 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2655                         off, obj->path, elf_errmsg(-1));
2656                 return NULL;
2657         }
2658
2659         return name;
2660 }
2661
2662 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
2663 {
2664         Elf_Scn *scn;
2665
2666         scn = elf_getscn(obj->efile.elf, idx);
2667         if (!scn) {
2668                 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
2669                         idx, obj->path, elf_errmsg(-1));
2670                 return NULL;
2671         }
2672         return scn;
2673 }
2674
2675 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
2676 {
2677         Elf_Scn *scn = NULL;
2678         Elf *elf = obj->efile.elf;
2679         const char *sec_name;
2680
2681         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2682                 sec_name = elf_sec_name(obj, scn);
2683                 if (!sec_name)
2684                         return NULL;
2685
2686                 if (strcmp(sec_name, name) != 0)
2687                         continue;
2688
2689                 return scn;
2690         }
2691         return NULL;
2692 }
2693
2694 static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr)
2695 {
2696         if (!scn)
2697                 return -EINVAL;
2698
2699         if (gelf_getshdr(scn, hdr) != hdr) {
2700                 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
2701                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2702                 return -EINVAL;
2703         }
2704
2705         return 0;
2706 }
2707
2708 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
2709 {
2710         const char *name;
2711         GElf_Shdr sh;
2712
2713         if (!scn)
2714                 return NULL;
2715
2716         if (elf_sec_hdr(obj, scn, &sh))
2717                 return NULL;
2718
2719         name = elf_sec_str(obj, sh.sh_name);
2720         if (!name) {
2721                 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
2722                         elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2723                 return NULL;
2724         }
2725
2726         return name;
2727 }
2728
2729 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
2730 {
2731         Elf_Data *data;
2732
2733         if (!scn)
2734                 return NULL;
2735
2736         data = elf_getdata(scn, 0);
2737         if (!data) {
2738                 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
2739                         elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
2740                         obj->path, elf_errmsg(-1));
2741                 return NULL;
2742         }
2743
2744         return data;
2745 }
2746
2747 static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
2748                               size_t off, __u32 sym_type, GElf_Sym *sym)
2749 {
2750         Elf_Data *symbols = obj->efile.symbols;
2751         size_t n = symbols->d_size / sizeof(GElf_Sym);
2752         int i;
2753
2754         for (i = 0; i < n; i++) {
2755                 if (!gelf_getsym(symbols, i, sym))
2756                         continue;
2757                 if (sym->st_shndx != sec_idx || sym->st_value != off)
2758                         continue;
2759                 if (GELF_ST_TYPE(sym->st_info) != sym_type)
2760                         continue;
2761                 return 0;
2762         }
2763
2764         return -ENOENT;
2765 }
2766
2767 static bool is_sec_name_dwarf(const char *name)
2768 {
2769         /* approximation, but the actual list is too long */
2770         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
2771 }
2772
2773 static bool ignore_elf_section(GElf_Shdr *hdr, const char *name)
2774 {
2775         /* no special handling of .strtab */
2776         if (hdr->sh_type == SHT_STRTAB)
2777                 return true;
2778
2779         /* ignore .llvm_addrsig section as well */
2780         if (hdr->sh_type == 0x6FFF4C03 /* SHT_LLVM_ADDRSIG */)
2781                 return true;
2782
2783         /* no subprograms will lead to an empty .text section, ignore it */
2784         if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
2785             strcmp(name, ".text") == 0)
2786                 return true;
2787
2788         /* DWARF sections */
2789         if (is_sec_name_dwarf(name))
2790                 return true;
2791
2792         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
2793                 name += sizeof(".rel") - 1;
2794                 /* DWARF section relocations */
2795                 if (is_sec_name_dwarf(name))
2796                         return true;
2797
2798                 /* .BTF and .BTF.ext don't need relocations */
2799                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
2800                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
2801                         return true;
2802         }
2803
2804         return false;
2805 }
2806
2807 static int cmp_progs(const void *_a, const void *_b)
2808 {
2809         const struct bpf_program *a = _a;
2810         const struct bpf_program *b = _b;
2811
2812         if (a->sec_idx != b->sec_idx)
2813                 return a->sec_idx < b->sec_idx ? -1 : 1;
2814
2815         /* sec_insn_off can't be the same within the section */
2816         return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
2817 }
2818
2819 static int bpf_object__elf_collect(struct bpf_object *obj)
2820 {
2821         Elf *elf = obj->efile.elf;
2822         Elf_Data *btf_ext_data = NULL;
2823         Elf_Data *btf_data = NULL;
2824         int idx = 0, err = 0;
2825         const char *name;
2826         Elf_Data *data;
2827         Elf_Scn *scn;
2828         GElf_Shdr sh;
2829
2830         /* a bunch of ELF parsing functionality depends on processing symbols,
2831          * so do the first pass and find the symbol table
2832          */
2833         scn = NULL;
2834         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2835                 if (elf_sec_hdr(obj, scn, &sh))
2836                         return -LIBBPF_ERRNO__FORMAT;
2837
2838                 if (sh.sh_type == SHT_SYMTAB) {
2839                         if (obj->efile.symbols) {
2840                                 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
2841                                 return -LIBBPF_ERRNO__FORMAT;
2842                         }
2843
2844                         data = elf_sec_data(obj, scn);
2845                         if (!data)
2846                                 return -LIBBPF_ERRNO__FORMAT;
2847
2848                         obj->efile.symbols = data;
2849                         obj->efile.symbols_shndx = elf_ndxscn(scn);
2850                         obj->efile.strtabidx = sh.sh_link;
2851                 }
2852         }
2853
2854         scn = NULL;
2855         while ((scn = elf_nextscn(elf, scn)) != NULL) {
2856                 idx++;
2857
2858                 if (elf_sec_hdr(obj, scn, &sh))
2859                         return -LIBBPF_ERRNO__FORMAT;
2860
2861                 name = elf_sec_str(obj, sh.sh_name);
2862                 if (!name)
2863                         return -LIBBPF_ERRNO__FORMAT;
2864
2865                 if (ignore_elf_section(&sh, name))
2866                         continue;
2867
2868                 data = elf_sec_data(obj, scn);
2869                 if (!data)
2870                         return -LIBBPF_ERRNO__FORMAT;
2871
2872                 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
2873                          idx, name, (unsigned long)data->d_size,
2874                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
2875                          (int)sh.sh_type);
2876
2877                 if (strcmp(name, "license") == 0) {
2878                         err = bpf_object__init_license(obj, data->d_buf, data->d_size);
2879                         if (err)
2880                                 return err;
2881                 } else if (strcmp(name, "version") == 0) {
2882                         err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
2883                         if (err)
2884                                 return err;
2885                 } else if (strcmp(name, "maps") == 0) {
2886                         obj->efile.maps_shndx = idx;
2887                 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
2888                         obj->efile.btf_maps_shndx = idx;
2889                 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
2890                         btf_data = data;
2891                 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
2892                         btf_ext_data = data;
2893                 } else if (sh.sh_type == SHT_SYMTAB) {
2894                         /* already processed during the first pass above */
2895                 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
2896                         if (sh.sh_flags & SHF_EXECINSTR) {
2897                                 if (strcmp(name, ".text") == 0)
2898                                         obj->efile.text_shndx = idx;
2899                                 err = bpf_object__add_programs(obj, data, name, idx);
2900                                 if (err)
2901                                         return err;
2902                         } else if (strcmp(name, DATA_SEC) == 0) {
2903                                 obj->efile.data = data;
2904                                 obj->efile.data_shndx = idx;
2905                         } else if (strcmp(name, RODATA_SEC) == 0) {
2906                                 obj->efile.rodata = data;
2907                                 obj->efile.rodata_shndx = idx;
2908                         } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
2909                                 obj->efile.st_ops_data = data;
2910                                 obj->efile.st_ops_shndx = idx;
2911                         } else {
2912                                 pr_info("elf: skipping unrecognized data section(%d) %s\n",
2913                                         idx, name);
2914                         }
2915                 } else if (sh.sh_type == SHT_REL) {
2916                         int nr_sects = obj->efile.nr_reloc_sects;
2917                         void *sects = obj->efile.reloc_sects;
2918                         int sec = sh.sh_info; /* points to other section */
2919
2920                         /* Only do relo for section with exec instructions */
2921                         if (!section_have_execinstr(obj, sec) &&
2922                             strcmp(name, ".rel" STRUCT_OPS_SEC) &&
2923                             strcmp(name, ".rel" MAPS_ELF_SEC)) {
2924                                 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
2925                                         idx, name, sec,
2926                                         elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>");
2927                                 continue;
2928                         }
2929
2930                         sects = libbpf_reallocarray(sects, nr_sects + 1,
2931                                                     sizeof(*obj->efile.reloc_sects));
2932                         if (!sects)
2933                                 return -ENOMEM;
2934
2935                         obj->efile.reloc_sects = sects;
2936                         obj->efile.nr_reloc_sects++;
2937
2938                         obj->efile.reloc_sects[nr_sects].shdr = sh;
2939                         obj->efile.reloc_sects[nr_sects].data = data;
2940                 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
2941                         obj->efile.bss = data;
2942                         obj->efile.bss_shndx = idx;
2943                 } else {
2944                         pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
2945                                 (size_t)sh.sh_size);
2946                 }
2947         }
2948
2949         if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
2950                 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
2951                 return -LIBBPF_ERRNO__FORMAT;
2952         }
2953
2954         /* sort BPF programs by section name and in-section instruction offset
2955          * for faster search */
2956         qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
2957
2958         return bpf_object__init_btf(obj, btf_data, btf_ext_data);
2959 }
2960
2961 static bool sym_is_extern(const GElf_Sym *sym)
2962 {
2963         int bind = GELF_ST_BIND(sym->st_info);
2964         /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
2965         return sym->st_shndx == SHN_UNDEF &&
2966                (bind == STB_GLOBAL || bind == STB_WEAK) &&
2967                GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
2968 }
2969
2970 static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
2971 {
2972         const struct btf_type *t;
2973         const char *var_name;
2974         int i, n;
2975
2976         if (!btf)
2977                 return -ESRCH;
2978
2979         n = btf__get_nr_types(btf);
2980         for (i = 1; i <= n; i++) {
2981                 t = btf__type_by_id(btf, i);
2982
2983                 if (!btf_is_var(t))
2984                         continue;
2985
2986                 var_name = btf__name_by_offset(btf, t->name_off);
2987                 if (strcmp(var_name, ext_name))
2988                         continue;
2989
2990                 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
2991                         return -EINVAL;
2992
2993                 return i;
2994         }
2995
2996         return -ENOENT;
2997 }
2998
2999 static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3000         const struct btf_var_secinfo *vs;
3001         const struct btf_type *t;
3002         int i, j, n;
3003
3004         if (!btf)
3005                 return -ESRCH;
3006
3007         n = btf__get_nr_types(btf);
3008         for (i = 1; i <= n; i++) {
3009                 t = btf__type_by_id(btf, i);
3010
3011                 if (!btf_is_datasec(t))
3012                         continue;
3013
3014                 vs = btf_var_secinfos(t);
3015                 for (j = 0; j < btf_vlen(t); j++, vs++) {
3016                         if (vs->type == ext_btf_id)
3017                                 return i;
3018                 }
3019         }
3020
3021         return -ENOENT;
3022 }
3023
3024 static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3025                                      bool *is_signed)
3026 {
3027         const struct btf_type *t;
3028         const char *name;
3029
3030         t = skip_mods_and_typedefs(btf, id, NULL);
3031         name = btf__name_by_offset(btf, t->name_off);
3032
3033         if (is_signed)
3034                 *is_signed = false;
3035         switch (btf_kind(t)) {
3036         case BTF_KIND_INT: {
3037                 int enc = btf_int_encoding(t);
3038
3039                 if (enc & BTF_INT_BOOL)
3040                         return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3041                 if (is_signed)
3042                         *is_signed = enc & BTF_INT_SIGNED;
3043                 if (t->size == 1)
3044                         return KCFG_CHAR;
3045                 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3046                         return KCFG_UNKNOWN;
3047                 return KCFG_INT;
3048         }
3049         case BTF_KIND_ENUM:
3050                 if (t->size != 4)
3051                         return KCFG_UNKNOWN;
3052                 if (strcmp(name, "libbpf_tristate"))
3053                         return KCFG_UNKNOWN;
3054                 return KCFG_TRISTATE;
3055         case BTF_KIND_ARRAY:
3056                 if (btf_array(t)->nelems == 0)
3057                         return KCFG_UNKNOWN;
3058                 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3059                         return KCFG_UNKNOWN;
3060                 return KCFG_CHAR_ARR;
3061         default:
3062                 return KCFG_UNKNOWN;
3063         }
3064 }
3065
3066 static int cmp_externs(const void *_a, const void *_b)
3067 {
3068         const struct extern_desc *a = _a;
3069         const struct extern_desc *b = _b;
3070
3071         if (a->type != b->type)
3072                 return a->type < b->type ? -1 : 1;
3073
3074         if (a->type == EXT_KCFG) {
3075                 /* descending order by alignment requirements */
3076                 if (a->kcfg.align != b->kcfg.align)
3077                         return a->kcfg.align > b->kcfg.align ? -1 : 1;
3078                 /* ascending order by size, within same alignment class */
3079                 if (a->kcfg.sz != b->kcfg.sz)
3080                         return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3081         }
3082
3083         /* resolve ties by name */
3084         return strcmp(a->name, b->name);
3085 }
3086
3087 static int find_int_btf_id(const struct btf *btf)
3088 {
3089         const struct btf_type *t;
3090         int i, n;
3091
3092         n = btf__get_nr_types(btf);
3093         for (i = 1; i <= n; i++) {
3094                 t = btf__type_by_id(btf, i);
3095
3096                 if (btf_is_int(t) && btf_int_bits(t) == 32)
3097                         return i;
3098         }
3099
3100         return 0;
3101 }
3102
3103 static int bpf_object__collect_externs(struct bpf_object *obj)
3104 {
3105         struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3106         const struct btf_type *t;
3107         struct extern_desc *ext;
3108         int i, n, off;
3109         const char *ext_name, *sec_name;
3110         Elf_Scn *scn;
3111         GElf_Shdr sh;
3112
3113         if (!obj->efile.symbols)
3114                 return 0;
3115
3116         scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3117         if (elf_sec_hdr(obj, scn, &sh))
3118                 return -LIBBPF_ERRNO__FORMAT;
3119
3120         n = sh.sh_size / sh.sh_entsize;
3121         pr_debug("looking for externs among %d symbols...\n", n);
3122
3123         for (i = 0; i < n; i++) {
3124                 GElf_Sym sym;
3125
3126                 if (!gelf_getsym(obj->efile.symbols, i, &sym))
3127                         return -LIBBPF_ERRNO__FORMAT;
3128                 if (!sym_is_extern(&sym))
3129                         continue;
3130                 ext_name = elf_sym_str(obj, sym.st_name);
3131                 if (!ext_name || !ext_name[0])
3132                         continue;
3133
3134                 ext = obj->externs;
3135                 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3136                 if (!ext)
3137                         return -ENOMEM;
3138                 obj->externs = ext;
3139                 ext = &ext[obj->nr_extern];
3140                 memset(ext, 0, sizeof(*ext));
3141                 obj->nr_extern++;
3142
3143                 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3144                 if (ext->btf_id <= 0) {
3145                         pr_warn("failed to find BTF for extern '%s': %d\n",
3146                                 ext_name, ext->btf_id);
3147                         return ext->btf_id;
3148                 }
3149                 t = btf__type_by_id(obj->btf, ext->btf_id);
3150                 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3151                 ext->sym_idx = i;
3152                 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
3153
3154                 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3155                 if (ext->sec_btf_id <= 0) {
3156                         pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3157                                 ext_name, ext->btf_id, ext->sec_btf_id);
3158                         return ext->sec_btf_id;
3159                 }
3160                 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3161                 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3162
3163                 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3164                         kcfg_sec = sec;
3165                         ext->type = EXT_KCFG;
3166                         ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3167                         if (ext->kcfg.sz <= 0) {
3168                                 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3169                                         ext_name, ext->kcfg.sz);
3170                                 return ext->kcfg.sz;
3171                         }
3172                         ext->kcfg.align = btf__align_of(obj->btf, t->type);
3173                         if (ext->kcfg.align <= 0) {
3174                                 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3175                                         ext_name, ext->kcfg.align);
3176                                 return -EINVAL;
3177                         }
3178                         ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3179                                                         &ext->kcfg.is_signed);
3180                         if (ext->kcfg.type == KCFG_UNKNOWN) {
3181                                 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name);
3182                                 return -ENOTSUP;
3183                         }
3184                 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3185                         ksym_sec = sec;
3186                         ext->type = EXT_KSYM;
3187                         skip_mods_and_typedefs(obj->btf, t->type,
3188                                                &ext->ksym.type_id);
3189                 } else {
3190                         pr_warn("unrecognized extern section '%s'\n", sec_name);
3191                         return -ENOTSUP;
3192                 }
3193         }
3194         pr_debug("collected %d externs total\n", obj->nr_extern);
3195
3196         if (!obj->nr_extern)
3197                 return 0;
3198
3199         /* sort externs by type, for kcfg ones also by (align, size, name) */
3200         qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3201
3202         /* for .ksyms section, we need to turn all externs into allocated
3203          * variables in BTF to pass kernel verification; we do this by
3204          * pretending that each extern is a 8-byte variable
3205          */
3206         if (ksym_sec) {
3207                 /* find existing 4-byte integer type in BTF to use for fake
3208                  * extern variables in DATASEC
3209                  */
3210                 int int_btf_id = find_int_btf_id(obj->btf);
3211
3212                 for (i = 0; i < obj->nr_extern; i++) {
3213                         ext = &obj->externs[i];
3214                         if (ext->type != EXT_KSYM)
3215                                 continue;
3216                         pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3217                                  i, ext->sym_idx, ext->name);
3218                 }
3219
3220                 sec = ksym_sec;
3221                 n = btf_vlen(sec);
3222                 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3223                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3224                         struct btf_type *vt;
3225
3226                         vt = (void *)btf__type_by_id(obj->btf, vs->type);
3227                         ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3228                         ext = find_extern_by_name(obj, ext_name);
3229                         if (!ext) {
3230                                 pr_warn("failed to find extern definition for BTF var '%s'\n",
3231                                         ext_name);
3232                                 return -ESRCH;
3233                         }
3234                         btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3235                         vt->type = int_btf_id;
3236                         vs->offset = off;
3237                         vs->size = sizeof(int);
3238                 }
3239                 sec->size = off;
3240         }
3241
3242         if (kcfg_sec) {
3243                 sec = kcfg_sec;
3244                 /* for kcfg externs calculate their offsets within a .kconfig map */
3245                 off = 0;
3246                 for (i = 0; i < obj->nr_extern; i++) {
3247                         ext = &obj->externs[i];
3248                         if (ext->type != EXT_KCFG)
3249                                 continue;
3250
3251                         ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3252                         off = ext->kcfg.data_off + ext->kcfg.sz;
3253                         pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3254                                  i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3255                 }
3256                 sec->size = off;
3257                 n = btf_vlen(sec);
3258                 for (i = 0; i < n; i++) {
3259                         struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3260
3261                         t = btf__type_by_id(obj->btf, vs->type);
3262                         ext_name = btf__name_by_offset(obj->btf, t->name_off);
3263                         ext = find_extern_by_name(obj, ext_name);
3264                         if (!ext) {
3265                                 pr_warn("failed to find extern definition for BTF var '%s'\n",
3266                                         ext_name);
3267                                 return -ESRCH;
3268                         }
3269                         btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3270                         vs->offset = ext->kcfg.data_off;
3271                 }
3272         }
3273         return 0;
3274 }
3275
3276 struct bpf_program *
3277 bpf_object__find_program_by_title(const struct bpf_object *obj,
3278                                   const char *title)
3279 {
3280         struct bpf_program *pos;
3281
3282         bpf_object__for_each_program(pos, obj) {
3283                 if (pos->sec_name && !strcmp(pos->sec_name, title))
3284                         return pos;
3285         }
3286         return NULL;
3287 }
3288
3289 static bool prog_is_subprog(const struct bpf_object *obj,
3290                             const struct bpf_program *prog)
3291 {
3292         /* For legacy reasons, libbpf supports an entry-point BPF programs
3293          * without SEC() attribute, i.e., those in the .text section. But if
3294          * there are 2 or more such programs in the .text section, they all
3295          * must be subprograms called from entry-point BPF programs in
3296          * designated SEC()'tions, otherwise there is no way to distinguish
3297          * which of those programs should be loaded vs which are a subprogram.
3298          * Similarly, if there is a function/program in .text and at least one
3299          * other BPF program with custom SEC() attribute, then we just assume
3300          * .text programs are subprograms (even if they are not called from
3301          * other programs), because libbpf never explicitly supported mixing
3302          * SEC()-designated BPF programs and .text entry-point BPF programs.
3303          */
3304         return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3305 }
3306
3307 struct bpf_program *
3308 bpf_object__find_program_by_name(const struct bpf_object *obj,
3309                                  const char *name)
3310 {
3311         struct bpf_program *prog;
3312
3313         bpf_object__for_each_program(prog, obj) {
3314                 if (prog_is_subprog(obj, prog))
3315                         continue;
3316                 if (!strcmp(prog->name, name))
3317                         return prog;
3318         }
3319         return NULL;
3320 }
3321
3322 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3323                                       int shndx)
3324 {
3325         return shndx == obj->efile.data_shndx ||
3326                shndx == obj->efile.bss_shndx ||
3327                shndx == obj->efile.rodata_shndx;
3328 }
3329
3330 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3331                                       int shndx)
3332 {
3333         return shndx == obj->efile.maps_shndx ||
3334                shndx == obj->efile.btf_maps_shndx;
3335 }
3336
3337 static enum libbpf_map_type
3338 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3339 {
3340         if (shndx == obj->efile.data_shndx)
3341                 return LIBBPF_MAP_DATA;
3342         else if (shndx == obj->efile.bss_shndx)
3343                 return LIBBPF_MAP_BSS;
3344         else if (shndx == obj->efile.rodata_shndx)
3345                 return LIBBPF_MAP_RODATA;
3346         else if (shndx == obj->efile.symbols_shndx)
3347                 return LIBBPF_MAP_KCONFIG;
3348         else
3349                 return LIBBPF_MAP_UNSPEC;
3350 }
3351
3352 static int bpf_program__record_reloc(struct bpf_program *prog,
3353                                      struct reloc_desc *reloc_desc,
3354                                      __u32 insn_idx, const char *sym_name,
3355                                      const GElf_Sym *sym, const GElf_Rel *rel)
3356 {
3357         struct bpf_insn *insn = &prog->insns[insn_idx];
3358         size_t map_idx, nr_maps = prog->obj->nr_maps;
3359         struct bpf_object *obj = prog->obj;
3360         __u32 shdr_idx = sym->st_shndx;
3361         enum libbpf_map_type type;
3362         const char *sym_sec_name;
3363         struct bpf_map *map;
3364
3365         reloc_desc->processed = false;
3366
3367         /* sub-program call relocation */
3368         if (insn->code == (BPF_JMP | BPF_CALL)) {
3369                 if (insn->src_reg != BPF_PSEUDO_CALL) {
3370                         pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3371                         return -LIBBPF_ERRNO__RELOC;
3372                 }
3373                 /* text_shndx can be 0, if no default "main" program exists */
3374                 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3375                         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3376                         pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3377                                 prog->name, sym_name, sym_sec_name);
3378                         return -LIBBPF_ERRNO__RELOC;
3379                 }
3380                 if (sym->st_value % BPF_INSN_SZ) {
3381                         pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3382                                 prog->name, sym_name, (size_t)sym->st_value);
3383                         return -LIBBPF_ERRNO__RELOC;
3384                 }
3385                 reloc_desc->type = RELO_CALL;
3386                 reloc_desc->insn_idx = insn_idx;
3387                 reloc_desc->sym_off = sym->st_value;
3388                 return 0;
3389         }
3390
3391         if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
3392                 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3393                         prog->name, sym_name, insn_idx, insn->code);
3394                 return -LIBBPF_ERRNO__RELOC;
3395         }
3396
3397         if (sym_is_extern(sym)) {
3398                 int sym_idx = GELF_R_SYM(rel->r_info);
3399                 int i, n = obj->nr_extern;
3400                 struct extern_desc *ext;
3401
3402                 for (i = 0; i < n; i++) {
3403                         ext = &obj->externs[i];
3404                         if (ext->sym_idx == sym_idx)
3405                                 break;
3406                 }
3407                 if (i >= n) {
3408                         pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3409                                 prog->name, sym_name, sym_idx);
3410                         return -LIBBPF_ERRNO__RELOC;
3411                 }
3412                 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3413                          prog->name, i, ext->name, ext->sym_idx, insn_idx);
3414                 reloc_desc->type = RELO_EXTERN;
3415                 reloc_desc->insn_idx = insn_idx;
3416                 reloc_desc->sym_off = i; /* sym_off stores extern index */
3417                 return 0;
3418         }
3419
3420         if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
3421                 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
3422                         prog->name, sym_name, shdr_idx);
3423                 return -LIBBPF_ERRNO__RELOC;
3424         }
3425
3426         type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
3427         sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3428
3429         /* generic map reference relocation */
3430         if (type == LIBBPF_MAP_UNSPEC) {
3431                 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
3432                         pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
3433                                 prog->name, sym_name, sym_sec_name);
3434                         return -LIBBPF_ERRNO__RELOC;
3435                 }
3436                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3437                         map = &obj->maps[map_idx];
3438                         if (map->libbpf_type != type ||
3439                             map->sec_idx != sym->st_shndx ||
3440                             map->sec_offset != sym->st_value)
3441                                 continue;
3442                         pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
3443                                  prog->name, map_idx, map->name, map->sec_idx,
3444                                  map->sec_offset, insn_idx);
3445                         break;
3446                 }
3447                 if (map_idx >= nr_maps) {
3448                         pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
3449                                 prog->name, sym_sec_name, (size_t)sym->st_value);
3450                         return -LIBBPF_ERRNO__RELOC;
3451                 }
3452                 reloc_desc->type = RELO_LD64;
3453                 reloc_desc->insn_idx = insn_idx;
3454                 reloc_desc->map_idx = map_idx;
3455                 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
3456                 return 0;
3457         }
3458
3459         /* global data map relocation */
3460         if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
3461                 pr_warn("prog '%s': bad data relo against section '%s'\n",
3462                         prog->name, sym_sec_name);
3463                 return -LIBBPF_ERRNO__RELOC;
3464         }
3465         for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3466                 map = &obj->maps[map_idx];
3467                 if (map->libbpf_type != type)
3468                         continue;
3469                 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
3470                          prog->name, map_idx, map->name, map->sec_idx,
3471                          map->sec_offset, insn_idx);
3472                 break;
3473         }
3474         if (map_idx >= nr_maps) {
3475                 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
3476                         prog->name, sym_sec_name);
3477                 return -LIBBPF_ERRNO__RELOC;
3478         }
3479
3480         reloc_desc->type = RELO_DATA;
3481         reloc_desc->insn_idx = insn_idx;
3482         reloc_desc->map_idx = map_idx;
3483         reloc_desc->sym_off = sym->st_value;
3484         return 0;
3485 }
3486
3487 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
3488 {
3489         return insn_idx >= prog->sec_insn_off &&
3490                insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
3491 }
3492
3493 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
3494                                                  size_t sec_idx, size_t insn_idx)
3495 {
3496         int l = 0, r = obj->nr_programs - 1, m;
3497         struct bpf_program *prog;
3498
3499         while (l < r) {
3500                 m = l + (r - l + 1) / 2;
3501                 prog = &obj->programs[m];
3502
3503                 if (prog->sec_idx < sec_idx ||
3504                     (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
3505                         l = m;
3506                 else
3507                         r = m - 1;
3508         }
3509         /* matching program could be at index l, but it still might be the
3510          * wrong one, so we need to double check conditions for the last time
3511          */
3512         prog = &obj->programs[l];
3513         if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
3514                 return prog;
3515         return NULL;
3516 }
3517
3518 static int
3519 bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data)
3520 {
3521         Elf_Data *symbols = obj->efile.symbols;
3522         const char *relo_sec_name, *sec_name;
3523         size_t sec_idx = shdr->sh_info;
3524         struct bpf_program *prog;
3525         struct reloc_desc *relos;
3526         int err, i, nrels;
3527         const char *sym_name;
3528         __u32 insn_idx;
3529         GElf_Sym sym;
3530         GElf_Rel rel;
3531
3532         relo_sec_name = elf_sec_str(obj, shdr->sh_name);
3533         sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
3534         if (!relo_sec_name || !sec_name)
3535                 return -EINVAL;
3536
3537         pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
3538                  relo_sec_name, sec_idx, sec_name);
3539         nrels = shdr->sh_size / shdr->sh_entsize;
3540
3541         for (i = 0; i < nrels; i++) {
3542                 if (!gelf_getrel(data, i, &rel)) {
3543                         pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
3544                         return -LIBBPF_ERRNO__FORMAT;
3545                 }
3546                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3547                         pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n",
3548                                 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3549                         return -LIBBPF_ERRNO__FORMAT;
3550                 }
3551                 if (rel.r_offset % BPF_INSN_SZ) {
3552                         pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
3553                                 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3554                         return -LIBBPF_ERRNO__FORMAT;
3555                 }
3556
3557                 insn_idx = rel.r_offset / BPF_INSN_SZ;
3558                 /* relocations against static functions are recorded as
3559                  * relocations against the section that contains a function;
3560                  * in such case, symbol will be STT_SECTION and sym.st_name
3561                  * will point to empty string (0), so fetch section name
3562                  * instead
3563                  */
3564                 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0)
3565                         sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx));
3566                 else
3567                         sym_name = elf_sym_str(obj, sym.st_name);
3568                 sym_name = sym_name ?: "<?";
3569
3570                 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
3571                          relo_sec_name, i, insn_idx, sym_name);
3572
3573                 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
3574                 if (!prog) {
3575                         pr_warn("sec '%s': relo #%d: program not found in section '%s' for insn #%u\n",
3576                                 relo_sec_name, i, sec_name, insn_idx);
3577                         return -LIBBPF_ERRNO__RELOC;
3578                 }
3579
3580                 relos = libbpf_reallocarray(prog->reloc_desc,
3581                                             prog->nr_reloc + 1, sizeof(*relos));
3582                 if (!relos)
3583                         return -ENOMEM;
3584                 prog->reloc_desc = relos;
3585
3586                 /* adjust insn_idx to local BPF program frame of reference */
3587                 insn_idx -= prog->sec_insn_off;
3588                 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
3589                                                 insn_idx, sym_name, &sym, &rel);
3590                 if (err)
3591                         return err;
3592
3593                 prog->nr_reloc++;
3594         }
3595         return 0;
3596 }
3597
3598 static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3599 {
3600         struct bpf_map_def *def = &map->def;
3601         __u32 key_type_id = 0, value_type_id = 0;
3602         int ret;
3603
3604         /* if it's BTF-defined map, we don't need to search for type IDs.
3605          * For struct_ops map, it does not need btf_key_type_id and
3606          * btf_value_type_id.
3607          */
3608         if (map->sec_idx == obj->efile.btf_maps_shndx ||
3609             bpf_map__is_struct_ops(map))
3610                 return 0;
3611
3612         if (!bpf_map__is_internal(map)) {
3613                 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3614                                            def->value_size, &key_type_id,
3615                                            &value_type_id);
3616         } else {
3617                 /*
3618                  * LLVM annotates global data differently in BTF, that is,
3619                  * only as '.data', '.bss' or '.rodata'.
3620                  */
3621                 ret = btf__find_by_name(obj->btf,
3622                                 libbpf_type_to_btf_name[map->libbpf_type]);
3623         }
3624         if (ret < 0)
3625                 return ret;
3626
3627         map->btf_key_type_id = key_type_id;
3628         map->btf_value_type_id = bpf_map__is_internal(map) ?
3629                                  ret : value_type_id;
3630         return 0;
3631 }
3632
3633 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3634 {
3635         struct bpf_map_info info = {};
3636         __u32 len = sizeof(info);
3637         int new_fd, err;
3638         char *new_name;
3639
3640         err = bpf_obj_get_info_by_fd(fd, &info, &len);
3641         if (err)
3642                 return err;
3643
3644         new_name = strdup(info.name);
3645         if (!new_name)
3646                 return -errno;
3647
3648         new_fd = open("/", O_RDONLY | O_CLOEXEC);
3649         if (new_fd < 0) {
3650                 err = -errno;
3651                 goto err_free_new_name;
3652         }
3653
3654         new_fd = dup3(fd, new_fd, O_CLOEXEC);
3655         if (new_fd < 0) {
3656                 err = -errno;
3657                 goto err_close_new_fd;
3658         }
3659
3660         err = zclose(map->fd);
3661         if (err) {
3662                 err = -errno;
3663                 goto err_close_new_fd;
3664         }
3665         free(map->name);
3666
3667         map->fd = new_fd;
3668         map->name = new_name;
3669         map->def.type = info.type;
3670         map->def.key_size = info.key_size;
3671         map->def.value_size = info.value_size;
3672         map->def.max_entries = info.max_entries;
3673         map->def.map_flags = info.map_flags;
3674         map->btf_key_type_id = info.btf_key_type_id;
3675         map->btf_value_type_id = info.btf_value_type_id;
3676         map->reused = true;
3677
3678         return 0;
3679
3680 err_close_new_fd:
3681         close(new_fd);
3682 err_free_new_name:
3683         free(new_name);
3684         return err;
3685 }
3686
3687 __u32 bpf_map__max_entries(const struct bpf_map *map)
3688 {
3689         return map->def.max_entries;
3690 }
3691
3692 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
3693 {
3694         if (map->fd >= 0)
3695                 return -EBUSY;
3696         map->def.max_entries = max_entries;
3697         return 0;
3698 }
3699
3700 int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
3701 {
3702         if (!map || !max_entries)
3703                 return -EINVAL;
3704
3705         return bpf_map__set_max_entries(map, max_entries);
3706 }
3707
3708 static int
3709 bpf_object__probe_loading(struct bpf_object *obj)
3710 {
3711         struct bpf_load_program_attr attr;
3712         char *cp, errmsg[STRERR_BUFSIZE];
3713         struct bpf_insn insns[] = {
3714                 BPF_MOV64_IMM(BPF_REG_0, 0),
3715                 BPF_EXIT_INSN(),
3716         };
3717         int ret;
3718
3719         /* make sure basic loading works */
3720
3721         memset(&attr, 0, sizeof(attr));
3722         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3723         attr.insns = insns;
3724         attr.insns_cnt = ARRAY_SIZE(insns);
3725         attr.license = "GPL";
3726
3727         ret = bpf_load_program_xattr(&attr, NULL, 0);
3728         if (ret < 0) {
3729                 ret = errno;
3730                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3731                 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
3732                         "program. Make sure your kernel supports BPF "
3733                         "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
3734                         "set to big enough value.\n", __func__, cp, ret);
3735                 return -ret;
3736         }
3737         close(ret);
3738
3739         return 0;
3740 }
3741
3742 static int probe_fd(int fd)
3743 {
3744         if (fd >= 0)
3745                 close(fd);
3746         return fd >= 0;
3747 }
3748
3749 static int probe_kern_prog_name(void)
3750 {
3751         struct bpf_load_program_attr attr;
3752         struct bpf_insn insns[] = {
3753                 BPF_MOV64_IMM(BPF_REG_0, 0),
3754                 BPF_EXIT_INSN(),
3755         };
3756         int ret;
3757
3758         /* make sure loading with name works */
3759
3760         memset(&attr, 0, sizeof(attr));
3761         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3762         attr.insns = insns;
3763         attr.insns_cnt = ARRAY_SIZE(insns);
3764         attr.license = "GPL";
3765         attr.name = "test";
3766         ret = bpf_load_program_xattr(&attr, NULL, 0);
3767         return probe_fd(ret);
3768 }
3769
3770 static int probe_kern_global_data(void)
3771 {
3772         struct bpf_load_program_attr prg_attr;
3773         struct bpf_create_map_attr map_attr;
3774         char *cp, errmsg[STRERR_BUFSIZE];
3775         struct bpf_insn insns[] = {
3776                 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
3777                 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
3778                 BPF_MOV64_IMM(BPF_REG_0, 0),
3779                 BPF_EXIT_INSN(),
3780         };
3781         int ret, map;
3782
3783         memset(&map_attr, 0, sizeof(map_attr));
3784         map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3785         map_attr.key_size = sizeof(int);
3786         map_attr.value_size = 32;
3787         map_attr.max_entries = 1;
3788
3789         map = bpf_create_map_xattr(&map_attr);
3790         if (map < 0) {
3791                 ret = -errno;
3792                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3793                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3794                         __func__, cp, -ret);
3795                 return ret;
3796         }
3797
3798         insns[0].imm = map;
3799
3800         memset(&prg_attr, 0, sizeof(prg_attr));
3801         prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3802         prg_attr.insns = insns;
3803         prg_attr.insns_cnt = ARRAY_SIZE(insns);
3804         prg_attr.license = "GPL";
3805
3806         ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
3807         close(map);
3808         return probe_fd(ret);
3809 }
3810
3811 static int probe_kern_btf(void)
3812 {
3813         static const char strs[] = "\0int";
3814         __u32 types[] = {
3815                 /* int */
3816                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3817         };
3818
3819         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3820                                              strs, sizeof(strs)));
3821 }
3822
3823 static int probe_kern_btf_func(void)
3824 {
3825         static const char strs[] = "\0int\0x\0a";
3826         /* void x(int a) {} */
3827         __u32 types[] = {
3828                 /* int */
3829                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
3830                 /* FUNC_PROTO */                                /* [2] */
3831                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3832                 BTF_PARAM_ENC(7, 1),
3833                 /* FUNC x */                                    /* [3] */
3834                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
3835         };
3836
3837         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3838                                              strs, sizeof(strs)));
3839 }
3840
3841 static int probe_kern_btf_func_global(void)
3842 {
3843         static const char strs[] = "\0int\0x\0a";
3844         /* static void x(int a) {} */
3845         __u32 types[] = {
3846                 /* int */
3847                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
3848                 /* FUNC_PROTO */                                /* [2] */
3849                 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3850                 BTF_PARAM_ENC(7, 1),
3851                 /* FUNC x BTF_FUNC_GLOBAL */                    /* [3] */
3852                 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
3853         };
3854
3855         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3856                                              strs, sizeof(strs)));
3857 }
3858
3859 static int probe_kern_btf_datasec(void)
3860 {
3861         static const char strs[] = "\0x\0.data";
3862         /* static int a; */
3863         __u32 types[] = {
3864                 /* int */
3865                 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
3866                 /* VAR x */                                     /* [2] */
3867                 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
3868                 BTF_VAR_STATIC,
3869                 /* DATASEC val */                               /* [3] */
3870                 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
3871                 BTF_VAR_SECINFO_ENC(2, 0, 4),
3872         };
3873
3874         return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3875                                              strs, sizeof(strs)));
3876 }
3877
3878 static int probe_kern_array_mmap(void)
3879 {
3880         struct bpf_create_map_attr attr = {
3881                 .map_type = BPF_MAP_TYPE_ARRAY,
3882                 .map_flags = BPF_F_MMAPABLE,
3883                 .key_size = sizeof(int),
3884                 .value_size = sizeof(int),
3885                 .max_entries = 1,
3886         };
3887
3888         return probe_fd(bpf_create_map_xattr(&attr));
3889 }
3890
3891 static int probe_kern_exp_attach_type(void)
3892 {
3893         struct bpf_load_program_attr attr;
3894         struct bpf_insn insns[] = {
3895                 BPF_MOV64_IMM(BPF_REG_0, 0),
3896                 BPF_EXIT_INSN(),
3897         };
3898
3899         memset(&attr, 0, sizeof(attr));
3900         /* use any valid combination of program type and (optional)
3901          * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
3902          * to see if kernel supports expected_attach_type field for
3903          * BPF_PROG_LOAD command
3904          */
3905         attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK;
3906         attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE;
3907         attr.insns = insns;
3908         attr.insns_cnt = ARRAY_SIZE(insns);
3909         attr.license = "GPL";
3910
3911         return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
3912 }
3913
3914 static int probe_kern_probe_read_kernel(void)
3915 {
3916         struct bpf_load_program_attr attr;
3917         struct bpf_insn insns[] = {
3918                 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),   /* r1 = r10 (fp) */
3919                 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),  /* r1 += -8 */
3920                 BPF_MOV64_IMM(BPF_REG_2, 8),            /* r2 = 8 */
3921                 BPF_MOV64_IMM(BPF_REG_3, 0),            /* r3 = 0 */
3922                 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
3923                 BPF_EXIT_INSN(),
3924         };
3925
3926         memset(&attr, 0, sizeof(attr));
3927         attr.prog_type = BPF_PROG_TYPE_KPROBE;
3928         attr.insns = insns;
3929         attr.insns_cnt = ARRAY_SIZE(insns);
3930         attr.license = "GPL";
3931
3932         return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
3933 }
3934
3935 static int probe_prog_bind_map(void)
3936 {
3937         struct bpf_load_program_attr prg_attr;
3938         struct bpf_create_map_attr map_attr;
3939         char *cp, errmsg[STRERR_BUFSIZE];
3940         struct bpf_insn insns[] = {
3941                 BPF_MOV64_IMM(BPF_REG_0, 0),
3942                 BPF_EXIT_INSN(),
3943         };
3944         int ret, map, prog;
3945
3946         memset(&map_attr, 0, sizeof(map_attr));
3947         map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3948         map_attr.key_size = sizeof(int);
3949         map_attr.value_size = 32;
3950         map_attr.max_entries = 1;
3951
3952         map = bpf_create_map_xattr(&map_attr);
3953         if (map < 0) {
3954                 ret = -errno;
3955                 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3956                 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3957                         __func__, cp, -ret);
3958                 return ret;
3959         }
3960
3961         memset(&prg_attr, 0, sizeof(prg_attr));
3962         prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3963         prg_attr.insns = insns;
3964         prg_attr.insns_cnt = ARRAY_SIZE(insns);
3965         prg_attr.license = "GPL";
3966
3967         prog = bpf_load_program_xattr(&prg_attr, NULL, 0);
3968         if (prog < 0) {
3969                 close(map);
3970                 return 0;
3971         }
3972
3973         ret = bpf_prog_bind_map(prog, map, NULL);
3974
3975         close(map);
3976         close(prog);
3977
3978         return ret >= 0;
3979 }
3980
3981 static int probe_module_btf(void)
3982 {
3983         static const char strs[] = "\0int";
3984         __u32 types[] = {
3985                 /* int */
3986                 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3987         };
3988         struct bpf_btf_info info;
3989         __u32 len = sizeof(info);
3990         char name[16];
3991         int fd, err;
3992
3993         fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
3994         if (fd < 0)
3995                 return 0; /* BTF not supported at all */
3996
3997         memset(&info, 0, sizeof(info));
3998         info.name = ptr_to_u64(name);
3999         info.name_len = sizeof(name);
4000
4001         /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4002          * kernel's module BTF support coincides with support for
4003          * name/name_len fields in struct bpf_btf_info.
4004          */
4005         err = bpf_obj_get_info_by_fd(fd, &info, &len);
4006         close(fd);
4007         return !err;
4008 }
4009
4010 enum kern_feature_result {
4011         FEAT_UNKNOWN = 0,
4012         FEAT_SUPPORTED = 1,
4013         FEAT_MISSING = 2,
4014 };
4015
4016 typedef int (*feature_probe_fn)(void);
4017
4018 static struct kern_feature_desc {
4019         const char *desc;
4020         feature_probe_fn probe;
4021         enum kern_feature_result res;
4022 } feature_probes[__FEAT_CNT] = {
4023         [FEAT_PROG_NAME] = {
4024                 "BPF program name", probe_kern_prog_name,
4025         },
4026         [FEAT_GLOBAL_DATA] = {
4027                 "global variables", probe_kern_global_data,
4028         },
4029         [FEAT_BTF] = {
4030                 "minimal BTF", probe_kern_btf,
4031         },
4032         [FEAT_BTF_FUNC] = {
4033                 "BTF functions", probe_kern_btf_func,
4034         },
4035         [FEAT_BTF_GLOBAL_FUNC] = {
4036                 "BTF global function", probe_kern_btf_func_global,
4037         },
4038         [FEAT_BTF_DATASEC] = {
4039                 "BTF data section and variable", probe_kern_btf_datasec,
4040         },
4041         [FEAT_ARRAY_MMAP] = {
4042                 "ARRAY map mmap()", probe_kern_array_mmap,
4043         },
4044         [FEAT_EXP_ATTACH_TYPE] = {
4045                 "BPF_PROG_LOAD expected_attach_type attribute",
4046                 probe_kern_exp_attach_type,
4047         },
4048         [FEAT_PROBE_READ_KERN] = {
4049                 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4050         },
4051         [FEAT_PROG_BIND_MAP] = {
4052                 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4053         },
4054         [FEAT_MODULE_BTF] = {
4055                 "module BTF support", probe_module_btf,
4056         },
4057 };
4058
4059 static bool kernel_supports(enum kern_feature_id feat_id)
4060 {
4061         struct kern_feature_desc *feat = &feature_probes[feat_id];
4062         int ret;
4063
4064         if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4065                 ret = feat->probe();
4066                 if (ret > 0) {
4067                         WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4068                 } else if (ret == 0) {
4069                         WRITE_ONCE(feat->res, FEAT_MISSING);
4070                 } else {
4071                         pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4072                         WRITE_ONCE(feat->res, FEAT_MISSING);
4073                 }
4074         }
4075
4076         return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4077 }
4078
4079 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4080 {
4081         struct bpf_map_info map_info = {};
4082         char msg[STRERR_BUFSIZE];
4083         __u32 map_info_len;
4084
4085         map_info_len = sizeof(map_info);
4086
4087         if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
4088                 pr_warn("failed to get map info for map FD %d: %s\n",
4089                         map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
4090                 return false;
4091         }
4092
4093         return (map_info.type == map->def.type &&
4094                 map_info.key_size == map->def.key_size &&
4095                 map_info.value_size == map->def.value_size &&
4096                 map_info.max_entries == map->def.max_entries &&
4097                 map_info.map_flags == map->def.map_flags);
4098 }
4099
4100 static int
4101 bpf_object__reuse_map(struct bpf_map *map)
4102 {
4103         char *cp, errmsg[STRERR_BUFSIZE];
4104         int err, pin_fd;
4105
4106         pin_fd = bpf_obj_get(map->pin_path);
4107         if (pin_fd < 0) {
4108                 err = -errno;
4109                 if (err == -ENOENT) {
4110                         pr_debug("found no pinned map to reuse at '%s'\n",
4111                                  map->pin_path);
4112                         return 0;
4113                 }
4114
4115                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4116                 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4117                         map->pin_path, cp);
4118                 return err;
4119         }
4120
4121         if (!map_is_reuse_compat(map, pin_fd)) {
4122                 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4123                         map->pin_path);
4124                 close(pin_fd);
4125                 return -EINVAL;
4126         }
4127
4128         err = bpf_map__reuse_fd(map, pin_fd);
4129         if (err) {
4130                 close(pin_fd);
4131                 return err;
4132         }
4133         map->pinned = true;
4134         pr_debug("reused pinned map at '%s'\n", map->pin_path);
4135
4136         return 0;
4137 }
4138
4139 static int
4140 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4141 {
4142         enum libbpf_map_type map_type = map->libbpf_type;
4143         char *cp, errmsg[STRERR_BUFSIZE];
4144         int err, zero = 0;
4145
4146         err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4147         if (err) {
4148                 err = -errno;
4149                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4150                 pr_warn("Error setting initial map(%s) contents: %s\n",
4151                         map->name, cp);
4152                 return err;
4153         }
4154
4155         /* Freeze .rodata and .kconfig map as read-only from syscall side. */
4156         if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4157                 err = bpf_map_freeze(map->fd);
4158                 if (err) {
4159                         err = -errno;
4160                         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4161                         pr_warn("Error freezing map(%s) as read-only: %s\n",
4162                                 map->name, cp);
4163                         return err;
4164                 }
4165         }
4166         return 0;
4167 }
4168
4169 static void bpf_map__destroy(struct bpf_map *map);
4170
4171 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map)
4172 {
4173         struct bpf_create_map_attr create_attr;
4174         struct bpf_map_def *def = &map->def;
4175
4176         memset(&create_attr, 0, sizeof(create_attr));
4177
4178         if (kernel_supports(FEAT_PROG_NAME))
4179                 create_attr.name = map->name;
4180         create_attr.map_ifindex = map->map_ifindex;
4181         create_attr.map_type = def->type;
4182         create_attr.map_flags = def->map_flags;
4183         create_attr.key_size = def->key_size;
4184         create_attr.value_size = def->value_size;
4185         create_attr.numa_node = map->numa_node;
4186
4187         if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) {
4188                 int nr_cpus;
4189
4190                 nr_cpus = libbpf_num_possible_cpus();
4191                 if (nr_cpus < 0) {
4192                         pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
4193                                 map->name, nr_cpus);
4194                         return nr_cpus;
4195                 }
4196                 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
4197                 create_attr.max_entries = nr_cpus;
4198         } else {
4199                 create_attr.max_entries = def->max_entries;
4200         }
4201
4202         if (bpf_map__is_struct_ops(map))
4203                 create_attr.btf_vmlinux_value_type_id =
4204                         map->btf_vmlinux_value_type_id;
4205
4206         create_attr.btf_fd = 0;
4207         create_attr.btf_key_type_id = 0;
4208         create_attr.btf_value_type_id = 0;
4209         if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) {
4210                 create_attr.btf_fd = btf__fd(obj->btf);
4211                 create_attr.btf_key_type_id = map->btf_key_type_id;
4212                 create_attr.btf_value_type_id = map->btf_value_type_id;
4213         }
4214
4215         if (bpf_map_type__is_map_in_map(def->type)) {
4216                 if (map->inner_map) {
4217                         int err;
4218
4219                         err = bpf_object__create_map(obj, map->inner_map);
4220                         if (err) {
4221                                 pr_warn("map '%s': failed to create inner map: %d\n",
4222                                         map->name, err);
4223                                 return err;
4224                         }
4225                         map->inner_map_fd = bpf_map__fd(map->inner_map);
4226                 }
4227                 if (map->inner_map_fd >= 0)
4228                         create_attr.inner_map_fd = map->inner_map_fd;
4229         }
4230
4231         map->fd = bpf_create_map_xattr(&create_attr);
4232         if (map->fd < 0 && (create_attr.btf_key_type_id ||
4233                             create_attr.btf_value_type_id)) {
4234                 char *cp, errmsg[STRERR_BUFSIZE];
4235                 int err = -errno;
4236
4237                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4238                 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
4239                         map->name, cp, err);
4240                 create_attr.btf_fd = 0;
4241                 create_attr.btf_key_type_id = 0;
4242                 create_attr.btf_value_type_id = 0;
4243                 map->btf_key_type_id = 0;
4244                 map->btf_value_type_id = 0;
4245                 map->fd = bpf_create_map_xattr(&create_attr);
4246         }
4247
4248         if (map->fd < 0)
4249                 return -errno;
4250
4251         if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
4252                 bpf_map__destroy(map->inner_map);
4253                 zfree(&map->inner_map);
4254         }
4255
4256         return 0;
4257 }
4258
4259 static int init_map_slots(struct bpf_map *map)
4260 {
4261         const struct bpf_map *targ_map;
4262         unsigned int i;
4263         int fd, err;
4264
4265         for (i = 0; i < map->init_slots_sz; i++) {
4266                 if (!map->init_slots[i])
4267                         continue;
4268
4269                 targ_map = map->init_slots[i];
4270                 fd = bpf_map__fd(targ_map);
4271                 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
4272                 if (err) {
4273                         err = -errno;
4274                         pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
4275                                 map->name, i, targ_map->name,
4276                                 fd, err);
4277                         return err;
4278                 }
4279                 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
4280                          map->name, i, targ_map->name, fd);
4281         }
4282
4283         zfree(&map->init_slots);
4284         map->init_slots_sz = 0;
4285
4286         return 0;
4287 }
4288
4289 static int
4290 bpf_object__create_maps(struct bpf_object *obj)
4291 {
4292         struct bpf_map *map;
4293         char *cp, errmsg[STRERR_BUFSIZE];
4294         unsigned int i, j;
4295         int err;
4296
4297         for (i = 0; i < obj->nr_maps; i++) {
4298                 map = &obj->maps[i];
4299
4300                 if (map->pin_path) {
4301                         err = bpf_object__reuse_map(map);
4302                         if (err) {
4303                                 pr_warn("map '%s': error reusing pinned map\n",
4304                                         map->name);
4305                                 goto err_out;
4306                         }
4307                 }
4308
4309                 if (map->fd >= 0) {
4310                         pr_debug("map '%s': skipping creation (preset fd=%d)\n",
4311                                  map->name, map->fd);
4312                 } else {
4313                         err = bpf_object__create_map(obj, map);
4314                         if (err)
4315                                 goto err_out;
4316
4317                         pr_debug("map '%s': created successfully, fd=%d\n",
4318                                  map->name, map->fd);
4319
4320                         if (bpf_map__is_internal(map)) {
4321                                 err = bpf_object__populate_internal_map(obj, map);
4322                                 if (err < 0) {
4323                                         zclose(map->fd);
4324                                         goto err_out;
4325                                 }
4326                         }
4327
4328                         if (map->init_slots_sz) {
4329                                 err = init_map_slots(map);
4330                                 if (err < 0) {
4331                                         zclose(map->fd);
4332                                         goto err_out;
4333                                 }
4334                         }
4335                 }
4336
4337                 if (map->pin_path && !map->pinned) {
4338                         err = bpf_map__pin(map, NULL);
4339                         if (err) {
4340                                 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
4341                                         map->name, map->pin_path, err);
4342                                 zclose(map->fd);
4343                                 goto err_out;
4344                         }
4345                 }
4346         }
4347
4348         return 0;
4349
4350 err_out:
4351         cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4352         pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
4353         pr_perm_msg(err);
4354         for (j = 0; j < i; j++)
4355                 zclose(obj->maps[j].fd);
4356         return err;
4357 }
4358
4359 #define BPF_CORE_SPEC_MAX_LEN 64
4360
4361 /* represents BPF CO-RE field or array element accessor */
4362 struct bpf_core_accessor {
4363         __u32 type_id;          /* struct/union type or array element type */
4364         __u32 idx;              /* field index or array index */
4365         const char *name;       /* field name or NULL for array accessor */
4366 };
4367
4368 struct bpf_core_spec {
4369         const struct btf *btf;
4370         /* high-level spec: named fields and array indices only */
4371         struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
4372         /* original unresolved (no skip_mods_or_typedefs) root type ID */
4373         __u32 root_type_id;
4374         /* CO-RE relocation kind */
4375         enum bpf_core_relo_kind relo_kind;
4376         /* high-level spec length */
4377         int len;
4378         /* raw, low-level spec: 1-to-1 with accessor spec string */
4379         int raw_spec[BPF_CORE_SPEC_MAX_LEN];
4380         /* raw spec length */
4381         int raw_len;
4382         /* field bit offset represented by spec */
4383         __u32 bit_offset;
4384 };
4385
4386 static bool str_is_empty(const char *s)
4387 {
4388         return !s || !s[0];
4389 }
4390
4391 static bool is_flex_arr(const struct btf *btf,
4392                         const struct bpf_core_accessor *acc,
4393                         const struct btf_array *arr)
4394 {
4395         const struct btf_type *t;
4396
4397         /* not a flexible array, if not inside a struct or has non-zero size */
4398         if (!acc->name || arr->nelems > 0)
4399                 return false;
4400
4401         /* has to be the last member of enclosing struct */
4402         t = btf__type_by_id(btf, acc->type_id);
4403         return acc->idx == btf_vlen(t) - 1;
4404 }
4405
4406 static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
4407 {
4408         switch (kind) {
4409         case BPF_FIELD_BYTE_OFFSET: return "byte_off";
4410         case BPF_FIELD_BYTE_SIZE: return "byte_sz";
4411         case BPF_FIELD_EXISTS: return "field_exists";
4412         case BPF_FIELD_SIGNED: return "signed";
4413         case BPF_FIELD_LSHIFT_U64: return "lshift_u64";
4414         case BPF_FIELD_RSHIFT_U64: return "rshift_u64";
4415         case BPF_TYPE_ID_LOCAL: return "local_type_id";
4416         case BPF_TYPE_ID_TARGET: return "target_type_id";
4417         case BPF_TYPE_EXISTS: return "type_exists";
4418         case BPF_TYPE_SIZE: return "type_size";
4419         case BPF_ENUMVAL_EXISTS: return "enumval_exists";
4420         case BPF_ENUMVAL_VALUE: return "enumval_value";
4421         default: return "unknown";
4422         }
4423 }
4424
4425 static bool core_relo_is_field_based(enum bpf_core_relo_kind kind)
4426 {
4427         switch (kind) {
4428         case BPF_FIELD_BYTE_OFFSET:
4429         case BPF_FIELD_BYTE_SIZE:
4430         case BPF_FIELD_EXISTS:
4431         case BPF_FIELD_SIGNED:
4432         case BPF_FIELD_LSHIFT_U64:
4433         case BPF_FIELD_RSHIFT_U64:
4434                 return true;
4435         default:
4436                 return false;
4437         }
4438 }
4439
4440 static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
4441 {
4442         switch (kind) {
4443         case BPF_TYPE_ID_LOCAL:
4444         case BPF_TYPE_ID_TARGET:
4445         case BPF_TYPE_EXISTS:
4446         case BPF_TYPE_SIZE:
4447                 return true;
4448         default:
4449                 return false;
4450         }
4451 }
4452
4453 static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
4454 {
4455         switch (kind) {
4456         case BPF_ENUMVAL_EXISTS:
4457         case BPF_ENUMVAL_VALUE:
4458                 return true;
4459         default:
4460                 return false;
4461         }
4462 }
4463
4464 /*
4465  * Turn bpf_core_relo into a low- and high-level spec representation,
4466  * validating correctness along the way, as well as calculating resulting
4467  * field bit offset, specified by accessor string. Low-level spec captures
4468  * every single level of nestedness, including traversing anonymous
4469  * struct/union members. High-level one only captures semantically meaningful
4470  * "turning points": named fields and array indicies.
4471  * E.g., for this case:
4472  *
4473  *   struct sample {
4474  *       int __unimportant;
4475  *       struct {
4476  *           int __1;
4477  *           int __2;
4478  *           int a[7];
4479  *       };
4480  *   };
4481  *
4482  *   struct sample *s = ...;
4483  *
4484  *   int x = &s->a[3]; // access string = '0:1:2:3'
4485  *
4486  * Low-level spec has 1:1 mapping with each element of access string (it's
4487  * just a parsed access string representation): [0, 1, 2, 3].
4488  *
4489  * High-level spec will capture only 3 points:
4490  *   - intial zero-index access by pointer (&s->... is the same as &s[0]...);
4491  *   - field 'a' access (corresponds to '2' in low-level spec);
4492  *   - array element #3 access (corresponds to '3' in low-level spec).
4493  *
4494  * Type-based relocations (TYPE_EXISTS/TYPE_SIZE,
4495  * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their
4496  * spec and raw_spec are kept empty.
4497  *
4498  * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access
4499  * string to specify enumerator's value index that need to be relocated.
4500  */
4501 static int bpf_core_parse_spec(const struct btf *btf,
4502                                __u32 type_id,
4503                                const char *spec_str,
4504                                enum bpf_core_relo_kind relo_kind,
4505                                struct bpf_core_spec *spec)
4506 {
4507         int access_idx, parsed_len, i;
4508         struct bpf_core_accessor *acc;
4509         const struct btf_type *t;
4510         const char *name;
4511         __u32 id;
4512         __s64 sz;
4513
4514         if (str_is_empty(spec_str) || *spec_str == ':')
4515                 return -EINVAL;
4516
4517         memset(spec, 0, sizeof(*spec));
4518         spec->btf = btf;
4519         spec->root_type_id = type_id;
4520         spec->relo_kind = relo_kind;
4521
4522         /* type-based relocations don't have a field access string */
4523         if (core_relo_is_type_based(relo_kind)) {
4524                 if (strcmp(spec_str, "0"))
4525                         return -EINVAL;
4526                 return 0;
4527         }
4528
4529         /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */
4530         while (*spec_str) {
4531                 if (*spec_str == ':')
4532                         ++spec_str;
4533                 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
4534                         return -EINVAL;
4535                 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4536                         return -E2BIG;
4537                 spec_str += parsed_len;
4538                 spec->raw_spec[spec->raw_len++] = access_idx;
4539         }
4540
4541         if (spec->raw_len == 0)
4542                 return -EINVAL;
4543
4544         t = skip_mods_and_typedefs(btf, type_id, &id);
4545         if (!t)
4546                 return -EINVAL;
4547
4548         access_idx = spec->raw_spec[0];
4549         acc = &spec->spec[0];
4550         acc->type_id = id;
4551         acc->idx = access_idx;
4552         spec->len++;
4553
4554         if (core_relo_is_enumval_based(relo_kind)) {
4555                 if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
4556                         return -EINVAL;
4557
4558                 /* record enumerator name in a first accessor */
4559                 acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off);
4560                 return 0;
4561         }
4562
4563         if (!core_relo_is_field_based(relo_kind))
4564                 return -EINVAL;
4565
4566         sz = btf__resolve_size(btf, id);
4567         if (sz < 0)
4568                 return sz;
4569         spec->bit_offset = access_idx * sz * 8;
4570
4571         for (i = 1; i < spec->raw_len; i++) {
4572                 t = skip_mods_and_typedefs(btf, id, &id);
4573                 if (!t)
4574                         return -EINVAL;
4575
4576                 access_idx = spec->raw_spec[i];
4577                 acc = &spec->spec[spec->len];
4578
4579                 if (btf_is_composite(t)) {
4580                         const struct btf_member *m;
4581                         __u32 bit_offset;
4582
4583                         if (access_idx >= btf_vlen(t))
4584                                 return -EINVAL;
4585
4586                         bit_offset = btf_member_bit_offset(t, access_idx);
4587                         spec->bit_offset += bit_offset;
4588
4589                         m = btf_members(t) + access_idx;
4590                         if (m->name_off) {
4591                                 name = btf__name_by_offset(btf, m->name_off);
4592                                 if (str_is_empty(name))
4593                                         return -EINVAL;
4594
4595                                 acc->type_id = id;
4596                                 acc->idx = access_idx;
4597                                 acc->name = name;
4598                                 spec->len++;
4599                         }
4600
4601                         id = m->type;
4602                 } else if (btf_is_array(t)) {
4603                         const struct btf_array *a = btf_array(t);
4604                         bool flex;
4605
4606                         t = skip_mods_and_typedefs(btf, a->type, &id);
4607                         if (!t)
4608                                 return -EINVAL;
4609
4610                         flex = is_flex_arr(btf, acc - 1, a);
4611                         if (!flex && access_idx >= a->nelems)
4612                                 return -EINVAL;
4613
4614                         spec->spec[spec->len].type_id = id;
4615                         spec->spec[spec->len].idx = access_idx;
4616                         spec->len++;
4617
4618                         sz = btf__resolve_size(btf, id);
4619                         if (sz < 0)
4620                                 return sz;
4621                         spec->bit_offset += access_idx * sz * 8;
4622                 } else {
4623                         pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
4624                                 type_id, spec_str, i, id, btf_kind_str(t));
4625                         return -EINVAL;
4626                 }
4627         }
4628
4629         return 0;
4630 }
4631
4632 static bool bpf_core_is_flavor_sep(const char *s)
4633 {
4634         /* check X___Y name pattern, where X and Y are not underscores */
4635         return s[0] != '_' &&                                 /* X */
4636                s[1] == '_' && s[2] == '_' && s[3] == '_' &&   /* ___ */
4637                s[4] != '_';                                   /* Y */
4638 }
4639
4640 /* Given 'some_struct_name___with_flavor' return the length of a name prefix
4641  * before last triple underscore. Struct name part after last triple
4642  * underscore is ignored by BPF CO-RE relocation during relocation matching.
4643  */
4644 static size_t bpf_core_essential_name_len(const char *name)
4645 {
4646         size_t n = strlen(name);
4647         int i;
4648
4649         for (i = n - 5; i >= 0; i--) {
4650                 if (bpf_core_is_flavor_sep(name + i))
4651                         return i + 1;
4652         }
4653         return n;
4654 }
4655
4656 struct core_cand
4657 {
4658         const struct btf *btf;
4659         const struct btf_type *t;
4660         const char *name;
4661         __u32 id;
4662 };
4663
4664 /* dynamically sized list of type IDs and its associated struct btf */
4665 struct core_cand_list {
4666         struct core_cand *cands;
4667         int len;
4668 };
4669
4670 static void bpf_core_free_cands(struct core_cand_list *cands)
4671 {
4672         free(cands->cands);
4673         free(cands);
4674 }
4675
4676 static int bpf_core_add_cands(struct core_cand *local_cand,
4677                               size_t local_essent_len,
4678                               const struct btf *targ_btf,
4679                               const char *targ_btf_name,
4680                               int targ_start_id,
4681                               struct core_cand_list *cands)
4682 {
4683         struct core_cand *new_cands, *cand;
4684         const struct btf_type *t;
4685         const char *targ_name;
4686         size_t targ_essent_len;
4687         int n, i;
4688
4689         n = btf__get_nr_types(targ_btf);
4690         for (i = targ_start_id; i <= n; i++) {
4691                 t = btf__type_by_id(targ_btf, i);
4692                 if (btf_kind(t) != btf_kind(local_cand->t))
4693                         continue;
4694
4695                 targ_name = btf__name_by_offset(targ_btf, t->name_off);
4696                 if (str_is_empty(targ_name))
4697                         continue;
4698
4699                 targ_essent_len = bpf_core_essential_name_len(targ_name);
4700                 if (targ_essent_len != local_essent_len)
4701                         continue;
4702
4703                 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
4704                         continue;
4705
4706                 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
4707                          local_cand->id, btf_kind_str(local_cand->t),
4708                          local_cand->name, i, btf_kind_str(t), targ_name,
4709                          targ_btf_name);
4710                 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
4711                                               sizeof(*cands->cands));
4712                 if (!new_cands)
4713                         return -ENOMEM;
4714
4715                 cand = &new_cands[cands->len];
4716                 cand->btf = targ_btf;
4717                 cand->t = t;
4718                 cand->name = targ_name;
4719                 cand->id = i;
4720
4721                 cands->cands = new_cands;
4722                 cands->len++;
4723         }
4724         return 0;
4725 }
4726
4727 static int load_module_btfs(struct bpf_object *obj)
4728 {
4729         struct bpf_btf_info info;
4730         struct module_btf *mod_btf;
4731         struct btf *btf;
4732         char name[64];
4733         __u32 id = 0, len;
4734         int err, fd;
4735
4736         if (obj->btf_modules_loaded)
4737                 return 0;
4738
4739         /* don't do this again, even if we find no module BTFs */
4740         obj->btf_modules_loaded = true;
4741
4742         /* kernel too old to support module BTFs */
4743         if (!kernel_supports(FEAT_MODULE_BTF))
4744                 return 0;
4745
4746         while (true) {
4747                 err = bpf_btf_get_next_id(id, &id);
4748                 if (err && errno == ENOENT)
4749                         return 0;
4750                 if (err) {
4751                         err = -errno;
4752                         pr_warn("failed to iterate BTF objects: %d\n", err);
4753                         return err;
4754                 }
4755
4756                 fd = bpf_btf_get_fd_by_id(id);
4757                 if (fd < 0) {
4758                         if (errno == ENOENT)
4759                                 continue; /* expected race: BTF was unloaded */
4760                         err = -errno;
4761                         pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
4762                         return err;
4763                 }
4764
4765                 len = sizeof(info);
4766                 memset(&info, 0, sizeof(info));
4767                 info.name = ptr_to_u64(name);
4768                 info.name_len = sizeof(name);
4769
4770                 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4771                 if (err) {
4772                         err = -errno;
4773                         pr_warn("failed to get BTF object #%d info: %d\n", id, err);
4774                         goto err_out;
4775                 }
4776
4777                 /* ignore non-module BTFs */
4778                 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
4779                         close(fd);
4780                         continue;
4781                 }
4782
4783                 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
4784                 if (IS_ERR(btf)) {
4785                         pr_warn("failed to load module [%s]'s BTF object #%d: %ld\n",
4786                                 name, id, PTR_ERR(btf));
4787                         err = PTR_ERR(btf);
4788                         goto err_out;
4789                 }
4790
4791                 err = btf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
4792                                      sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
4793                 if (err)
4794                         goto err_out;
4795
4796                 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
4797
4798                 mod_btf->btf = btf;
4799                 mod_btf->id = id;
4800                 mod_btf->fd = fd;
4801                 mod_btf->name = strdup(name);
4802                 if (!mod_btf->name) {
4803                         err = -ENOMEM;
4804                         goto err_out;
4805                 }
4806                 continue;
4807
4808 err_out:
4809                 close(fd);
4810                 return err;
4811         }
4812
4813         return 0;
4814 }
4815
4816 static struct core_cand_list *
4817 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
4818 {
4819         struct core_cand local_cand = {};
4820         struct core_cand_list *cands;
4821         const struct btf *main_btf;
4822         size_t local_essent_len;
4823         int err, i;
4824
4825         local_cand.btf = local_btf;
4826         local_cand.t = btf__type_by_id(local_btf, local_type_id);
4827         if (!local_cand.t)
4828                 return ERR_PTR(-EINVAL);
4829
4830         local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
4831         if (str_is_empty(local_cand.name))
4832                 return ERR_PTR(-EINVAL);
4833         local_essent_len = bpf_core_essential_name_len(local_cand.name);
4834
4835         cands = calloc(1, sizeof(*cands));
4836         if (!cands)
4837                 return ERR_PTR(-ENOMEM);
4838
4839         /* Attempt to find target candidates in vmlinux BTF first */
4840         main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
4841         err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
4842         if (err)
4843                 goto err_out;
4844
4845         /* if vmlinux BTF has any candidate, don't got for module BTFs */
4846         if (cands->len)
4847                 return cands;
4848
4849         /* if vmlinux BTF was overridden, don't attempt to load module BTFs */
4850         if (obj->btf_vmlinux_override)
4851                 return cands;
4852
4853         /* now look through module BTFs, trying to still find candidates */
4854         err = load_module_btfs(obj);
4855         if (err)
4856                 goto err_out;
4857
4858         for (i = 0; i < obj->btf_module_cnt; i++) {
4859                 err = bpf_core_add_cands(&local_cand, local_essent_len,
4860                                          obj->btf_modules[i].btf,
4861                                          obj->btf_modules[i].name,
4862                                          btf__get_nr_types(obj->btf_vmlinux) + 1,
4863                                          cands);
4864                 if (err)
4865                         goto err_out;
4866         }
4867
4868         return cands;
4869 err_out:
4870         bpf_core_free_cands(cands);
4871         return ERR_PTR(err);
4872 }
4873
4874 /* Check two types for compatibility for the purpose of field access
4875  * relocation. const/volatile/restrict and typedefs are skipped to ensure we
4876  * are relocating semantically compatible entities:
4877  *   - any two STRUCTs/UNIONs are compatible and can be mixed;
4878  *   - any two FWDs are compatible, if their names match (modulo flavor suffix);
4879  *   - any two PTRs are always compatible;
4880  *   - for ENUMs, names should be the same (ignoring flavor suffix) or at
4881  *     least one of enums should be anonymous;
4882  *   - for ENUMs, check sizes, names are ignored;
4883  *   - for INT, size and signedness are ignored;
4884  *   - for ARRAY, dimensionality is ignored, element types are checked for
4885  *     compatibility recursively;
4886  *   - everything else shouldn't be ever a target of relocation.
4887  * These rules are not set in stone and probably will be adjusted as we get
4888  * more experience with using BPF CO-RE relocations.
4889  */
4890 static int bpf_core_fields_are_compat(const struct btf *local_btf,
4891                                       __u32 local_id,
4892                                       const struct btf *targ_btf,
4893                                       __u32 targ_id)
4894 {
4895         const struct btf_type *local_type, *targ_type;
4896
4897 recur:
4898         local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
4899         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4900         if (!local_type || !targ_type)
4901                 return -EINVAL;
4902
4903         if (btf_is_composite(local_type) && btf_is_composite(targ_type))
4904                 return 1;
4905         if (btf_kind(local_type) != btf_kind(targ_type))
4906                 return 0;
4907
4908         switch (btf_kind(local_type)) {
4909         case BTF_KIND_PTR:
4910                 return 1;
4911         case BTF_KIND_FWD:
4912         case BTF_KIND_ENUM: {
4913                 const char *local_name, *targ_name;
4914                 size_t local_len, targ_len;
4915
4916                 local_name = btf__name_by_offset(local_btf,
4917                                                  local_type->name_off);
4918                 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
4919                 local_len = bpf_core_essential_name_len(local_name);
4920                 targ_len = bpf_core_essential_name_len(targ_name);
4921                 /* one of them is anonymous or both w/ same flavor-less names */
4922                 return local_len == 0 || targ_len == 0 ||
4923                        (local_len == targ_len &&
4924                         strncmp(local_name, targ_name, local_len) == 0);
4925         }
4926         case BTF_KIND_INT:
4927                 /* just reject deprecated bitfield-like integers; all other
4928                  * integers are by default compatible between each other
4929                  */
4930                 return btf_int_offset(local_type) == 0 &&
4931                        btf_int_offset(targ_type) == 0;
4932         case BTF_KIND_ARRAY:
4933                 local_id = btf_array(local_type)->type;
4934                 targ_id = btf_array(targ_type)->type;
4935                 goto recur;
4936         default:
4937                 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
4938                         btf_kind(local_type), local_id, targ_id);
4939                 return 0;
4940         }
4941 }
4942
4943 /*
4944  * Given single high-level named field accessor in local type, find
4945  * corresponding high-level accessor for a target type. Along the way,
4946  * maintain low-level spec for target as well. Also keep updating target
4947  * bit offset.
4948  *
4949  * Searching is performed through recursive exhaustive enumeration of all
4950  * fields of a struct/union. If there are any anonymous (embedded)
4951  * structs/unions, they are recursively searched as well. If field with
4952  * desired name is found, check compatibility between local and target types,
4953  * before returning result.
4954  *
4955  * 1 is returned, if field is found.
4956  * 0 is returned if no compatible field is found.
4957  * <0 is returned on error.
4958  */
4959 static int bpf_core_match_member(const struct btf *local_btf,
4960                                  const struct bpf_core_accessor *local_acc,
4961                                  const struct btf *targ_btf,
4962                                  __u32 targ_id,
4963                                  struct bpf_core_spec *spec,
4964                                  __u32 *next_targ_id)
4965 {
4966         const struct btf_type *local_type, *targ_type;
4967         const struct btf_member *local_member, *m;
4968         const char *local_name, *targ_name;
4969         __u32 local_id;
4970         int i, n, found;
4971
4972         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4973         if (!targ_type)
4974                 return -EINVAL;
4975         if (!btf_is_composite(targ_type))
4976                 return 0;
4977
4978         local_id = local_acc->type_id;
4979         local_type = btf__type_by_id(local_btf, local_id);
4980         local_member = btf_members(local_type) + local_acc->idx;
4981         local_name = btf__name_by_offset(local_btf, local_member->name_off);
4982
4983         n = btf_vlen(targ_type);
4984         m = btf_members(targ_type);
4985         for (i = 0; i < n; i++, m++) {
4986                 __u32 bit_offset;
4987
4988                 bit_offset = btf_member_bit_offset(targ_type, i);
4989
4990                 /* too deep struct/union/array nesting */
4991                 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4992                         return -E2BIG;
4993
4994                 /* speculate this member will be the good one */
4995                 spec->bit_offset += bit_offset;
4996                 spec->raw_spec[spec->raw_len++] = i;
4997
4998                 targ_name = btf__name_by_offset(targ_btf, m->name_off);
4999                 if (str_is_empty(targ_name)) {
5000                         /* embedded struct/union, we need to go deeper */
5001                         found = bpf_core_match_member(local_btf, local_acc,
5002                                                       targ_btf, m->type,
5003                                                       spec, next_targ_id);
5004                         if (found) /* either found or error */
5005                                 return found;
5006                 } else if (strcmp(local_name, targ_name) == 0) {
5007                         /* matching named field */
5008                         struct bpf_core_accessor *targ_acc;
5009
5010                         targ_acc = &spec->spec[spec->len++];
5011                         targ_acc->type_id = targ_id;
5012                         targ_acc->idx = i;
5013                         targ_acc->name = targ_name;
5014
5015                         *next_targ_id = m->type;
5016                         found = bpf_core_fields_are_compat(local_btf,
5017                                                            local_member->type,
5018                                                            targ_btf, m->type);
5019                         if (!found)
5020                                 spec->len--; /* pop accessor */
5021                         return found;
5022                 }
5023                 /* member turned out not to be what we looked for */
5024                 spec->bit_offset -= bit_offset;
5025                 spec->raw_len--;
5026         }
5027
5028         return 0;
5029 }
5030
5031 /* Check local and target types for compatibility. This check is used for
5032  * type-based CO-RE relocations and follow slightly different rules than
5033  * field-based relocations. This function assumes that root types were already
5034  * checked for name match. Beyond that initial root-level name check, names
5035  * are completely ignored. Compatibility rules are as follows:
5036  *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5037  *     kind should match for local and target types (i.e., STRUCT is not
5038  *     compatible with UNION);
5039  *   - for ENUMs, the size is ignored;
5040  *   - for INT, size and signedness are ignored;
5041  *   - for ARRAY, dimensionality is ignored, element types are checked for
5042  *     compatibility recursively;
5043  *   - CONST/VOLATILE/RESTRICT modifiers are ignored;
5044  *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5045  *   - FUNC_PROTOs are compatible if they have compatible signature: same
5046  *     number of input args and compatible return and argument types.
5047  * These rules are not set in stone and probably will be adjusted as we get
5048  * more experience with using BPF CO-RE relocations.
5049  */
5050 static int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5051                                      const struct btf *targ_btf, __u32 targ_id)
5052 {
5053         const struct btf_type *local_type, *targ_type;
5054         int depth = 32; /* max recursion depth */
5055
5056         /* caller made sure that names match (ignoring flavor suffix) */
5057         local_type = btf__type_by_id(local_btf, local_id);
5058         targ_type = btf__type_by_id(targ_btf, targ_id);
5059         if (btf_kind(local_type) != btf_kind(targ_type))
5060                 return 0;
5061
5062 recur:
5063         depth--;
5064         if (depth < 0)
5065                 return -EINVAL;
5066
5067         local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
5068         targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
5069         if (!local_type || !targ_type)
5070                 return -EINVAL;
5071
5072         if (btf_kind(local_type) != btf_kind(targ_type))
5073                 return 0;
5074
5075         switch (btf_kind(local_type)) {
5076         case BTF_KIND_UNKN:
5077         case BTF_KIND_STRUCT:
5078         case BTF_KIND_UNION:
5079         case BTF_KIND_ENUM:
5080         case BTF_KIND_FWD:
5081                 return 1;
5082         case BTF_KIND_INT:
5083                 /* just reject deprecated bitfield-like integers; all other
5084                  * integers are by default compatible between each other
5085                  */
5086                 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
5087         case BTF_KIND_PTR:
5088                 local_id = local_type->type;
5089                 targ_id = targ_type->type;
5090                 goto recur;
5091         case BTF_KIND_ARRAY:
5092                 local_id = btf_array(local_type)->type;
5093                 targ_id = btf_array(targ_type)->type;
5094                 goto recur;
5095         case BTF_KIND_FUNC_PROTO: {
5096                 struct btf_param *local_p = btf_params(local_type);
5097                 struct btf_param *targ_p = btf_params(targ_type);
5098                 __u16 local_vlen = btf_vlen(local_type);
5099                 __u16 targ_vlen = btf_vlen(targ_type);
5100                 int i, err;
5101
5102                 if (local_vlen != targ_vlen)
5103                         return 0;
5104
5105                 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
5106                         skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
5107                         skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
5108                         err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id);
5109                         if (err <= 0)
5110                                 return err;
5111                 }
5112
5113                 /* tail recurse for return type check */
5114                 skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
5115                 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
5116                 goto recur;
5117         }
5118         default:
5119                 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
5120                         btf_kind_str(local_type), local_id, targ_id);
5121                 return 0;
5122         }
5123 }
5124
5125 /*
5126  * Try to match local spec to a target type and, if successful, produce full
5127  * target spec (high-level, low-level + bit offset).
5128  */
5129 static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
5130                                const struct btf *targ_btf, __u32 targ_id,
5131                                struct bpf_core_spec *targ_spec)
5132 {
5133         const struct btf_type *targ_type;
5134         const struct bpf_core_accessor *local_acc;
5135         struct bpf_core_accessor *targ_acc;
5136         int i, sz, matched;
5137
5138         memset(targ_spec, 0, sizeof(*targ_spec));
5139         targ_spec->btf = targ_btf;
5140         targ_spec->root_type_id = targ_id;
5141         targ_spec->relo_kind = local_spec->relo_kind;
5142
5143         if (core_relo_is_type_based(local_spec->relo_kind)) {
5144                 return bpf_core_types_are_compat(local_spec->btf,
5145                                                  local_spec->root_type_id,
5146                                                  targ_btf, targ_id);
5147         }
5148
5149         local_acc = &local_spec->spec[0];
5150         targ_acc = &targ_spec->spec[0];
5151
5152         if (core_relo_is_enumval_based(local_spec->relo_kind)) {
5153                 size_t local_essent_len, targ_essent_len;
5154                 const struct btf_enum *e;
5155                 const char *targ_name;
5156
5157                 /* has to resolve to an enum */
5158                 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
5159                 if (!btf_is_enum(targ_type))
5160                         return 0;
5161
5162                 local_essent_len = bpf_core_essential_name_len(local_acc->name);
5163
5164                 for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) {
5165                         targ_name = btf__name_by_offset(targ_spec->btf, e->name_off);
5166                         targ_essent_len = bpf_core_essential_name_len(targ_name);
5167                         if (targ_essent_len != local_essent_len)
5168                                 continue;
5169                         if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {
5170                                 targ_acc->type_id = targ_id;
5171                                 targ_acc->idx = i;
5172                                 targ_acc->name = targ_name;
5173                                 targ_spec->len++;
5174                                 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5175                                 targ_spec->raw_len++;
5176                                 return 1;
5177                         }
5178                 }
5179                 return 0;
5180         }
5181
5182         if (!core_relo_is_field_based(local_spec->relo_kind))
5183                 return -EINVAL;
5184
5185         for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
5186                 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
5187                                                    &targ_id);
5188                 if (!targ_type)
5189                         return -EINVAL;
5190
5191                 if (local_acc->name) {
5192                         matched = bpf_core_match_member(local_spec->btf,
5193                                                         local_acc,
5194                                                         targ_btf, targ_id,
5195                                                         targ_spec, &targ_id);
5196                         if (matched <= 0)
5197                                 return matched;
5198                 } else {
5199                         /* for i=0, targ_id is already treated as array element
5200                          * type (because it's the original struct), for others
5201                          * we should find array element type first
5202                          */
5203                         if (i > 0) {
5204                                 const struct btf_array *a;
5205                                 bool flex;
5206
5207                                 if (!btf_is_array(targ_type))
5208                                         return 0;
5209
5210                                 a = btf_array(targ_type);
5211                                 flex = is_flex_arr(targ_btf, targ_acc - 1, a);
5212                                 if (!flex && local_acc->idx >= a->nelems)
5213                                         return 0;
5214                                 if (!skip_mods_and_typedefs(targ_btf, a->type,
5215                                                             &targ_id))
5216                                         return -EINVAL;
5217                         }
5218
5219                         /* too deep struct/union/array nesting */
5220                         if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
5221                                 return -E2BIG;
5222
5223                         targ_acc->type_id = targ_id;
5224                         targ_acc->idx = local_acc->idx;
5225                         targ_acc->name = NULL;
5226                         targ_spec->len++;
5227                         targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5228                         targ_spec->raw_len++;
5229
5230                         sz = btf__resolve_size(targ_btf, targ_id);
5231                         if (sz < 0)
5232                                 return sz;
5233                         targ_spec->bit_offset += local_acc->idx * sz * 8;
5234                 }
5235         }
5236
5237         return 1;
5238 }
5239
5240 static int bpf_core_calc_field_relo(const struct bpf_program *prog,
5241                                     const struct bpf_core_relo *relo,
5242                                     const struct bpf_core_spec *spec,
5243                                     __u32 *val, __u32 *field_sz, __u32 *type_id,
5244                                     bool *validate)
5245 {
5246         const struct bpf_core_accessor *acc;
5247         const struct btf_type *t;
5248         __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
5249         const struct btf_member *m;
5250         const struct btf_type *mt;
5251         bool bitfield;
5252         __s64 sz;
5253
5254         *field_sz = 0;
5255
5256         if (relo->kind == BPF_FIELD_EXISTS) {
5257                 *val = spec ? 1 : 0;
5258                 return 0;
5259         }
5260
5261         if (!spec)
5262                 return -EUCLEAN; /* request instruction poisoning */
5263
5264         acc = &spec->spec[spec->len - 1];
5265         t = btf__type_by_id(spec->btf, acc->type_id);
5266
5267         /* a[n] accessor needs special handling */
5268         if (!acc->name) {
5269                 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
5270                         *val = spec->bit_offset / 8;
5271                         /* remember field size for load/store mem size */
5272                         sz = btf__resolve_size(spec->btf, acc->type_id);
5273                         if (sz < 0)
5274                                 return -EINVAL;
5275                         *field_sz = sz;
5276                         *type_id = acc->type_id;
5277                 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
5278                         sz = btf__resolve_size(spec->btf, acc->type_id);
5279                         if (sz < 0)
5280                                 return -EINVAL;
5281                         *val = sz;
5282                 } else {
5283                         pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
5284                                 prog->name, relo->kind, relo->insn_off / 8);
5285                         return -EINVAL;
5286                 }
5287                 if (validate)
5288                         *validate = true;
5289                 return 0;
5290         }
5291
5292         m = btf_members(t) + acc->idx;
5293         mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
5294         bit_off = spec->bit_offset;
5295         bit_sz = btf_member_bitfield_size(t, acc->idx);
5296
5297         bitfield = bit_sz > 0;
5298         if (bitfield) {
5299                 byte_sz = mt->size;
5300                 byte_off = bit_off / 8 / byte_sz * byte_sz;
5301                 /* figure out smallest int size necessary for bitfield load */
5302                 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
5303                         if (byte_sz >= 8) {
5304                                 /* bitfield can't be read with 64-bit read */
5305                                 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
5306                                         prog->name, relo->kind, relo->insn_off / 8);
5307                                 return -E2BIG;
5308                         }
5309                         byte_sz *= 2;
5310                         byte_off = bit_off / 8 / byte_sz * byte_sz;
5311                 }
5312         } else {
5313                 sz = btf__resolve_size(spec->btf, field_type_id);
5314                 if (sz < 0)
5315                         return -EINVAL;
5316                 byte_sz = sz;
5317                 byte_off = spec->bit_offset / 8;
5318                 bit_sz = byte_sz * 8;
5319         }
5320
5321         /* for bitfields, all the relocatable aspects are ambiguous and we
5322          * might disagree with compiler, so turn off validation of expected
5323          * value, except for signedness
5324          */
5325         if (validate)
5326                 *validate = !bitfield;
5327
5328         switch (relo->kind) {
5329         case BPF_FIELD_BYTE_OFFSET:
5330                 *val = byte_off;
5331                 if (!bitfield) {
5332                         *field_sz = byte_sz;
5333                         *type_id = field_type_id;
5334                 }
5335                 break;
5336         case BPF_FIELD_BYTE_SIZE:
5337                 *val = byte_sz;
5338                 break;
5339         case BPF_FIELD_SIGNED:
5340                 /* enums will be assumed unsigned */
5341                 *val = btf_is_enum(mt) ||
5342                        (btf_int_encoding(mt) & BTF_INT_SIGNED);
5343                 if (validate)
5344                         *validate = true; /* signedness is never ambiguous */
5345                 break;
5346         case BPF_FIELD_LSHIFT_U64:
5347 #if __BYTE_ORDER == __LITTLE_ENDIAN
5348                 *val = 64 - (bit_off + bit_sz - byte_off  * 8);
5349 #else
5350                 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
5351 #endif
5352                 break;
5353         case BPF_FIELD_RSHIFT_U64:
5354                 *val = 64 - bit_sz;
5355                 if (validate)
5356                         *validate = true; /* right shift is never ambiguous */
5357                 break;
5358         case BPF_FIELD_EXISTS:
5359         default:
5360                 return -EOPNOTSUPP;
5361         }
5362
5363         return 0;
5364 }
5365
5366 static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
5367                                    const struct bpf_core_spec *spec,
5368                                    __u32 *val)
5369 {
5370         __s64 sz;
5371
5372         /* type-based relos return zero when target type is not found */
5373         if (!spec) {
5374                 *val = 0;
5375                 return 0;
5376         }
5377
5378         switch (relo->kind) {
5379         case BPF_TYPE_ID_TARGET:
5380                 *val = spec->root_type_id;
5381                 break;
5382         case BPF_TYPE_EXISTS:
5383                 *val = 1;
5384                 break;
5385         case BPF_TYPE_SIZE:
5386                 sz = btf__resolve_size(spec->btf, spec->root_type_id);
5387                 if (sz < 0)
5388                         return -EINVAL;
5389                 *val = sz;
5390                 break;
5391         case BPF_TYPE_ID_LOCAL:
5392         /* BPF_TYPE_ID_LOCAL is handled specially and shouldn't get here */
5393         default:
5394                 return -EOPNOTSUPP;
5395         }
5396
5397         return 0;
5398 }
5399
5400 static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
5401                                       const struct bpf_core_spec *spec,
5402                                       __u32 *val)
5403 {
5404         const struct btf_type *t;
5405         const struct btf_enum *e;
5406
5407         switch (relo->kind) {
5408         case BPF_ENUMVAL_EXISTS:
5409                 *val = spec ? 1 : 0;
5410                 break;
5411         case BPF_ENUMVAL_VALUE:
5412                 if (!spec)
5413                         return -EUCLEAN; /* request instruction poisoning */
5414                 t = btf__type_by_id(spec->btf, spec->spec[0].type_id);
5415                 e = btf_enum(t) + spec->spec[0].idx;
5416                 *val = e->val;
5417                 break;
5418         default:
5419                 return -EOPNOTSUPP;
5420         }
5421
5422         return 0;
5423 }
5424
5425 struct bpf_core_relo_res
5426 {
5427         /* expected value in the instruction, unless validate == false */
5428         __u32 orig_val;
5429         /* new value that needs to be patched up to */
5430         __u32 new_val;
5431         /* relocation unsuccessful, poison instruction, but don't fail load */
5432         bool poison;
5433         /* some relocations can't be validated against orig_val */
5434         bool validate;
5435         /* for field byte offset relocations or the forms:
5436          *     *(T *)(rX + <off>) = rY
5437          *     rX = *(T *)(rY + <off>),
5438          * we remember original and resolved field size to adjust direct
5439          * memory loads of pointers and integers; this is necessary for 32-bit
5440          * host kernel architectures, but also allows to automatically
5441          * relocate fields that were resized from, e.g., u32 to u64, etc.
5442          */
5443         bool fail_memsz_adjust;
5444         __u32 orig_sz;
5445         __u32 orig_type_id;
5446         __u32 new_sz;
5447         __u32 new_type_id;
5448 };
5449
5450 /* Calculate original and target relocation values, given local and target
5451  * specs and relocation kind. These values are calculated for each candidate.
5452  * If there are multiple candidates, resulting values should all be consistent
5453  * with each other. Otherwise, libbpf will refuse to proceed due to ambiguity.
5454  * If instruction has to be poisoned, *poison will be set to true.
5455  */
5456 static int bpf_core_calc_relo(const struct bpf_program *prog,
5457                               const struct bpf_core_relo *relo,
5458                               int relo_idx,
5459                               const struct bpf_core_spec *local_spec,
5460                               const struct bpf_core_spec *targ_spec,
5461                               struct bpf_core_relo_res *res)
5462 {
5463         int err = -EOPNOTSUPP;
5464
5465         res->orig_val = 0;
5466         res->new_val = 0;
5467         res->poison = false;
5468         res->validate = true;
5469         res->fail_memsz_adjust = false;
5470         res->orig_sz = res->new_sz = 0;
5471         res->orig_type_id = res->new_type_id = 0;
5472
5473         if (core_relo_is_field_based(relo->kind)) {
5474                 err = bpf_core_calc_field_relo(prog, relo, local_spec,
5475                                                &res->orig_val, &res->orig_sz,
5476                                                &res->orig_type_id, &res->validate);
5477                 err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec,
5478                                                       &res->new_val, &res->new_sz,
5479                                                       &res->new_type_id, NULL);
5480                 if (err)
5481                         goto done;
5482                 /* Validate if it's safe to adjust load/store memory size.
5483                  * Adjustments are performed only if original and new memory
5484                  * sizes differ.
5485                  */
5486                 res->fail_memsz_adjust = false;
5487                 if (res->orig_sz != res->new_sz) {
5488                         const struct btf_type *orig_t, *new_t;
5489
5490                         orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id);
5491                         new_t = btf__type_by_id(targ_spec->btf, res->new_type_id);
5492
5493                         /* There are two use cases in which it's safe to
5494                          * adjust load/store's mem size:
5495                          *   - reading a 32-bit kernel pointer, while on BPF
5496                          *   size pointers are always 64-bit; in this case
5497                          *   it's safe to "downsize" instruction size due to
5498                          *   pointer being treated as unsigned integer with
5499                          *   zero-extended upper 32-bits;
5500                          *   - reading unsigned integers, again due to
5501                          *   zero-extension is preserving the value correctly.
5502                          *
5503                          * In all other cases it's incorrect to attempt to
5504                          * load/store field because read value will be
5505                          * incorrect, so we poison relocated instruction.
5506                          */
5507                         if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
5508                                 goto done;
5509                         if (btf_is_int(orig_t) && btf_is_int(new_t) &&
5510                             btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
5511                             btf_int_encoding(new_t) != BTF_INT_SIGNED)
5512                                 goto done;
5513
5514                         /* mark as invalid mem size adjustment, but this will
5515                          * only be checked for LDX/STX/ST insns
5516                          */
5517                         res->fail_memsz_adjust = true;
5518                 }
5519         } else if (core_relo_is_type_based(relo->kind)) {
5520                 err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
5521                 err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
5522         } else if (core_relo_is_enumval_based(relo->kind)) {
5523                 err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
5524                 err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
5525         }
5526
5527 done:
5528         if (err == -EUCLEAN) {
5529                 /* EUCLEAN is used to signal instruction poisoning request */
5530                 res->poison = true;
5531                 err = 0;
5532         } else if (err == -EOPNOTSUPP) {
5533                 /* EOPNOTSUPP means unknown/unsupported relocation */
5534                 pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",
5535                         prog->name, relo_idx, core_relo_kind_str(relo->kind),
5536                         relo->kind, relo->insn_off / 8);
5537         }
5538
5539         return err;
5540 }
5541
5542 /*
5543  * Turn instruction for which CO_RE relocation failed into invalid one with
5544  * distinct signature.
5545  */
5546 static void bpf_core_poison_insn(struct bpf_program *prog, int relo_idx,
5547                                  int insn_idx, struct bpf_insn *insn)
5548 {
5549         pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
5550                  prog->name, relo_idx, insn_idx);
5551         insn->code = BPF_JMP | BPF_CALL;
5552         insn->dst_reg = 0;
5553         insn->src_reg = 0;
5554         insn->off = 0;
5555         /* if this instruction is reachable (not a dead code),
5556          * verifier will complain with the following message:
5557          * invalid func unknown#195896080
5558          */
5559         insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
5560 }
5561
5562 static bool is_ldimm64(struct bpf_insn *insn)
5563 {
5564         return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
5565 }
5566
5567 static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
5568 {
5569         switch (BPF_SIZE(insn->code)) {
5570         case BPF_DW: return 8;
5571         case BPF_W: return 4;
5572         case BPF_H: return 2;
5573         case BPF_B: return 1;
5574         default: return -1;
5575         }
5576 }
5577
5578 static int insn_bytes_to_bpf_size(__u32 sz)
5579 {
5580         switch (sz) {
5581         case 8: return BPF_DW;
5582         case 4: return BPF_W;
5583         case 2: return BPF_H;
5584         case 1: return BPF_B;
5585         default: return -1;
5586         }
5587 }
5588
5589 /*
5590  * Patch relocatable BPF instruction.
5591  *
5592  * Patched value is determined by relocation kind and target specification.
5593  * For existence relocations target spec will be NULL if field/type is not found.
5594  * Expected insn->imm value is determined using relocation kind and local
5595  * spec, and is checked before patching instruction. If actual insn->imm value
5596  * is wrong, bail out with error.
5597  *
5598  * Currently supported classes of BPF instruction are:
5599  * 1. rX = <imm> (assignment with immediate operand);
5600  * 2. rX += <imm> (arithmetic operations with immediate operand);
5601  * 3. rX = <imm64> (load with 64-bit immediate value);
5602  * 4. rX = *(T *)(rY + <off>), where T is one of {u8, u16, u32, u64};
5603  * 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
5604  * 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
5605  */
5606 static int bpf_core_patch_insn(struct bpf_program *prog,
5607                                const struct bpf_core_relo *relo,
5608                                int relo_idx,
5609                                const struct bpf_core_relo_res *res)
5610 {
5611         __u32 orig_val, new_val;
5612         struct bpf_insn *insn;
5613         int insn_idx;
5614         __u8 class;
5615
5616         if (relo->insn_off % BPF_INSN_SZ)
5617                 return -EINVAL;
5618         insn_idx = relo->insn_off / BPF_INSN_SZ;
5619         /* adjust insn_idx from section frame of reference to the local
5620          * program's frame of reference; (sub-)program code is not yet
5621          * relocated, so it's enough to just subtract in-section offset
5622          */
5623         insn_idx = insn_idx - prog->sec_insn_off;
5624         insn = &prog->insns[insn_idx];
5625         class = BPF_CLASS(insn->code);
5626
5627         if (res->poison) {
5628 poison:
5629                 /* poison second part of ldimm64 to avoid confusing error from
5630                  * verifier about "unknown opcode 00"
5631                  */
5632                 if (is_ldimm64(insn))
5633                         bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);
5634                 bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);
5635                 return 0;
5636         }
5637
5638         orig_val = res->orig_val;
5639         new_val = res->new_val;
5640
5641         switch (class) {
5642         case BPF_ALU:
5643         case BPF_ALU64:
5644                 if (BPF_SRC(insn->code) != BPF_K)
5645                         return -EINVAL;
5646                 if (res->validate && insn->imm != orig_val) {
5647                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
5648                                 prog->name, relo_idx,
5649                                 insn_idx, insn->imm, orig_val, new_val);
5650                         return -EINVAL;
5651                 }
5652                 orig_val = insn->imm;
5653                 insn->imm = new_val;
5654                 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
5655                          prog->name, relo_idx, insn_idx,
5656                          orig_val, new_val);
5657                 break;
5658         case BPF_LDX:
5659         case BPF_ST:
5660         case BPF_STX:
5661                 if (res->validate && insn->off != orig_val) {
5662                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n",
5663                                 prog->name, relo_idx, insn_idx, insn->off, orig_val, new_val);
5664                         return -EINVAL;
5665                 }
5666                 if (new_val > SHRT_MAX) {
5667                         pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
5668                                 prog->name, relo_idx, insn_idx, new_val);
5669                         return -ERANGE;
5670                 }
5671                 if (res->fail_memsz_adjust) {
5672                         pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
5673                                 "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
5674                                 prog->name, relo_idx, insn_idx);
5675                         goto poison;
5676                 }
5677
5678                 orig_val = insn->off;
5679                 insn->off = new_val;
5680                 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
5681                          prog->name, relo_idx, insn_idx, orig_val, new_val);
5682
5683                 if (res->new_sz != res->orig_sz) {
5684                         int insn_bytes_sz, insn_bpf_sz;
5685
5686                         insn_bytes_sz = insn_bpf_size_to_bytes(insn);
5687                         if (insn_bytes_sz != res->orig_sz) {
5688                                 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
5689                                         prog->name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
5690                                 return -EINVAL;
5691                         }
5692
5693                         insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
5694                         if (insn_bpf_sz < 0) {
5695                                 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
5696                                         prog->name, relo_idx, insn_idx, res->new_sz);
5697                                 return -EINVAL;
5698                         }
5699
5700                         insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code);
5701                         pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
5702                                  prog->name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
5703                 }
5704                 break;
5705         case BPF_LD: {
5706                 __u64 imm;
5707
5708                 if (!is_ldimm64(insn) ||
5709                     insn[0].src_reg != 0 || insn[0].off != 0 ||
5710                     insn_idx + 1 >= prog->insns_cnt ||
5711                     insn[1].code != 0 || insn[1].dst_reg != 0 ||
5712                     insn[1].src_reg != 0 || insn[1].off != 0) {
5713                         pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",
5714                                 prog->name, relo_idx, insn_idx);
5715                         return -EINVAL;
5716                 }
5717
5718                 imm = insn[0].imm + ((__u64)insn[1].imm << 32);
5719                 if (res->validate && imm != orig_val) {
5720                         pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n",
5721                                 prog->name, relo_idx,
5722                                 insn_idx, (unsigned long long)imm,
5723                                 orig_val, new_val);
5724                         return -EINVAL;
5725                 }
5726
5727                 insn[0].imm = new_val;
5728                 insn[1].imm = 0; /* currently only 32-bit values are supported */
5729                 pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n",
5730                          prog->name, relo_idx, insn_idx,
5731                          (unsigned long long)imm, new_val);
5732                 break;
5733         }
5734         default:
5735                 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
5736                         prog->name, relo_idx, insn_idx, insn->code,
5737                         insn->src_reg, insn->dst_reg, insn->off, insn->imm);
5738                 return -EINVAL;
5739         }
5740
5741         return 0;
5742 }
5743
5744 /* Output spec definition in the format:
5745  * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
5746  * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
5747  */
5748 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
5749 {
5750         const struct btf_type *t;
5751         const struct btf_enum *e;
5752         const char *s;
5753         __u32 type_id;
5754         int i;
5755
5756         type_id = spec->root_type_id;
5757         t = btf__type_by_id(spec->btf, type_id);
5758         s = btf__name_by_offset(spec->btf, t->name_off);
5759
5760         libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
5761
5762         if (core_relo_is_type_based(spec->relo_kind))
5763                 return;
5764
5765         if (core_relo_is_enumval_based(spec->relo_kind)) {
5766                 t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
5767                 e = btf_enum(t) + spec->raw_spec[0];
5768                 s = btf__name_by_offset(spec->btf, e->name_off);
5769
5770                 libbpf_print(level, "::%s = %u", s, e->val);
5771                 return;
5772         }
5773
5774         if (core_relo_is_field_based(spec->relo_kind)) {
5775                 for (i = 0; i < spec->len; i++) {
5776                         if (spec->spec[i].name)
5777                                 libbpf_print(level, ".%s", spec->spec[i].name);
5778                         else if (i > 0 || spec->spec[i].idx > 0)
5779                                 libbpf_print(level, "[%u]", spec->spec[i].idx);
5780                 }
5781
5782                 libbpf_print(level, " (");
5783                 for (i = 0; i < spec->raw_len; i++)
5784                         libbpf_print(level, "%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
5785
5786                 if (spec->bit_offset % 8)
5787                         libbpf_print(level, " @ offset %u.%u)",
5788                                      spec->bit_offset / 8, spec->bit_offset % 8);
5789                 else
5790                         libbpf_print(level, " @ offset %u)", spec->bit_offset / 8);
5791                 return;
5792         }
5793 }
5794
5795 static size_t bpf_core_hash_fn(const void *key, void *ctx)
5796 {
5797         return (size_t)key;
5798 }
5799
5800 static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
5801 {
5802         return k1 == k2;
5803 }
5804
5805 static void *u32_as_hash_key(__u32 x)
5806 {
5807         return (void *)(uintptr_t)x;
5808 }
5809
5810 /*
5811  * CO-RE relocate single instruction.
5812  *
5813  * The outline and important points of the algorithm:
5814  * 1. For given local type, find corresponding candidate target types.
5815  *    Candidate type is a type with the same "essential" name, ignoring
5816  *    everything after last triple underscore (___). E.g., `sample`,
5817  *    `sample___flavor_one`, `sample___flavor_another_one`, are all candidates
5818  *    for each other. Names with triple underscore are referred to as
5819  *    "flavors" and are useful, among other things, to allow to
5820  *    specify/support incompatible variations of the same kernel struct, which
5821  *    might differ between different kernel versions and/or build
5822  *    configurations.
5823  *
5824  *    N.B. Struct "flavors" could be generated by bpftool's BTF-to-C
5825  *    converter, when deduplicated BTF of a kernel still contains more than
5826  *    one different types with the same name. In that case, ___2, ___3, etc
5827  *    are appended starting from second name conflict. But start flavors are
5828  *    also useful to be defined "locally", in BPF program, to extract same
5829  *    data from incompatible changes between different kernel
5830  *    versions/configurations. For instance, to handle field renames between
5831  *    kernel versions, one can use two flavors of the struct name with the
5832  *    same common name and use conditional relocations to extract that field,
5833  *    depending on target kernel version.
5834  * 2. For each candidate type, try to match local specification to this
5835  *    candidate target type. Matching involves finding corresponding
5836  *    high-level spec accessors, meaning that all named fields should match,
5837  *    as well as all array accesses should be within the actual bounds. Also,
5838  *    types should be compatible (see bpf_core_fields_are_compat for details).
5839  * 3. It is supported and expected that there might be multiple flavors
5840  *    matching the spec. As long as all the specs resolve to the same set of
5841  *    offsets across all candidates, there is no error. If there is any
5842  *    ambiguity, CO-RE relocation will fail. This is necessary to accomodate
5843  *    imprefection of BTF deduplication, which can cause slight duplication of
5844  *    the same BTF type, if some directly or indirectly referenced (by
5845  *    pointer) type gets resolved to different actual types in different
5846  *    object files. If such situation occurs, deduplicated BTF will end up
5847  *    with two (or more) structurally identical types, which differ only in
5848  *    types they refer to through pointer. This should be OK in most cases and
5849  *    is not an error.
5850  * 4. Candidate types search is performed by linearly scanning through all
5851  *    types in target BTF. It is anticipated that this is overall more
5852  *    efficient memory-wise and not significantly worse (if not better)
5853  *    CPU-wise compared to prebuilding a map from all local type names to
5854  *    a list of candidate type names. It's also sped up by caching resolved
5855  *    list of matching candidates per each local "root" type ID, that has at
5856  *    least one bpf_core_relo associated with it. This list is shared
5857  *    between multiple relocations for the same type ID and is updated as some
5858  *    of the candidates are pruned due to structural incompatibility.
5859  */
5860 static int bpf_core_apply_relo(struct bpf_program *prog,
5861                                const struct bpf_core_relo *relo,
5862                                int relo_idx,
5863                                const struct btf *local_btf,
5864                                struct hashmap *cand_cache)
5865 {
5866         struct bpf_core_spec local_spec, cand_spec, targ_spec = {};
5867         const void *type_key = u32_as_hash_key(relo->type_id);
5868         struct bpf_core_relo_res cand_res, targ_res;
5869         const struct btf_type *local_type;
5870         const char *local_name;
5871         struct core_cand_list *cands = NULL;
5872         __u32 local_id;
5873         const char *spec_str;
5874         int i, j, err;
5875
5876         local_id = relo->type_id;
5877         local_type = btf__type_by_id(local_btf, local_id);
5878         if (!local_type)
5879                 return -EINVAL;
5880
5881         local_name = btf__name_by_offset(local_btf, local_type->name_off);
5882         if (!local_name)
5883                 return -EINVAL;
5884
5885         spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
5886         if (str_is_empty(spec_str))
5887                 return -EINVAL;
5888
5889         err = bpf_core_parse_spec(local_btf, local_id, spec_str, relo->kind, &local_spec);
5890         if (err) {
5891                 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
5892                         prog->name, relo_idx, local_id, btf_kind_str(local_type),
5893                         str_is_empty(local_name) ? "<anon>" : local_name,
5894                         spec_str, err);
5895                 return -EINVAL;
5896         }
5897
5898         pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog->name,
5899                  relo_idx, core_relo_kind_str(relo->kind), relo->kind);
5900         bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
5901         libbpf_print(LIBBPF_DEBUG, "\n");
5902
5903         /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
5904         if (relo->kind == BPF_TYPE_ID_LOCAL) {
5905                 targ_res.validate = true;
5906                 targ_res.poison = false;
5907                 targ_res.orig_val = local_spec.root_type_id;
5908                 targ_res.new_val = local_spec.root_type_id;
5909                 goto patch_insn;
5910         }
5911
5912         /* libbpf doesn't support candidate search for anonymous types */
5913         if (str_is_empty(spec_str)) {
5914                 pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
5915                         prog->name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
5916                 return -EOPNOTSUPP;
5917         }
5918
5919         if (!hashmap__find(cand_cache, type_key, (void **)&cands)) {
5920                 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5921                 if (IS_ERR(cands)) {
5922                         pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5923                                 prog->name, relo_idx, local_id, btf_kind_str(local_type),
5924                                 local_name, PTR_ERR(cands));
5925                         return PTR_ERR(cands);
5926                 }
5927                 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
5928                 if (err) {
5929                         bpf_core_free_cands(cands);
5930                         return err;
5931                 }
5932         }
5933
5934         for (i = 0, j = 0; i < cands->len; i++) {
5935                 err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,
5936                                           cands->cands[i].id, &cand_spec);
5937                 if (err < 0) {
5938                         pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
5939                                 prog->name, relo_idx, i);
5940                         bpf_core_dump_spec(LIBBPF_WARN, &cand_spec);
5941                         libbpf_print(LIBBPF_WARN, ": %d\n", err);
5942                         return err;
5943                 }
5944
5945                 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog->name,
5946                          relo_idx, err == 0 ? "non-matching" : "matching", i);
5947                 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
5948                 libbpf_print(LIBBPF_DEBUG, "\n");
5949
5950                 if (err == 0)
5951                         continue;
5952
5953                 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, &cand_spec, &cand_res);
5954                 if (err)
5955                         return err;
5956
5957                 if (j == 0) {
5958                         targ_res = cand_res;
5959                         targ_spec = cand_spec;
5960                 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
5961                         /* if there are many field relo candidates, they
5962                          * should all resolve to the same bit offset
5963                          */
5964                         pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
5965                                 prog->name, relo_idx, cand_spec.bit_offset,
5966                                 targ_spec.bit_offset);
5967                         return -EINVAL;
5968                 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {
5969                         /* all candidates should result in the same relocation
5970                          * decision and value, otherwise it's dangerous to
5971                          * proceed due to ambiguity
5972                          */
5973                         pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n",
5974                                 prog->name, relo_idx,
5975                                 cand_res.poison ? "failure" : "success", cand_res.new_val,
5976                                 targ_res.poison ? "failure" : "success", targ_res.new_val);
5977                         return -EINVAL;
5978                 }
5979
5980                 cands->cands[j++] = cands->cands[i];
5981         }
5982
5983         /*
5984          * For BPF_FIELD_EXISTS relo or when used BPF program has field
5985          * existence checks or kernel version/config checks, it's expected
5986          * that we might not find any candidates. In this case, if field
5987          * wasn't found in any candidate, the list of candidates shouldn't
5988          * change at all, we'll just handle relocating appropriately,
5989          * depending on relo's kind.
5990          */
5991         if (j > 0)
5992                 cands->len = j;
5993
5994         /*
5995          * If no candidates were found, it might be both a programmer error,
5996          * as well as expected case, depending whether instruction w/
5997          * relocation is guarded in some way that makes it unreachable (dead
5998          * code) if relocation can't be resolved. This is handled in
5999          * bpf_core_patch_insn() uniformly by replacing that instruction with
6000          * BPF helper call insn (using invalid helper ID). If that instruction
6001          * is indeed unreachable, then it will be ignored and eliminated by
6002          * verifier. If it was an error, then verifier will complain and point
6003          * to a specific instruction number in its log.
6004          */
6005         if (j == 0) {
6006                 pr_debug("prog '%s': relo #%d: no matching targets found\n",
6007                          prog->name, relo_idx);
6008
6009                 /* calculate single target relo result explicitly */
6010                 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, NULL, &targ_res);
6011                 if (err)
6012                         return err;
6013         }
6014
6015 patch_insn:
6016         /* bpf_core_patch_insn() should know how to handle missing targ_spec */
6017         err = bpf_core_patch_insn(prog, relo, relo_idx, &targ_res);
6018         if (err) {
6019                 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
6020                         prog->name, relo_idx, relo->insn_off, err);
6021                 return -EINVAL;
6022         }
6023
6024         return 0;
6025 }
6026
6027 static int
6028 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
6029 {
6030         const struct btf_ext_info_sec *sec;
6031         const struct bpf_core_relo *rec;
6032         const struct btf_ext_info *seg;
6033         struct hashmap_entry *entry;
6034         struct hashmap *cand_cache = NULL;
6035         struct bpf_program *prog;
6036         const char *sec_name;
6037         int i, err = 0, insn_idx, sec_idx;
6038
6039         if (obj->btf_ext->core_relo_info.len == 0)
6040                 return 0;
6041
6042         if (targ_btf_path) {
6043                 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6044                 if (IS_ERR_OR_NULL(obj->btf_vmlinux_override)) {
6045                         err = PTR_ERR(obj->btf_vmlinux_override);
6046                         pr_warn("failed to parse target BTF: %d\n", err);
6047                         return err;
6048                 }
6049         }
6050
6051         cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
6052         if (IS_ERR(cand_cache)) {
6053                 err = PTR_ERR(cand_cache);
6054                 goto out;
6055         }
6056
6057         seg = &obj->btf_ext->core_relo_info;
6058         for_each_btf_ext_sec(seg, sec) {
6059                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6060                 if (str_is_empty(sec_name)) {
6061                         err = -EINVAL;
6062                         goto out;
6063                 }
6064                 /* bpf_object's ELF is gone by now so it's not easy to find
6065                  * section index by section name, but we can find *any*
6066                  * bpf_program within desired section name and use it's
6067                  * prog->sec_idx to do a proper search by section index and
6068                  * instruction offset
6069                  */
6070                 prog = NULL;
6071                 for (i = 0; i < obj->nr_programs; i++) {
6072                         prog = &obj->programs[i];
6073                         if (strcmp(prog->sec_name, sec_name) == 0)
6074                                 break;
6075                 }
6076                 if (!prog) {
6077                         pr_warn("sec '%s': failed to find a BPF program\n", sec_name);
6078                         return -ENOENT;
6079                 }
6080                 sec_idx = prog->sec_idx;
6081
6082                 pr_debug("sec '%s': found %d CO-RE relocations\n",
6083                          sec_name, sec->num_info);
6084
6085                 for_each_btf_ext_rec(seg, sec, i, rec) {
6086                         insn_idx = rec->insn_off / BPF_INSN_SZ;
6087                         prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
6088                         if (!prog) {
6089                                 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n",
6090                                         sec_name, insn_idx, i);
6091                                 err = -EINVAL;
6092                                 goto out;
6093                         }
6094                         /* no need to apply CO-RE relocation if the program is
6095                          * not going to be loaded
6096                          */
6097                         if (!prog->load)
6098                                 continue;
6099
6100                         err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache);
6101                         if (err) {
6102                                 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
6103                                         prog->name, i, err);
6104                                 goto out;
6105                         }
6106                 }
6107         }
6108
6109 out:
6110         /* obj->btf_vmlinux and module BTFs are freed after object load */
6111         btf__free(obj->btf_vmlinux_override);
6112         obj->btf_vmlinux_override = NULL;
6113
6114         if (!IS_ERR_OR_NULL(cand_cache)) {
6115                 hashmap__for_each_entry(cand_cache, entry, i) {
6116                         bpf_core_free_cands(entry->value);
6117                 }
6118                 hashmap__free(cand_cache);
6119         }
6120         return err;
6121 }
6122
6123 /* Relocate data references within program code:
6124  *  - map references;
6125  *  - global variable references;
6126  *  - extern references.
6127  */
6128 static int
6129 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
6130 {
6131         int i;
6132
6133         for (i = 0; i < prog->nr_reloc; i++) {
6134                 struct reloc_desc *relo = &prog->reloc_desc[i];
6135                 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6136                 struct extern_desc *ext;
6137
6138                 switch (relo->type) {
6139                 case RELO_LD64:
6140                         insn[0].src_reg = BPF_PSEUDO_MAP_FD;
6141                         insn[0].imm = obj->maps[relo->map_idx].fd;
6142                         relo->processed = true;
6143                         break;
6144                 case RELO_DATA:
6145                         insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6146                         insn[1].imm = insn[0].imm + relo->sym_off;
6147                         insn[0].imm = obj->maps[relo->map_idx].fd;
6148                         relo->processed = true;
6149                         break;
6150                 case RELO_EXTERN:
6151                         ext = &obj->externs[relo->sym_off];
6152                         if (ext->type == EXT_KCFG) {
6153                                 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6154                                 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6155                                 insn[1].imm = ext->kcfg.data_off;
6156                         } else /* EXT_KSYM */ {
6157                                 if (ext->ksym.type_id) { /* typed ksyms */
6158                                         insn[0].src_reg = BPF_PSEUDO_BTF_ID;
6159                                         insn[0].imm = ext->ksym.vmlinux_btf_id;
6160                                 } else { /* typeless ksyms */
6161                                         insn[0].imm = (__u32)ext->ksym.addr;
6162                                         insn[1].imm = ext->ksym.addr >> 32;
6163                                 }
6164                         }
6165                         relo->processed = true;
6166                         break;
6167                 case RELO_CALL:
6168                         /* will be handled as a follow up pass */
6169                         break;
6170                 default:
6171                         pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6172                                 prog->name, i, relo->type);
6173                         return -EINVAL;
6174                 }
6175         }
6176
6177         return 0;
6178 }
6179
6180 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
6181                                     const struct bpf_program *prog,
6182                                     const struct btf_ext_info *ext_info,
6183                                     void **prog_info, __u32 *prog_rec_cnt,
6184                                     __u32 *prog_rec_sz)
6185 {
6186         void *copy_start = NULL, *copy_end = NULL;
6187         void *rec, *rec_end, *new_prog_info;
6188         const struct btf_ext_info_sec *sec;
6189         size_t old_sz, new_sz;
6190         const char *sec_name;
6191         int i, off_adj;
6192
6193         for_each_btf_ext_sec(ext_info, sec) {
6194                 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6195                 if (!sec_name)
6196                         return -EINVAL;
6197                 if (strcmp(sec_name, prog->sec_name) != 0)
6198                         continue;
6199
6200                 for_each_btf_ext_rec(ext_info, sec, i, rec) {
6201                         __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
6202
6203                         if (insn_off < prog->sec_insn_off)
6204                                 continue;
6205                         if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6206                                 break;
6207
6208                         if (!copy_start)
6209                                 copy_start = rec;
6210                         copy_end = rec + ext_info->rec_size;
6211                 }
6212
6213                 if (!copy_start)
6214                         return -ENOENT;
6215
6216                 /* append func/line info of a given (sub-)program to the main
6217                  * program func/line info
6218                  */
6219                 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6220                 new_sz = old_sz + (copy_end - copy_start);
6221                 new_prog_info = realloc(*prog_info, new_sz);
6222                 if (!new_prog_info)
6223                         return -ENOMEM;
6224                 *prog_info = new_prog_info;
6225                 *prog_rec_cnt = new_sz / ext_info->rec_size;
6226                 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6227
6228                 /* Kernel instruction offsets are in units of 8-byte
6229                  * instructions, while .BTF.ext instruction offsets generated
6230                  * by Clang are in units of bytes. So convert Clang offsets
6231                  * into kernel offsets and adjust offset according to program
6232                  * relocated position.
6233                  */
6234                 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6235                 rec = new_prog_info + old_sz;
6236                 rec_end = new_prog_info + new_sz;
6237                 for (; rec < rec_end; rec += ext_info->rec_size) {
6238                         __u32 *insn_off = rec;
6239
6240                         *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
6241                 }
6242                 *prog_rec_sz = ext_info->rec_size;
6243                 return 0;
6244         }
6245
6246         return -ENOENT;
6247 }
6248
6249 static int
6250 reloc_prog_func_and_line_info(const struct bpf_object *obj,
6251                               struct bpf_program *main_prog,
6252                               const struct bpf_program *prog)
6253 {
6254         int err;
6255
6256         /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't
6257          * supprot func/line info
6258          */
6259         if (!obj->btf_ext || !kernel_supports(FEAT_BTF_FUNC))
6260                 return 0;
6261
6262         /* only attempt func info relocation if main program's func_info
6263          * relocation was successful
6264          */
6265         if (main_prog != prog && !main_prog->func_info)
6266                 goto line_info;
6267
6268         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6269                                        &main_prog->func_info,
6270                                        &main_prog->func_info_cnt,
6271                                        &main_prog->func_info_rec_size);
6272         if (err) {
6273                 if (err != -ENOENT) {
6274                         pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6275                                 prog->name, err);
6276                         return err;
6277                 }
6278                 if (main_prog->func_info) {
6279                         /*
6280                          * Some info has already been found but has problem
6281                          * in the last btf_ext reloc. Must have to error out.
6282                          */
6283                         pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6284                         return err;
6285                 }
6286                 /* Have problem loading the very first info. Ignore the rest. */
6287                 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6288                         prog->name);
6289         }
6290
6291 line_info:
6292         /* don't relocate line info if main program's relocation failed */
6293         if (main_prog != prog && !main_prog->line_info)
6294                 return 0;
6295
6296         err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6297                                        &main_prog->line_info,
6298                                        &main_prog->line_info_cnt,
6299                                        &main_prog->line_info_rec_size);
6300         if (err) {
6301                 if (err != -ENOENT) {
6302                         pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6303                                 prog->name, err);
6304                         return err;
6305                 }
6306                 if (main_prog->line_info) {
6307                         /*
6308                          * Some info has already been found but has problem
6309                          * in the last btf_ext reloc. Must have to error out.
6310                          */
6311                         pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6312                         return err;
6313                 }
6314                 /* Have problem loading the very first info. Ignore the rest. */
6315                 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6316                         prog->name);
6317         }
6318         return 0;
6319 }
6320
6321 static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6322 {
6323         size_t insn_idx = *(const size_t *)key;
6324         const struct reloc_desc *relo = elem;
6325
6326         if (insn_idx == relo->insn_idx)
6327                 return 0;
6328         return insn_idx < relo->insn_idx ? -1 : 1;
6329 }
6330
6331 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6332 {
6333         return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6334                        sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6335 }
6336
6337 static int
6338 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6339                        struct bpf_program *prog)
6340 {
6341         size_t sub_insn_idx, insn_idx, new_cnt;
6342         struct bpf_program *subprog;
6343         struct bpf_insn *insns, *insn;
6344         struct reloc_desc *relo;
6345         int err;
6346
6347         err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6348         if (err)
6349                 return err;
6350
6351         for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6352                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6353                 if (!insn_is_subprog_call(insn))
6354                         continue;
6355
6356                 relo = find_prog_insn_relo(prog, insn_idx);
6357                 if (relo && relo->type != RELO_CALL) {
6358                         pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6359                                 prog->name, insn_idx, relo->type);
6360                         return -LIBBPF_ERRNO__RELOC;
6361                 }
6362                 if (relo) {
6363                         /* sub-program instruction index is a combination of
6364                          * an offset of a symbol pointed to by relocation and
6365                          * call instruction's imm field; for global functions,
6366                          * call always has imm = -1, but for static functions
6367                          * relocation is against STT_SECTION and insn->imm
6368                          * points to a start of a static function
6369                          */
6370                         sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6371                 } else {
6372                         /* if subprogram call is to a static function within
6373                          * the same ELF section, there won't be any relocation
6374                          * emitted, but it also means there is no additional
6375                          * offset necessary, insns->imm is relative to
6376                          * instruction's original position within the section
6377                          */
6378                         sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6379                 }
6380
6381                 /* we enforce that sub-programs should be in .text section */
6382                 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6383                 if (!subprog) {
6384                         pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6385                                 prog->name);
6386                         return -LIBBPF_ERRNO__RELOC;
6387                 }
6388
6389                 /* if it's the first call instruction calling into this
6390                  * subprogram (meaning this subprog hasn't been processed
6391                  * yet) within the context of current main program:
6392                  *   - append it at the end of main program's instructions blog;
6393                  *   - process is recursively, while current program is put on hold;
6394                  *   - if that subprogram calls some other not yet processes
6395                  *   subprogram, same thing will happen recursively until
6396                  *   there are no more unprocesses subprograms left to append
6397                  *   and relocate.
6398                  */
6399                 if (subprog->sub_insn_off == 0) {
6400                         subprog->sub_insn_off = main_prog->insns_cnt;
6401
6402                         new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6403                         insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6404                         if (!insns) {
6405                                 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6406                                 return -ENOMEM;
6407                         }
6408                         main_prog->insns = insns;
6409                         main_prog->insns_cnt = new_cnt;
6410
6411                         memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6412                                subprog->insns_cnt * sizeof(*insns));
6413
6414                         pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6415                                  main_prog->name, subprog->insns_cnt, subprog->name);
6416
6417                         err = bpf_object__reloc_code(obj, main_prog, subprog);
6418                         if (err)
6419                                 return err;
6420                 }
6421
6422                 /* main_prog->insns memory could have been re-allocated, so
6423                  * calculate pointer again
6424                  */
6425                 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6426                 /* calculate correct instruction position within current main
6427                  * prog; each main prog can have a different set of
6428                  * subprograms appended (potentially in different order as
6429                  * well), so position of any subprog can be different for
6430                  * different main programs */
6431                 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6432
6433                 if (relo)
6434                         relo->processed = true;
6435
6436                 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6437                          prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6438         }
6439
6440         return 0;
6441 }
6442
6443 /*
6444  * Relocate sub-program calls.
6445  *
6446  * Algorithm operates as follows. Each entry-point BPF program (referred to as
6447  * main prog) is processed separately. For each subprog (non-entry functions,
6448  * that can be called from either entry progs or other subprogs) gets their
6449  * sub_insn_off reset to zero. This serves as indicator that this subprogram
6450  * hasn't been yet appended and relocated within current main prog. Once its
6451  * relocated, sub_insn_off will point at the position within current main prog
6452  * where given subprog was appended. This will further be used to relocate all
6453  * the call instructions jumping into this subprog.
6454  *
6455  * We start with main program and process all call instructions. If the call
6456  * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6457  * is zero), subprog instructions are appended at the end of main program's
6458  * instruction array. Then main program is "put on hold" while we recursively
6459  * process newly appended subprogram. If that subprogram calls into another
6460  * subprogram that hasn't been appended, new subprogram is appended again to
6461  * the *main* prog's instructions (subprog's instructions are always left
6462  * untouched, as they need to be in unmodified state for subsequent main progs
6463  * and subprog instructions are always sent only as part of a main prog) and
6464  * the process continues recursively. Once all the subprogs called from a main
6465  * prog or any of its subprogs are appended (and relocated), all their
6466  * positions within finalized instructions array are known, so it's easy to
6467  * rewrite call instructions with correct relative offsets, corresponding to
6468  * desired target subprog.
6469  *
6470  * Its important to realize that some subprogs might not be called from some
6471  * main prog and any of its called/used subprogs. Those will keep their
6472  * subprog->sub_insn_off as zero at all times and won't be appended to current
6473  * main prog and won't be relocated within the context of current main prog.
6474  * They might still be used from other main progs later.
6475  *
6476  * Visually this process can be shown as below. Suppose we have two main
6477  * programs mainA and mainB and BPF object contains three subprogs: subA,
6478  * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and
6479  * subC both call subB:
6480  *
6481  *        +--------+ +-------+
6482  *        |        v v       |
6483  *     +--+---+ +--+-+-+ +---+--+
6484  *     | subA | | subB | | subC |
6485  *     +--+---+ +------+ +---+--+
6486  *        ^                  ^
6487  *        |                  |
6488  *    +---+-------+   +------+----+
6489  *    |   mainA   |   |   mainB   |
6490  *    +-----------+   +-----------+
6491  *
6492  * We'll start relocating mainA, will find subA, append it and start
6493  * processing sub A recursively:
6494  *
6495  *    +-----------+------+
6496  *    |   mainA   | subA |
6497  *    +-----------+------+
6498  *
6499  * At this point we notice that subB is used from subA, so we append it and
6500  * relocate (there are no further subcalls from subB):
6501  *
6502  *    +-----------+------+------+
6503  *    |   mainA   | subA | subB |
6504  *    +-----------+------+------+
6505  *
6506  * At this point, we relocate subA calls, then go one level up and finish with
6507  * relocatin mainA calls. mainA is done.
6508  *
6509  * For mainB process is similar but results in different order. We start with
6510  * mainB and skip subA and subB, as mainB never calls them (at least
6511  * directly), but we see subC is needed, so we append and start processing it:
6512  *
6513  *    +-----------+------+
6514  *    |   mainB   | subC |
6515  *    +-----------+------+
6516  * Now we see subC needs subB, so we go back to it, append and relocate it:
6517  *
6518  *    +-----------+------+------+
6519  *    |   mainB   | subC | subB |
6520  *    +-----------+------+------+
6521  *
6522  * At this point we unwind recursion, relocate calls in subC, then in mainB.
6523  */
6524 static int
6525 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6526 {
6527         struct bpf_program *subprog;
6528         int i, j, err;
6529
6530         /* mark all subprogs as not relocated (yet) within the context of
6531          * current main program
6532          */
6533         for (i = 0; i < obj->nr_programs; i++) {
6534                 subprog = &obj->programs[i];
6535                 if (!prog_is_subprog(obj, subprog))
6536                         continue;
6537
6538                 subprog->sub_insn_off = 0;
6539                 for (j = 0; j < subprog->nr_reloc; j++)
6540                         if (subprog->reloc_desc[j].type == RELO_CALL)
6541                                 subprog->reloc_desc[j].processed = false;
6542         }
6543
6544         err = bpf_object__reloc_code(obj, prog, prog);
6545         if (err)
6546                 return err;
6547
6548
6549         return 0;
6550 }
6551
6552 static int
6553 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6554 {
6555         struct bpf_program *prog;
6556         size_t i;
6557         int err;
6558
6559         if (obj->btf_ext) {
6560                 err = bpf_object__relocate_core(obj, targ_btf_path);
6561                 if (err) {
6562                         pr_warn("failed to perform CO-RE relocations: %d\n",
6563                                 err);
6564                         return err;
6565                 }
6566         }
6567         /* relocate data references first for all programs and sub-programs,
6568          * as they don't change relative to code locations, so subsequent
6569          * subprogram processing won't need to re-calculate any of them
6570          */
6571         for (i = 0; i < obj->nr_programs; i++) {
6572                 prog = &obj->programs[i];
6573                 err = bpf_object__relocate_data(obj, prog);
6574                 if (err) {
6575                         pr_warn("prog '%s': failed to relocate data references: %d\n",
6576                                 prog->name, err);
6577                         return err;
6578                 }
6579         }
6580         /* now relocate subprogram calls and append used subprograms to main
6581          * programs; each copy of subprogram code needs to be relocated
6582          * differently for each main program, because its code location might
6583          * have changed
6584          */
6585         for (i = 0; i < obj->nr_programs; i++) {
6586                 prog = &obj->programs[i];
6587                 /* sub-program's sub-calls are relocated within the context of
6588                  * its main program only
6589                  */
6590                 if (prog_is_subprog(obj, prog))
6591                         continue;
6592
6593                 err = bpf_object__relocate_calls(obj, prog);
6594                 if (err) {
6595                         pr_warn("prog '%s': failed to relocate calls: %d\n",
6596                                 prog->name, err);
6597                         return err;
6598                 }
6599         }
6600         /* free up relocation descriptors */
6601         for (i = 0; i < obj->nr_programs; i++) {
6602                 prog = &obj->programs[i];
6603                 zfree(&prog->reloc_desc);
6604                 prog->nr_reloc = 0;
6605         }
6606         return 0;
6607 }
6608
6609 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6610                                             GElf_Shdr *shdr, Elf_Data *data);
6611
6612 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6613                                          GElf_Shdr *shdr, Elf_Data *data)
6614 {
6615         const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6616         int i, j, nrels, new_sz;
6617         const struct btf_var_secinfo *vi = NULL;
6618         const struct btf_type *sec, *var, *def;
6619         struct bpf_map *map = NULL, *targ_map;
6620         const struct btf_member *member;
6621         const char *name, *mname;
6622         Elf_Data *symbols;
6623         unsigned int moff;
6624         GElf_Sym sym;
6625         GElf_Rel rel;
6626         void *tmp;
6627
6628         if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6629                 return -EINVAL;
6630         sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6631         if (!sec)
6632                 return -EINVAL;
6633
6634         symbols = obj->efile.symbols;
6635         nrels = shdr->sh_size / shdr->sh_entsize;
6636         for (i = 0; i < nrels; i++) {
6637                 if (!gelf_getrel(data, i, &rel)) {
6638                         pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6639                         return -LIBBPF_ERRNO__FORMAT;
6640                 }
6641                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
6642                         pr_warn(".maps relo #%d: symbol %zx not found\n",
6643                                 i, (size_t)GELF_R_SYM(rel.r_info));
6644                         return -LIBBPF_ERRNO__FORMAT;
6645                 }
6646                 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
6647                 if (sym.st_shndx != obj->efile.btf_maps_shndx) {
6648                         pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6649                                 i, name);
6650                         return -LIBBPF_ERRNO__RELOC;
6651                 }
6652
6653                 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n",
6654                          i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value,
6655                          (size_t)rel.r_offset, sym.st_name, name);
6656
6657                 for (j = 0; j < obj->nr_maps; j++) {
6658                         map = &obj->maps[j];
6659                         if (map->sec_idx != obj->efile.btf_maps_shndx)
6660                                 continue;
6661
6662                         vi = btf_var_secinfos(sec) + map->btf_var_idx;
6663                         if (vi->offset <= rel.r_offset &&
6664                             rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6665                                 break;
6666                 }
6667                 if (j == obj->nr_maps) {
6668                         pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n",
6669                                 i, name, (size_t)rel.r_offset);
6670                         return -EINVAL;
6671                 }
6672
6673                 if (!bpf_map_type__is_map_in_map(map->def.type))
6674                         return -EINVAL;
6675                 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6676                     map->def.key_size != sizeof(int)) {
6677                         pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6678                                 i, map->name, sizeof(int));
6679                         return -EINVAL;
6680                 }
6681
6682                 targ_map = bpf_object__find_map_by_name(obj, name);
6683                 if (!targ_map)
6684                         return -ESRCH;
6685
6686                 var = btf__type_by_id(obj->btf, vi->type);
6687                 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6688                 if (btf_vlen(def) == 0)
6689                         return -EINVAL;
6690                 member = btf_members(def) + btf_vlen(def) - 1;
6691                 mname = btf__name_by_offset(obj->btf, member->name_off);
6692                 if (strcmp(mname, "values"))
6693                         return -EINVAL;
6694
6695                 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6696                 if (rel.r_offset - vi->offset < moff)
6697                         return -EINVAL;
6698
6699                 moff = rel.r_offset - vi->offset - moff;
6700                 /* here we use BPF pointer size, which is always 64 bit, as we
6701                  * are parsing ELF that was built for BPF target
6702                  */
6703                 if (moff % bpf_ptr_sz)
6704                         return -EINVAL;
6705                 moff /= bpf_ptr_sz;
6706                 if (moff >= map->init_slots_sz) {
6707                         new_sz = moff + 1;
6708                         tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6709                         if (!tmp)
6710                                 return -ENOMEM;
6711                         map->init_slots = tmp;
6712                         memset(map->init_slots + map->init_slots_sz, 0,
6713                                (new_sz - map->init_slots_sz) * host_ptr_sz);
6714                         map->init_slots_sz = new_sz;
6715                 }
6716                 map->init_slots[moff] = targ_map;
6717
6718                 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n",
6719                          i, map->name, moff, name);
6720         }
6721
6722         return 0;
6723 }
6724
6725 static int cmp_relocs(const void *_a, const void *_b)
6726 {
6727         const struct reloc_desc *a = _a;
6728         const struct reloc_desc *b = _b;
6729
6730         if (a->insn_idx != b->insn_idx)
6731                 return a->insn_idx < b->insn_idx ? -1 : 1;
6732
6733         /* no two relocations should have the same insn_idx, but ... */
6734         if (a->type != b->type)
6735                 return a->type < b->type ? -1 : 1;
6736
6737         return 0;
6738 }
6739
6740 static int bpf_object__collect_relos(struct bpf_object *obj)
6741 {
6742         int i, err;
6743
6744         for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
6745                 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
6746                 Elf_Data *data = obj->efile.reloc_sects[i].data;
6747                 int idx = shdr->sh_info;
6748
6749                 if (shdr->sh_type != SHT_REL) {
6750                         pr_warn("internal error at %d\n", __LINE__);
6751                         return -LIBBPF_ERRNO__INTERNAL;
6752                 }
6753
6754                 if (idx == obj->efile.st_ops_shndx)
6755                         err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6756                 else if (idx == obj->efile.btf_maps_shndx)
6757                         err = bpf_object__collect_map_relos(obj, shdr, data);
6758                 else
6759                         err = bpf_object__collect_prog_relos(obj, shdr, data);
6760                 if (err)
6761                         return err;
6762         }
6763
6764         for (i = 0; i < obj->nr_programs; i++) {
6765                 struct bpf_program *p = &obj->programs[i];
6766
6767                 if (!p->nr_reloc)
6768                         continue;
6769
6770                 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6771         }
6772         return 0;
6773 }
6774
6775 static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6776 {
6777         if (BPF_CLASS(insn->code) == BPF_JMP &&
6778             BPF_OP(insn->code) == BPF_CALL &&
6779             BPF_SRC(insn->code) == BPF_K &&
6780             insn->src_reg == 0 &&
6781             insn->dst_reg == 0) {
6782                     *func_id = insn->imm;
6783                     return true;
6784         }
6785         return false;
6786 }
6787
6788 static int bpf_object__sanitize_prog(struct bpf_object* obj, struct bpf_program *prog)
6789 {
6790         struct bpf_insn *insn = prog->insns;
6791         enum bpf_func_id func_id;
6792         int i;
6793
6794         for (i = 0; i < prog->insns_cnt; i++, insn++) {
6795                 if (!insn_is_helper_call(insn, &func_id))
6796                         continue;
6797
6798                 /* on kernels that don't yet support
6799                  * bpf_probe_read_{kernel,user}[_str] helpers, fall back
6800                  * to bpf_probe_read() which works well for old kernels
6801                  */
6802                 switch (func_id) {
6803                 case BPF_FUNC_probe_read_kernel:
6804                 case BPF_FUNC_probe_read_user:
6805                         if (!kernel_supports(FEAT_PROBE_READ_KERN))
6806                                 insn->imm = BPF_FUNC_probe_read;
6807                         break;
6808                 case BPF_FUNC_probe_read_kernel_str:
6809                 case BPF_FUNC_probe_read_user_str:
6810                         if (!kernel_supports(FEAT_PROBE_READ_KERN))
6811                                 insn->imm = BPF_FUNC_probe_read_str;
6812                         break;
6813                 default:
6814                         break;
6815                 }
6816         }
6817         return 0;
6818 }
6819
6820 static int
6821 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
6822              char *license, __u32 kern_version, int *pfd)
6823 {
6824         struct bpf_prog_load_params load_attr = {};
6825         char *cp, errmsg[STRERR_BUFSIZE];
6826         size_t log_buf_size = 0;
6827         char *log_buf = NULL;
6828         int btf_fd, ret;
6829
6830         if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6831                 /*
6832                  * The program type must be set.  Most likely we couldn't find a proper
6833                  * section definition at load time, and thus we didn't infer the type.
6834                  */
6835                 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6836                         prog->name, prog->sec_name);
6837                 return -EINVAL;
6838         }
6839
6840         if (!insns || !insns_cnt)
6841                 return -EINVAL;
6842
6843         load_attr.prog_type = prog->type;
6844         /* old kernels might not support specifying expected_attach_type */
6845         if (!kernel_supports(FEAT_EXP_ATTACH_TYPE) && prog->sec_def &&
6846             prog->sec_def->is_exp_attach_type_optional)
6847                 load_attr.expected_attach_type = 0;
6848         else
6849                 load_attr.expected_attach_type = prog->expected_attach_type;
6850         if (kernel_supports(FEAT_PROG_NAME))
6851                 load_attr.name = prog->name;
6852         load_attr.insns = insns;
6853         load_attr.insn_cnt = insns_cnt;
6854         load_attr.license = license;
6855         load_attr.attach_btf_id = prog->attach_btf_id;
6856         if (prog->attach_prog_fd)
6857                 load_attr.attach_prog_fd = prog->attach_prog_fd;
6858         else
6859                 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6860         load_attr.attach_btf_id = prog->attach_btf_id;
6861         load_attr.kern_version = kern_version;
6862         load_attr.prog_ifindex = prog->prog_ifindex;
6863
6864         /* specify func_info/line_info only if kernel supports them */
6865         btf_fd = bpf_object__btf_fd(prog->obj);
6866         if (btf_fd >= 0 && kernel_supports(FEAT_BTF_FUNC)) {
6867                 load_attr.prog_btf_fd = btf_fd;
6868                 load_attr.func_info = prog->func_info;
6869                 load_attr.func_info_rec_size = prog->func_info_rec_size;
6870                 load_attr.func_info_cnt = prog->func_info_cnt;
6871                 load_attr.line_info = prog->line_info;
6872                 load_attr.line_info_rec_size = prog->line_info_rec_size;
6873                 load_attr.line_info_cnt = prog->line_info_cnt;
6874         }
6875         load_attr.log_level = prog->log_level;
6876         load_attr.prog_flags = prog->prog_flags;
6877
6878 retry_load:
6879         if (log_buf_size) {
6880                 log_buf = malloc(log_buf_size);
6881                 if (!log_buf)
6882                         return -ENOMEM;
6883
6884                 *log_buf = 0;
6885         }
6886
6887         load_attr.log_buf = log_buf;
6888         load_attr.log_buf_sz = log_buf_size;
6889         ret = libbpf__bpf_prog_load(&load_attr);
6890
6891         if (ret >= 0) {
6892                 if (log_buf && load_attr.log_level)
6893                         pr_debug("verifier log:\n%s", log_buf);
6894
6895                 if (prog->obj->rodata_map_idx >= 0 &&
6896                     kernel_supports(FEAT_PROG_BIND_MAP)) {
6897                         struct bpf_map *rodata_map =
6898                                 &prog->obj->maps[prog->obj->rodata_map_idx];
6899
6900                         if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) {
6901                                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6902                                 pr_warn("prog '%s': failed to bind .rodata map: %s\n",
6903                                         prog->name, cp);
6904                                 /* Don't fail hard if can't bind rodata. */
6905                         }
6906                 }
6907
6908                 *pfd = ret;
6909                 ret = 0;
6910                 goto out;
6911         }
6912
6913         if (!log_buf || errno == ENOSPC) {
6914                 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
6915                                    log_buf_size << 1);
6916
6917                 free(log_buf);
6918                 goto retry_load;
6919         }
6920         ret = errno ? -errno : -LIBBPF_ERRNO__LOAD;
6921         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6922         pr_warn("load bpf program failed: %s\n", cp);
6923         pr_perm_msg(ret);
6924
6925         if (log_buf && log_buf[0] != '\0') {
6926                 ret = -LIBBPF_ERRNO__VERIFY;
6927                 pr_warn("-- BEGIN DUMP LOG ---\n");
6928                 pr_warn("\n%s\n", log_buf);
6929                 pr_warn("-- END LOG --\n");
6930         } else if (load_attr.insn_cnt >= BPF_MAXINSNS) {
6931                 pr_warn("Program too large (%zu insns), at most %d insns\n",
6932                         load_attr.insn_cnt, BPF_MAXINSNS);
6933                 ret = -LIBBPF_ERRNO__PROG2BIG;
6934         } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
6935                 /* Wrong program type? */
6936                 int fd;
6937
6938                 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
6939                 load_attr.expected_attach_type = 0;
6940                 load_attr.log_buf = NULL;
6941                 load_attr.log_buf_sz = 0;
6942                 fd = libbpf__bpf_prog_load(&load_attr);
6943                 if (fd >= 0) {
6944                         close(fd);
6945                         ret = -LIBBPF_ERRNO__PROGTYPE;
6946                         goto out;
6947                 }
6948         }
6949
6950 out:
6951         free(log_buf);
6952         return ret;
6953 }
6954
6955 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id);
6956
6957 int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
6958 {
6959         int err = 0, fd, i;
6960
6961         if (prog->obj->loaded) {
6962                 pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
6963                 return -EINVAL;
6964         }
6965
6966         if ((prog->type == BPF_PROG_TYPE_TRACING ||
6967              prog->type == BPF_PROG_TYPE_LSM ||
6968              prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
6969                 int btf_obj_fd = 0, btf_type_id = 0;
6970
6971                 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id);
6972                 if (err)
6973                         return err;
6974
6975                 prog->attach_btf_obj_fd = btf_obj_fd;
6976                 prog->attach_btf_id = btf_type_id;
6977         }
6978
6979         if (prog->instances.nr < 0 || !prog->instances.fds) {
6980                 if (prog->preprocessor) {
6981                         pr_warn("Internal error: can't load program '%s'\n",
6982                                 prog->name);
6983                         return -LIBBPF_ERRNO__INTERNAL;
6984                 }
6985
6986                 prog->instances.fds = malloc(sizeof(int));
6987                 if (!prog->instances.fds) {
6988                         pr_warn("Not enough memory for BPF fds\n");
6989                         return -ENOMEM;
6990                 }
6991                 prog->instances.nr = 1;
6992                 prog->instances.fds[0] = -1;
6993         }
6994
6995         if (!prog->preprocessor) {
6996                 if (prog->instances.nr != 1) {
6997                         pr_warn("prog '%s': inconsistent nr(%d) != 1\n",
6998                                 prog->name, prog->instances.nr);
6999                 }
7000                 err = load_program(prog, prog->insns, prog->insns_cnt,
7001                                    license, kern_ver, &fd);
7002                 if (!err)
7003                         prog->instances.fds[0] = fd;
7004                 goto out;
7005         }
7006
7007         for (i = 0; i < prog->instances.nr; i++) {
7008                 struct bpf_prog_prep_result result;
7009                 bpf_program_prep_t preprocessor = prog->preprocessor;
7010
7011                 memset(&result, 0, sizeof(result));
7012                 err = preprocessor(prog, i, prog->insns,
7013                                    prog->insns_cnt, &result);
7014                 if (err) {
7015                         pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
7016                                 i, prog->name);
7017                         goto out;
7018                 }
7019
7020                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
7021                         pr_debug("Skip loading the %dth instance of program '%s'\n",
7022                                  i, prog->name);
7023                         prog->instances.fds[i] = -1;
7024                         if (result.pfd)
7025                                 *result.pfd = -1;
7026                         continue;
7027                 }
7028
7029                 err = load_program(prog, result.new_insn_ptr,
7030                                    result.new_insn_cnt, license, kern_ver, &fd);
7031                 if (err) {
7032                         pr_warn("Loading the %dth instance of program '%s' failed\n",
7033                                 i, prog->name);
7034                         goto out;
7035                 }
7036
7037                 if (result.pfd)
7038                         *result.pfd = fd;
7039                 prog->instances.fds[i] = fd;
7040         }
7041 out:
7042         if (err)
7043                 pr_warn("failed to load program '%s'\n", prog->name);
7044         zfree(&prog->insns);
7045         prog->insns_cnt = 0;
7046         return err;
7047 }
7048
7049 static int
7050 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7051 {
7052         struct bpf_program *prog;
7053         size_t i;
7054         int err;
7055
7056         for (i = 0; i < obj->nr_programs; i++) {
7057                 prog = &obj->programs[i];
7058                 err = bpf_object__sanitize_prog(obj, prog);
7059                 if (err)
7060                         return err;
7061         }
7062
7063         for (i = 0; i < obj->nr_programs; i++) {
7064                 prog = &obj->programs[i];
7065                 if (prog_is_subprog(obj, prog))
7066                         continue;
7067                 if (!prog->load) {
7068                         pr_debug("prog '%s': skipped loading\n", prog->name);
7069                         continue;
7070                 }
7071                 prog->log_level |= log_level;
7072                 err = bpf_program__load(prog, obj->license, obj->kern_version);
7073                 if (err)
7074                         return err;
7075         }
7076         return 0;
7077 }
7078
7079 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7080
7081 static struct bpf_object *
7082 __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7083                    const struct bpf_object_open_opts *opts)
7084 {
7085         const char *obj_name, *kconfig;
7086         struct bpf_program *prog;
7087         struct bpf_object *obj;
7088         char tmp_name[64];
7089         int err;
7090
7091         if (elf_version(EV_CURRENT) == EV_NONE) {
7092                 pr_warn("failed to init libelf for %s\n",
7093                         path ? : "(mem buf)");
7094                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7095         }
7096
7097         if (!OPTS_VALID(opts, bpf_object_open_opts))
7098                 return ERR_PTR(-EINVAL);
7099
7100         obj_name = OPTS_GET(opts, object_name, NULL);
7101         if (obj_buf) {
7102                 if (!obj_name) {
7103                         snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7104                                  (unsigned long)obj_buf,
7105                                  (unsigned long)obj_buf_sz);
7106                         obj_name = tmp_name;
7107                 }
7108                 path = obj_name;
7109                 pr_debug("loading object '%s' from buffer\n", obj_name);
7110         }
7111
7112         obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7113         if (IS_ERR(obj))
7114                 return obj;
7115
7116         kconfig = OPTS_GET(opts, kconfig, NULL);
7117         if (kconfig) {
7118                 obj->kconfig = strdup(kconfig);
7119                 if (!obj->kconfig)
7120                         return ERR_PTR(-ENOMEM);
7121         }
7122
7123         err = bpf_object__elf_init(obj);
7124         err = err ? : bpf_object__check_endianness(obj);
7125         err = err ? : bpf_object__elf_collect(obj);
7126         err = err ? : bpf_object__collect_externs(obj);
7127         err = err ? : bpf_object__finalize_btf(obj);
7128         err = err ? : bpf_object__init_maps(obj, opts);
7129         err = err ? : bpf_object__collect_relos(obj);
7130         if (err)
7131                 goto out;
7132         bpf_object__elf_finish(obj);
7133
7134         bpf_object__for_each_program(prog, obj) {
7135                 prog->sec_def = find_sec_def(prog->sec_name);
7136                 if (!prog->sec_def) {
7137                         /* couldn't guess, but user might manually specify */
7138                         pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7139                                 prog->name, prog->sec_name);
7140                         continue;
7141                 }
7142
7143                 if (prog->sec_def->is_sleepable)
7144                         prog->prog_flags |= BPF_F_SLEEPABLE;
7145                 bpf_program__set_type(prog, prog->sec_def->prog_type);
7146                 bpf_program__set_expected_attach_type(prog,
7147                                 prog->sec_def->expected_attach_type);
7148
7149                 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
7150                     prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
7151                         prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
7152         }
7153
7154         return obj;
7155 out:
7156         bpf_object__close(obj);
7157         return ERR_PTR(err);
7158 }
7159
7160 static struct bpf_object *
7161 __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
7162 {
7163         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7164                 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
7165         );
7166
7167         /* param validation */
7168         if (!attr->file)
7169                 return NULL;
7170
7171         pr_debug("loading %s\n", attr->file);
7172         return __bpf_object__open(attr->file, NULL, 0, &opts);
7173 }
7174
7175 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
7176 {
7177         return __bpf_object__open_xattr(attr, 0);
7178 }
7179
7180 struct bpf_object *bpf_object__open(const char *path)
7181 {
7182         struct bpf_object_open_attr attr = {
7183                 .file           = path,
7184                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
7185         };
7186
7187         return bpf_object__open_xattr(&attr);
7188 }
7189
7190 struct bpf_object *
7191 bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7192 {
7193         if (!path)
7194                 return ERR_PTR(-EINVAL);
7195
7196         pr_debug("loading %s\n", path);
7197
7198         return __bpf_object__open(path, NULL, 0, opts);
7199 }
7200
7201 struct bpf_object *
7202 bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7203                      const struct bpf_object_open_opts *opts)
7204 {
7205         if (!obj_buf || obj_buf_sz == 0)
7206                 return ERR_PTR(-EINVAL);
7207
7208         return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
7209 }
7210
7211 struct bpf_object *
7212 bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
7213                         const char *name)
7214 {
7215         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7216                 .object_name = name,
7217                 /* wrong default, but backwards-compatible */
7218                 .relaxed_maps = true,
7219         );
7220
7221         /* returning NULL is wrong, but backwards-compatible */
7222         if (!obj_buf || obj_buf_sz == 0)
7223                 return NULL;
7224
7225         return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
7226 }
7227
7228 int bpf_object__unload(struct bpf_object *obj)
7229 {
7230         size_t i;
7231
7232         if (!obj)
7233                 return -EINVAL;
7234
7235         for (i = 0; i < obj->nr_maps; i++) {
7236                 zclose(obj->maps[i].fd);
7237                 if (obj->maps[i].st_ops)
7238                         zfree(&obj->maps[i].st_ops->kern_vdata);
7239         }
7240
7241         for (i = 0; i < obj->nr_programs; i++)
7242                 bpf_program__unload(&obj->programs[i]);
7243
7244         return 0;
7245 }
7246
7247 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7248 {
7249         struct bpf_map *m;
7250
7251         bpf_object__for_each_map(m, obj) {
7252                 if (!bpf_map__is_internal(m))
7253                         continue;
7254                 if (!kernel_supports(FEAT_GLOBAL_DATA)) {
7255                         pr_warn("kernel doesn't support global data\n");
7256                         return -ENOTSUP;
7257                 }
7258                 if (!kernel_supports(FEAT_ARRAY_MMAP))
7259                         m->def.map_flags ^= BPF_F_MMAPABLE;
7260         }
7261
7262         return 0;
7263 }
7264
7265 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7266 {
7267         char sym_type, sym_name[500];
7268         unsigned long long sym_addr;
7269         struct extern_desc *ext;
7270         int ret, err = 0;
7271         FILE *f;
7272
7273         f = fopen("/proc/kallsyms", "r");
7274         if (!f) {
7275                 err = -errno;
7276                 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7277                 return err;
7278         }
7279
7280         while (true) {
7281                 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7282                              &sym_addr, &sym_type, sym_name);
7283                 if (ret == EOF && feof(f))
7284                         break;
7285                 if (ret != 3) {
7286                         pr_warn("failed to read kallsyms entry: %d\n", ret);
7287                         err = -EINVAL;
7288                         goto out;
7289                 }
7290
7291                 ext = find_extern_by_name(obj, sym_name);
7292                 if (!ext || ext->type != EXT_KSYM)
7293                         continue;
7294
7295                 if (ext->is_set && ext->ksym.addr != sym_addr) {
7296                         pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
7297                                 sym_name, ext->ksym.addr, sym_addr);
7298                         err = -EINVAL;
7299                         goto out;
7300                 }
7301                 if (!ext->is_set) {
7302                         ext->is_set = true;
7303                         ext->ksym.addr = sym_addr;
7304                         pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr);
7305                 }
7306         }
7307
7308 out:
7309         fclose(f);
7310         return err;
7311 }
7312
7313 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7314 {
7315         struct extern_desc *ext;
7316         int i, id;
7317
7318         for (i = 0; i < obj->nr_extern; i++) {
7319                 const struct btf_type *targ_var, *targ_type;
7320                 __u32 targ_type_id, local_type_id;
7321                 const char *targ_var_name;
7322                 int ret;
7323
7324                 ext = &obj->externs[i];
7325                 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7326                         continue;
7327
7328                 id = btf__find_by_name_kind(obj->btf_vmlinux, ext->name,
7329                                             BTF_KIND_VAR);
7330                 if (id <= 0) {
7331                         pr_warn("extern (ksym) '%s': failed to find BTF ID in vmlinux BTF.\n",
7332                                 ext->name);
7333                         return -ESRCH;
7334                 }
7335
7336                 /* find local type_id */
7337                 local_type_id = ext->ksym.type_id;
7338
7339                 /* find target type_id */
7340                 targ_var = btf__type_by_id(obj->btf_vmlinux, id);
7341                 targ_var_name = btf__name_by_offset(obj->btf_vmlinux,
7342                                                     targ_var->name_off);
7343                 targ_type = skip_mods_and_typedefs(obj->btf_vmlinux,
7344                                                    targ_var->type,
7345                                                    &targ_type_id);
7346
7347                 ret = bpf_core_types_are_compat(obj->btf, local_type_id,
7348                                                 obj->btf_vmlinux, targ_type_id);
7349                 if (ret <= 0) {
7350                         const struct btf_type *local_type;
7351                         const char *targ_name, *local_name;
7352
7353                         local_type = btf__type_by_id(obj->btf, local_type_id);
7354                         local_name = btf__name_by_offset(obj->btf,
7355                                                          local_type->name_off);
7356                         targ_name = btf__name_by_offset(obj->btf_vmlinux,
7357                                                         targ_type->name_off);
7358
7359                         pr_warn("extern (ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7360                                 ext->name, local_type_id,
7361                                 btf_kind_str(local_type), local_name, targ_type_id,
7362                                 btf_kind_str(targ_type), targ_name);
7363                         return -EINVAL;
7364                 }
7365
7366                 ext->is_set = true;
7367                 ext->ksym.vmlinux_btf_id = id;
7368                 pr_debug("extern (ksym) '%s': resolved to [%d] %s %s\n",
7369                          ext->name, id, btf_kind_str(targ_var), targ_var_name);
7370         }
7371         return 0;
7372 }
7373
7374 static int bpf_object__resolve_externs(struct bpf_object *obj,
7375                                        const char *extra_kconfig)
7376 {
7377         bool need_config = false, need_kallsyms = false;
7378         bool need_vmlinux_btf = false;
7379         struct extern_desc *ext;
7380         void *kcfg_data = NULL;
7381         int err, i;
7382
7383         if (obj->nr_extern == 0)
7384                 return 0;
7385
7386         if (obj->kconfig_map_idx >= 0)
7387                 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7388
7389         for (i = 0; i < obj->nr_extern; i++) {
7390                 ext = &obj->externs[i];
7391
7392                 if (ext->type == EXT_KCFG &&
7393                     strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7394                         void *ext_val = kcfg_data + ext->kcfg.data_off;
7395                         __u32 kver = get_kernel_version();
7396
7397                         if (!kver) {
7398                                 pr_warn("failed to get kernel version\n");
7399                                 return -EINVAL;
7400                         }
7401                         err = set_kcfg_value_num(ext, ext_val, kver);
7402                         if (err)
7403                                 return err;
7404                         pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver);
7405                 } else if (ext->type == EXT_KCFG &&
7406                            strncmp(ext->name, "CONFIG_", 7) == 0) {
7407                         need_config = true;
7408                 } else if (ext->type == EXT_KSYM) {
7409                         if (ext->ksym.type_id)
7410                                 need_vmlinux_btf = true;
7411                         else
7412                                 need_kallsyms = true;
7413                 } else {
7414                         pr_warn("unrecognized extern '%s'\n", ext->name);
7415                         return -EINVAL;
7416                 }
7417         }
7418         if (need_config && extra_kconfig) {
7419                 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7420                 if (err)
7421                         return -EINVAL;
7422                 need_config = false;
7423                 for (i = 0; i < obj->nr_extern; i++) {
7424                         ext = &obj->externs[i];
7425                         if (ext->type == EXT_KCFG && !ext->is_set) {
7426                                 need_config = true;
7427                                 break;
7428                         }
7429                 }
7430         }
7431         if (need_config) {
7432                 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7433                 if (err)
7434                         return -EINVAL;
7435         }
7436         if (need_kallsyms) {
7437                 err = bpf_object__read_kallsyms_file(obj);
7438                 if (err)
7439                         return -EINVAL;
7440         }
7441         if (need_vmlinux_btf) {
7442                 err = bpf_object__resolve_ksyms_btf_id(obj);
7443                 if (err)
7444                         return -EINVAL;
7445         }
7446         for (i = 0; i < obj->nr_extern; i++) {
7447                 ext = &obj->externs[i];
7448
7449                 if (!ext->is_set && !ext->is_weak) {
7450                         pr_warn("extern %s (strong) not resolved\n", ext->name);
7451                         return -ESRCH;
7452                 } else if (!ext->is_set) {
7453                         pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
7454                                  ext->name);
7455                 }
7456         }
7457
7458         return 0;
7459 }
7460
7461 int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
7462 {
7463         struct bpf_object *obj;
7464         int err, i;
7465
7466         if (!attr)
7467                 return -EINVAL;
7468         obj = attr->obj;
7469         if (!obj)
7470                 return -EINVAL;
7471
7472         if (obj->loaded) {
7473                 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7474                 return -EINVAL;
7475         }
7476
7477         err = bpf_object__probe_loading(obj);
7478         err = err ? : bpf_object__load_vmlinux_btf(obj);
7479         err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7480         err = err ? : bpf_object__sanitize_and_load_btf(obj);
7481         err = err ? : bpf_object__sanitize_maps(obj);
7482         err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7483         err = err ? : bpf_object__create_maps(obj);
7484         err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
7485         err = err ? : bpf_object__load_progs(obj, attr->log_level);
7486
7487         /* clean up module BTFs */
7488         for (i = 0; i < obj->btf_module_cnt; i++) {
7489                 close(obj->btf_modules[i].fd);
7490                 btf__free(obj->btf_modules[i].btf);
7491                 free(obj->btf_modules[i].name);
7492         }
7493         free(obj->btf_modules);
7494
7495         /* clean up vmlinux BTF */
7496         btf__free(obj->btf_vmlinux);
7497         obj->btf_vmlinux = NULL;
7498
7499         obj->loaded = true; /* doesn't matter if successfully or not */
7500
7501         if (err)
7502                 goto out;
7503
7504         return 0;
7505 out:
7506         /* unpin any maps that were auto-pinned during load */
7507         for (i = 0; i < obj->nr_maps; i++)
7508                 if (obj->maps[i].pinned && !obj->maps[i].reused)
7509                         bpf_map__unpin(&obj->maps[i], NULL);
7510
7511         bpf_object__unload(obj);
7512         pr_warn("failed to load object '%s'\n", obj->path);
7513         return err;
7514 }
7515
7516 int bpf_object__load(struct bpf_object *obj)
7517 {
7518         struct bpf_object_load_attr attr = {
7519                 .obj = obj,
7520         };
7521
7522         return bpf_object__load_xattr(&attr);
7523 }
7524
7525 static int make_parent_dir(const char *path)
7526 {
7527         char *cp, errmsg[STRERR_BUFSIZE];
7528         char *dname, *dir;
7529         int err = 0;
7530
7531         dname = strdup(path);
7532         if (dname == NULL)
7533                 return -ENOMEM;
7534
7535         dir = dirname(dname);
7536         if (mkdir(dir, 0700) && errno != EEXIST)
7537                 err = -errno;
7538
7539         free(dname);
7540         if (err) {
7541                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7542                 pr_warn("failed to mkdir %s: %s\n", path, cp);
7543         }
7544         return err;
7545 }
7546
7547 static int check_path(const char *path)
7548 {
7549         char *cp, errmsg[STRERR_BUFSIZE];
7550         struct statfs st_fs;
7551         char *dname, *dir;
7552         int err = 0;
7553
7554         if (path == NULL)
7555                 return -EINVAL;
7556
7557         dname = strdup(path);
7558         if (dname == NULL)
7559                 return -ENOMEM;
7560
7561         dir = dirname(dname);
7562         if (statfs(dir, &st_fs)) {
7563                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7564                 pr_warn("failed to statfs %s: %s\n", dir, cp);
7565                 err = -errno;
7566         }
7567         free(dname);
7568
7569         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7570                 pr_warn("specified path %s is not on BPF FS\n", path);
7571                 err = -EINVAL;
7572         }
7573
7574         return err;
7575 }
7576
7577 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
7578                               int instance)
7579 {
7580         char *cp, errmsg[STRERR_BUFSIZE];
7581         int err;
7582
7583         err = make_parent_dir(path);
7584         if (err)
7585                 return err;
7586
7587         err = check_path(path);
7588         if (err)
7589                 return err;
7590
7591         if (prog == NULL) {
7592                 pr_warn("invalid program pointer\n");
7593                 return -EINVAL;
7594         }
7595
7596         if (instance < 0 || instance >= prog->instances.nr) {
7597                 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7598                         instance, prog->name, prog->instances.nr);
7599                 return -EINVAL;
7600         }
7601
7602         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
7603                 err = -errno;
7604                 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7605                 pr_warn("failed to pin program: %s\n", cp);
7606                 return err;
7607         }
7608         pr_debug("pinned program '%s'\n", path);
7609
7610         return 0;
7611 }
7612
7613 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
7614                                 int instance)
7615 {
7616         int err;
7617
7618         err = check_path(path);
7619         if (err)
7620                 return err;
7621
7622         if (prog == NULL) {
7623                 pr_warn("invalid program pointer\n");
7624                 return -EINVAL;
7625         }
7626
7627         if (instance < 0 || instance >= prog->instances.nr) {
7628                 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7629                         instance, prog->name, prog->instances.nr);
7630                 return -EINVAL;
7631         }
7632
7633         err = unlink(path);
7634         if (err != 0)
7635                 return -errno;
7636         pr_debug("unpinned program '%s'\n", path);
7637
7638         return 0;
7639 }
7640
7641 int bpf_program__pin(struct bpf_program *prog, const char *path)
7642 {
7643         int i, err;
7644
7645         err = make_parent_dir(path);
7646         if (err)
7647                 return err;
7648
7649         err = check_path(path);
7650         if (err)
7651                 return err;
7652
7653         if (prog == NULL) {
7654                 pr_warn("invalid program pointer\n");
7655                 return -EINVAL;
7656         }
7657
7658         if (prog->instances.nr <= 0) {
7659                 pr_warn("no instances of prog %s to pin\n", prog->name);
7660                 return -EINVAL;
7661         }
7662
7663         if (prog->instances.nr == 1) {
7664                 /* don't create subdirs when pinning single instance */
7665                 return bpf_program__pin_instance(prog, path, 0);
7666         }
7667
7668         for (i = 0; i < prog->instances.nr; i++) {
7669                 char buf[PATH_MAX];
7670                 int len;
7671
7672                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7673                 if (len < 0) {
7674                         err = -EINVAL;
7675                         goto err_unpin;
7676                 } else if (len >= PATH_MAX) {
7677                         err = -ENAMETOOLONG;
7678                         goto err_unpin;
7679                 }
7680
7681                 err = bpf_program__pin_instance(prog, buf, i);
7682                 if (err)
7683                         goto err_unpin;
7684         }
7685
7686         return 0;
7687
7688 err_unpin:
7689         for (i = i - 1; i >= 0; i--) {
7690                 char buf[PATH_MAX];
7691                 int len;
7692
7693                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7694                 if (len < 0)
7695                         continue;
7696                 else if (len >= PATH_MAX)
7697                         continue;
7698
7699                 bpf_program__unpin_instance(prog, buf, i);
7700         }
7701
7702         rmdir(path);
7703
7704         return err;
7705 }
7706
7707 int bpf_program__unpin(struct bpf_program *prog, const char *path)
7708 {
7709         int i, err;
7710
7711         err = check_path(path);
7712         if (err)
7713                 return err;
7714
7715         if (prog == NULL) {
7716                 pr_warn("invalid program pointer\n");
7717                 return -EINVAL;
7718         }
7719
7720         if (prog->instances.nr <= 0) {
7721                 pr_warn("no instances of prog %s to pin\n", prog->name);
7722                 return -EINVAL;
7723         }
7724
7725         if (prog->instances.nr == 1) {
7726                 /* don't create subdirs when pinning single instance */
7727                 return bpf_program__unpin_instance(prog, path, 0);
7728         }
7729
7730         for (i = 0; i < prog->instances.nr; i++) {
7731                 char buf[PATH_MAX];
7732                 int len;
7733
7734                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7735                 if (len < 0)
7736                         return -EINVAL;
7737                 else if (len >= PATH_MAX)
7738                         return -ENAMETOOLONG;
7739
7740                 err = bpf_program__unpin_instance(prog, buf, i);
7741                 if (err)
7742                         return err;
7743         }
7744
7745         err = rmdir(path);
7746         if (err)
7747                 return -errno;
7748
7749         return 0;
7750 }
7751
7752 int bpf_map__pin(struct bpf_map *map, const char *path)
7753 {
7754         char *cp, errmsg[STRERR_BUFSIZE];
7755         int err;
7756
7757         if (map == NULL) {
7758                 pr_warn("invalid map pointer\n");
7759                 return -EINVAL;
7760         }
7761
7762         if (map->pin_path) {
7763                 if (path && strcmp(path, map->pin_path)) {
7764                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7765                                 bpf_map__name(map), map->pin_path, path);
7766                         return -EINVAL;
7767                 } else if (map->pinned) {
7768                         pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7769                                  bpf_map__name(map), map->pin_path);
7770                         return 0;
7771                 }
7772         } else {
7773                 if (!path) {
7774                         pr_warn("missing a path to pin map '%s' at\n",
7775                                 bpf_map__name(map));
7776                         return -EINVAL;
7777                 } else if (map->pinned) {
7778                         pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7779                         return -EEXIST;
7780                 }
7781
7782                 map->pin_path = strdup(path);
7783                 if (!map->pin_path) {
7784                         err = -errno;
7785                         goto out_err;
7786                 }
7787         }
7788
7789         err = make_parent_dir(map->pin_path);
7790         if (err)
7791                 return err;
7792
7793         err = check_path(map->pin_path);
7794         if (err)
7795                 return err;
7796
7797         if (bpf_obj_pin(map->fd, map->pin_path)) {
7798                 err = -errno;
7799                 goto out_err;
7800         }
7801
7802         map->pinned = true;
7803         pr_debug("pinned map '%s'\n", map->pin_path);
7804
7805         return 0;
7806
7807 out_err:
7808         cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7809         pr_warn("failed to pin map: %s\n", cp);
7810         return err;
7811 }
7812
7813 int bpf_map__unpin(struct bpf_map *map, const char *path)
7814 {
7815         int err;
7816
7817         if (map == NULL) {
7818                 pr_warn("invalid map pointer\n");
7819                 return -EINVAL;
7820         }
7821
7822         if (map->pin_path) {
7823                 if (path && strcmp(path, map->pin_path)) {
7824                         pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7825                                 bpf_map__name(map), map->pin_path, path);
7826                         return -EINVAL;
7827                 }
7828                 path = map->pin_path;
7829         } else if (!path) {
7830                 pr_warn("no path to unpin map '%s' from\n",
7831                         bpf_map__name(map));
7832                 return -EINVAL;
7833         }
7834
7835         err = check_path(path);
7836         if (err)
7837                 return err;
7838
7839         err = unlink(path);
7840         if (err != 0)
7841                 return -errno;
7842
7843         map->pinned = false;
7844         pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7845
7846         return 0;
7847 }
7848
7849 int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7850 {
7851         char *new = NULL;
7852
7853         if (path) {
7854                 new = strdup(path);
7855                 if (!new)
7856                         return -errno;
7857         }
7858
7859         free(map->pin_path);
7860         map->pin_path = new;
7861         return 0;
7862 }
7863
7864 const char *bpf_map__get_pin_path(const struct bpf_map *map)
7865 {
7866         return map->pin_path;
7867 }
7868
7869 bool bpf_map__is_pinned(const struct bpf_map *map)
7870 {
7871         return map->pinned;
7872 }
7873
7874 static void sanitize_pin_path(char *s)
7875 {
7876         /* bpffs disallows periods in path names */
7877         while (*s) {
7878                 if (*s == '.')
7879                         *s = '_';
7880                 s++;
7881         }
7882 }
7883
7884 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
7885 {
7886         struct bpf_map *map;
7887         int err;
7888
7889         if (!obj)
7890                 return -ENOENT;
7891
7892         if (!obj->loaded) {
7893                 pr_warn("object not yet loaded; load it first\n");
7894                 return -ENOENT;
7895         }
7896
7897         bpf_object__for_each_map(map, obj) {
7898                 char *pin_path = NULL;
7899                 char buf[PATH_MAX];
7900
7901                 if (path) {
7902                         int len;
7903
7904                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
7905                                        bpf_map__name(map));
7906                         if (len < 0) {
7907                                 err = -EINVAL;
7908                                 goto err_unpin_maps;
7909                         } else if (len >= PATH_MAX) {
7910                                 err = -ENAMETOOLONG;
7911                                 goto err_unpin_maps;
7912                         }
7913                         sanitize_pin_path(buf);
7914                         pin_path = buf;
7915                 } else if (!map->pin_path) {
7916                         continue;
7917                 }
7918
7919                 err = bpf_map__pin(map, pin_path);
7920                 if (err)
7921                         goto err_unpin_maps;
7922         }
7923
7924         return 0;
7925
7926 err_unpin_maps:
7927         while ((map = bpf_map__prev(map, obj))) {
7928                 if (!map->pin_path)
7929                         continue;
7930
7931                 bpf_map__unpin(map, NULL);
7932         }
7933
7934         return err;
7935 }
7936
7937 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
7938 {
7939         struct bpf_map *map;
7940         int err;
7941
7942         if (!obj)
7943                 return -ENOENT;
7944
7945         bpf_object__for_each_map(map, obj) {
7946                 char *pin_path = NULL;
7947                 char buf[PATH_MAX];
7948
7949                 if (path) {
7950                         int len;
7951
7952                         len = snprintf(buf, PATH_MAX, "%s/%s", path,
7953                                        bpf_map__name(map));
7954                         if (len < 0)
7955                                 return -EINVAL;
7956                         else if (len >= PATH_MAX)
7957                                 return -ENAMETOOLONG;
7958                         sanitize_pin_path(buf);
7959                         pin_path = buf;
7960                 } else if (!map->pin_path) {
7961                         continue;
7962                 }
7963
7964                 err = bpf_map__unpin(map, pin_path);
7965                 if (err)
7966                         return err;
7967         }
7968
7969         return 0;
7970 }
7971
7972 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
7973 {
7974         struct bpf_program *prog;
7975         int err;
7976
7977         if (!obj)
7978                 return -ENOENT;
7979
7980         if (!obj->loaded) {
7981                 pr_warn("object not yet loaded; load it first\n");
7982                 return -ENOENT;
7983         }
7984
7985         bpf_object__for_each_program(prog, obj) {
7986                 char buf[PATH_MAX];
7987                 int len;
7988
7989                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7990                                prog->pin_name);
7991                 if (len < 0) {
7992                         err = -EINVAL;
7993                         goto err_unpin_programs;
7994                 } else if (len >= PATH_MAX) {
7995                         err = -ENAMETOOLONG;
7996                         goto err_unpin_programs;
7997                 }
7998
7999                 err = bpf_program__pin(prog, buf);
8000                 if (err)
8001                         goto err_unpin_programs;
8002         }
8003
8004         return 0;
8005
8006 err_unpin_programs:
8007         while ((prog = bpf_program__prev(prog, obj))) {
8008                 char buf[PATH_MAX];
8009                 int len;
8010
8011                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8012                                prog->pin_name);
8013                 if (len < 0)
8014                         continue;
8015                 else if (len >= PATH_MAX)
8016                         continue;
8017
8018                 bpf_program__unpin(prog, buf);
8019         }
8020
8021         return err;
8022 }
8023
8024 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8025 {
8026         struct bpf_program *prog;
8027         int err;
8028
8029         if (!obj)
8030                 return -ENOENT;
8031
8032         bpf_object__for_each_program(prog, obj) {
8033                 char buf[PATH_MAX];
8034                 int len;
8035
8036                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8037                                prog->pin_name);
8038                 if (len < 0)
8039                         return -EINVAL;
8040                 else if (len >= PATH_MAX)
8041                         return -ENAMETOOLONG;
8042
8043                 err = bpf_program__unpin(prog, buf);
8044                 if (err)
8045                         return err;
8046         }
8047
8048         return 0;
8049 }
8050
8051 int bpf_object__pin(struct bpf_object *obj, const char *path)
8052 {
8053         int err;
8054
8055         err = bpf_object__pin_maps(obj, path);
8056         if (err)
8057                 return err;
8058
8059         err = bpf_object__pin_programs(obj, path);
8060         if (err) {
8061                 bpf_object__unpin_maps(obj, path);
8062                 return err;
8063         }
8064
8065         return 0;
8066 }
8067
8068 static void bpf_map__destroy(struct bpf_map *map)
8069 {
8070         if (map->clear_priv)
8071                 map->clear_priv(map, map->priv);
8072         map->priv = NULL;
8073         map->clear_priv = NULL;
8074
8075         if (map->inner_map) {
8076                 bpf_map__destroy(map->inner_map);
8077                 zfree(&map->inner_map);
8078         }
8079
8080         zfree(&map->init_slots);
8081         map->init_slots_sz = 0;
8082
8083         if (map->mmaped) {
8084                 munmap(map->mmaped, bpf_map_mmap_sz(map));
8085                 map->mmaped = NULL;
8086         }
8087
8088         if (map->st_ops) {
8089                 zfree(&map->st_ops->data);
8090                 zfree(&map->st_ops->progs);
8091                 zfree(&map->st_ops->kern_func_off);
8092                 zfree(&map->st_ops);
8093         }
8094
8095         zfree(&map->name);
8096         zfree(&map->pin_path);
8097
8098         if (map->fd >= 0)
8099                 zclose(map->fd);
8100 }
8101
8102 void bpf_object__close(struct bpf_object *obj)
8103 {
8104         size_t i;
8105
8106         if (IS_ERR_OR_NULL(obj))
8107                 return;
8108
8109         if (obj->clear_priv)
8110                 obj->clear_priv(obj, obj->priv);
8111
8112         bpf_object__elf_finish(obj);
8113         bpf_object__unload(obj);
8114         btf__free(obj->btf);
8115         btf_ext__free(obj->btf_ext);
8116
8117         for (i = 0; i < obj->nr_maps; i++)
8118                 bpf_map__destroy(&obj->maps[i]);
8119
8120         zfree(&obj->kconfig);
8121         zfree(&obj->externs);
8122         obj->nr_extern = 0;
8123
8124         zfree(&obj->maps);
8125         obj->nr_maps = 0;
8126
8127         if (obj->programs && obj->nr_programs) {
8128                 for (i = 0; i < obj->nr_programs; i++)
8129                         bpf_program__exit(&obj->programs[i]);
8130         }
8131         zfree(&obj->programs);
8132
8133         list_del(&obj->list);
8134         free(obj);
8135 }
8136
8137 struct bpf_object *
8138 bpf_object__next(struct bpf_object *prev)
8139 {
8140         struct bpf_object *next;
8141
8142         if (!prev)
8143                 next = list_first_entry(&bpf_objects_list,
8144                                         struct bpf_object,
8145                                         list);
8146         else
8147                 next = list_next_entry(prev, list);
8148
8149         /* Empty list is noticed here so don't need checking on entry. */
8150         if (&next->list == &bpf_objects_list)
8151                 return NULL;
8152
8153         return next;
8154 }
8155
8156 const char *bpf_object__name(const struct bpf_object *obj)
8157 {
8158         return obj ? obj->name : ERR_PTR(-EINVAL);
8159 }
8160
8161 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8162 {
8163         return obj ? obj->kern_version : 0;
8164 }
8165
8166 struct btf *bpf_object__btf(const struct bpf_object *obj)
8167 {
8168         return obj ? obj->btf : NULL;
8169 }
8170
8171 int bpf_object__btf_fd(const struct bpf_object *obj)
8172 {
8173         return obj->btf ? btf__fd(obj->btf) : -1;
8174 }
8175
8176 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
8177                          bpf_object_clear_priv_t clear_priv)
8178 {
8179         if (obj->priv && obj->clear_priv)
8180                 obj->clear_priv(obj, obj->priv);
8181
8182         obj->priv = priv;
8183         obj->clear_priv = clear_priv;
8184         return 0;
8185 }
8186
8187 void *bpf_object__priv(const struct bpf_object *obj)
8188 {
8189         return obj ? obj->priv : ERR_PTR(-EINVAL);
8190 }
8191
8192 static struct bpf_program *
8193 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8194                     bool forward)
8195 {
8196         size_t nr_programs = obj->nr_programs;
8197         ssize_t idx;
8198
8199         if (!nr_programs)
8200                 return NULL;
8201
8202         if (!p)
8203                 /* Iter from the beginning */
8204                 return forward ? &obj->programs[0] :
8205                         &obj->programs[nr_programs - 1];
8206
8207         if (p->obj != obj) {
8208                 pr_warn("error: program handler doesn't match object\n");
8209                 return NULL;
8210         }
8211
8212         idx = (p - obj->programs) + (forward ? 1 : -1);
8213         if (idx >= obj->nr_programs || idx < 0)
8214                 return NULL;
8215         return &obj->programs[idx];
8216 }
8217
8218 struct bpf_program *
8219 bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
8220 {
8221         struct bpf_program *prog = prev;
8222
8223         do {
8224                 prog = __bpf_program__iter(prog, obj, true);
8225         } while (prog && prog_is_subprog(obj, prog));
8226
8227         return prog;
8228 }
8229
8230 struct bpf_program *
8231 bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
8232 {
8233         struct bpf_program *prog = next;
8234
8235         do {
8236                 prog = __bpf_program__iter(prog, obj, false);
8237         } while (prog && prog_is_subprog(obj, prog));
8238
8239         return prog;
8240 }
8241
8242 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
8243                           bpf_program_clear_priv_t clear_priv)
8244 {
8245         if (prog->priv && prog->clear_priv)
8246                 prog->clear_priv(prog, prog->priv);
8247
8248         prog->priv = priv;
8249         prog->clear_priv = clear_priv;
8250         return 0;
8251 }
8252
8253 void *bpf_program__priv(const struct bpf_program *prog)
8254 {
8255         return prog ? prog->priv : ERR_PTR(-EINVAL);
8256 }
8257
8258 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8259 {
8260         prog->prog_ifindex = ifindex;
8261 }
8262
8263 const char *bpf_program__name(const struct bpf_program *prog)
8264 {
8265         return prog->name;
8266 }
8267
8268 const char *bpf_program__section_name(const struct bpf_program *prog)
8269 {
8270         return prog->sec_name;
8271 }
8272
8273 const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
8274 {
8275         const char *title;
8276
8277         title = prog->sec_name;
8278         if (needs_copy) {
8279                 title = strdup(title);
8280                 if (!title) {
8281                         pr_warn("failed to strdup program title\n");
8282                         return ERR_PTR(-ENOMEM);
8283                 }
8284         }
8285
8286         return title;
8287 }
8288
8289 bool bpf_program__autoload(const struct bpf_program *prog)
8290 {
8291         return prog->load;
8292 }
8293
8294 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8295 {
8296         if (prog->obj->loaded)
8297                 return -EINVAL;
8298
8299         prog->load = autoload;
8300         return 0;
8301 }
8302
8303 int bpf_program__fd(const struct bpf_program *prog)
8304 {
8305         return bpf_program__nth_fd(prog, 0);
8306 }
8307
8308 size_t bpf_program__size(const struct bpf_program *prog)
8309 {
8310         return prog->insns_cnt * BPF_INSN_SZ;
8311 }
8312
8313 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
8314                           bpf_program_prep_t prep)
8315 {
8316         int *instances_fds;
8317
8318         if (nr_instances <= 0 || !prep)
8319                 return -EINVAL;
8320
8321         if (prog->instances.nr > 0 || prog->instances.fds) {
8322                 pr_warn("Can't set pre-processor after loading\n");
8323                 return -EINVAL;
8324         }
8325
8326         instances_fds = malloc(sizeof(int) * nr_instances);
8327         if (!instances_fds) {
8328                 pr_warn("alloc memory failed for fds\n");
8329                 return -ENOMEM;
8330         }
8331
8332         /* fill all fd with -1 */
8333         memset(instances_fds, -1, sizeof(int) * nr_instances);
8334
8335         prog->instances.nr = nr_instances;
8336         prog->instances.fds = instances_fds;
8337         prog->preprocessor = prep;
8338         return 0;
8339 }
8340
8341 int bpf_program__nth_fd(const struct bpf_program *prog, int n)
8342 {
8343         int fd;
8344
8345         if (!prog)
8346                 return -EINVAL;
8347
8348         if (n >= prog->instances.nr || n < 0) {
8349                 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
8350                         n, prog->name, prog->instances.nr);
8351                 return -EINVAL;
8352         }
8353
8354         fd = prog->instances.fds[n];
8355         if (fd < 0) {
8356                 pr_warn("%dth instance of program '%s' is invalid\n",
8357                         n, prog->name);
8358                 return -ENOENT;
8359         }
8360
8361         return fd;
8362 }
8363
8364 enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
8365 {
8366         return prog->type;
8367 }
8368
8369 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8370 {
8371         prog->type = type;
8372 }
8373
8374 static bool bpf_program__is_type(const struct bpf_program *prog,
8375                                  enum bpf_prog_type type)
8376 {
8377         return prog ? (prog->type == type) : false;
8378 }
8379
8380 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                           \
8381 int bpf_program__set_##NAME(struct bpf_program *prog)           \
8382 {                                                               \
8383         if (!prog)                                              \
8384                 return -EINVAL;                                 \
8385         bpf_program__set_type(prog, TYPE);                      \
8386         return 0;                                               \
8387 }                                                               \
8388                                                                 \
8389 bool bpf_program__is_##NAME(const struct bpf_program *prog)     \
8390 {                                                               \
8391         return bpf_program__is_type(prog, TYPE);                \
8392 }                                                               \
8393
8394 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
8395 BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
8396 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
8397 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
8398 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
8399 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
8400 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
8401 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
8402 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
8403 BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
8404 BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
8405 BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
8406 BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
8407
8408 enum bpf_attach_type
8409 bpf_program__get_expected_attach_type(struct bpf_program *prog)
8410 {
8411         return prog->expected_attach_type;
8412 }
8413
8414 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
8415                                            enum bpf_attach_type type)
8416 {
8417         prog->expected_attach_type = type;
8418 }
8419
8420 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional,           \
8421                           attachable, attach_btf)                           \
8422         {                                                                   \
8423                 .sec = string,                                              \
8424                 .len = sizeof(string) - 1,                                  \
8425                 .prog_type = ptype,                                         \
8426                 .expected_attach_type = eatype,                             \
8427                 .is_exp_attach_type_optional = eatype_optional,             \
8428                 .is_attachable = attachable,                                \
8429                 .is_attach_btf = attach_btf,                                \
8430         }
8431
8432 /* Programs that can NOT be attached. */
8433 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
8434
8435 /* Programs that can be attached. */
8436 #define BPF_APROG_SEC(string, ptype, atype) \
8437         BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0)
8438
8439 /* Programs that must specify expected attach type at load time. */
8440 #define BPF_EAPROG_SEC(string, ptype, eatype) \
8441         BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0)
8442
8443 /* Programs that use BTF to identify attach point */
8444 #define BPF_PROG_BTF(string, ptype, eatype) \
8445         BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1)
8446
8447 /* Programs that can be attached but attach type can't be identified by section
8448  * name. Kept for backward compatibility.
8449  */
8450 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
8451
8452 #define SEC_DEF(sec_pfx, ptype, ...) {                                      \
8453         .sec = sec_pfx,                                                     \
8454         .len = sizeof(sec_pfx) - 1,                                         \
8455         .prog_type = BPF_PROG_TYPE_##ptype,                                 \
8456         __VA_ARGS__                                                         \
8457 }
8458
8459 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
8460                                       struct bpf_program *prog);
8461 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
8462                                   struct bpf_program *prog);
8463 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
8464                                       struct bpf_program *prog);
8465 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
8466                                      struct bpf_program *prog);
8467 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
8468                                    struct bpf_program *prog);
8469 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
8470                                     struct bpf_program *prog);
8471
8472 static const struct bpf_sec_def section_defs[] = {
8473         BPF_PROG_SEC("socket",                  BPF_PROG_TYPE_SOCKET_FILTER),
8474         BPF_PROG_SEC("sk_reuseport",            BPF_PROG_TYPE_SK_REUSEPORT),
8475         SEC_DEF("kprobe/", KPROBE,
8476                 .attach_fn = attach_kprobe),
8477         BPF_PROG_SEC("uprobe/",                 BPF_PROG_TYPE_KPROBE),
8478         SEC_DEF("kretprobe/", KPROBE,
8479                 .attach_fn = attach_kprobe),
8480         BPF_PROG_SEC("uretprobe/",              BPF_PROG_TYPE_KPROBE),
8481         BPF_PROG_SEC("classifier",              BPF_PROG_TYPE_SCHED_CLS),
8482         BPF_PROG_SEC("action",                  BPF_PROG_TYPE_SCHED_ACT),
8483         SEC_DEF("tracepoint/", TRACEPOINT,
8484                 .attach_fn = attach_tp),
8485         SEC_DEF("tp/", TRACEPOINT,
8486                 .attach_fn = attach_tp),
8487         SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
8488                 .attach_fn = attach_raw_tp),
8489         SEC_DEF("raw_tp/", RAW_TRACEPOINT,
8490                 .attach_fn = attach_raw_tp),
8491         SEC_DEF("tp_btf/", TRACING,
8492                 .expected_attach_type = BPF_TRACE_RAW_TP,
8493                 .is_attach_btf = true,
8494                 .attach_fn = attach_trace),
8495         SEC_DEF("fentry/", TRACING,
8496                 .expected_attach_type = BPF_TRACE_FENTRY,
8497                 .is_attach_btf = true,
8498                 .attach_fn = attach_trace),
8499         SEC_DEF("fmod_ret/", TRACING,
8500                 .expected_attach_type = BPF_MODIFY_RETURN,
8501                 .is_attach_btf = true,
8502                 .attach_fn = attach_trace),
8503         SEC_DEF("fexit/", TRACING,
8504                 .expected_attach_type = BPF_TRACE_FEXIT,
8505                 .is_attach_btf = true,
8506                 .attach_fn = attach_trace),
8507         SEC_DEF("fentry.s/", TRACING,
8508                 .expected_attach_type = BPF_TRACE_FENTRY,
8509                 .is_attach_btf = true,
8510                 .is_sleepable = true,
8511                 .attach_fn = attach_trace),
8512         SEC_DEF("fmod_ret.s/", TRACING,
8513                 .expected_attach_type = BPF_MODIFY_RETURN,
8514                 .is_attach_btf = true,
8515                 .is_sleepable = true,
8516                 .attach_fn = attach_trace),
8517         SEC_DEF("fexit.s/", TRACING,
8518                 .expected_attach_type = BPF_TRACE_FEXIT,
8519                 .is_attach_btf = true,
8520                 .is_sleepable = true,
8521                 .attach_fn = attach_trace),
8522         SEC_DEF("freplace/", EXT,
8523                 .is_attach_btf = true,
8524                 .attach_fn = attach_trace),
8525         SEC_DEF("lsm/", LSM,
8526                 .is_attach_btf = true,
8527                 .expected_attach_type = BPF_LSM_MAC,
8528                 .attach_fn = attach_lsm),
8529         SEC_DEF("lsm.s/", LSM,
8530                 .is_attach_btf = true,
8531                 .is_sleepable = true,
8532                 .expected_attach_type = BPF_LSM_MAC,
8533                 .attach_fn = attach_lsm),
8534         SEC_DEF("iter/", TRACING,
8535                 .expected_attach_type = BPF_TRACE_ITER,
8536                 .is_attach_btf = true,
8537                 .attach_fn = attach_iter),
8538         BPF_EAPROG_SEC("xdp_devmap/",           BPF_PROG_TYPE_XDP,
8539                                                 BPF_XDP_DEVMAP),
8540         BPF_EAPROG_SEC("xdp_cpumap/",           BPF_PROG_TYPE_XDP,
8541                                                 BPF_XDP_CPUMAP),
8542         BPF_APROG_SEC("xdp",                    BPF_PROG_TYPE_XDP,
8543                                                 BPF_XDP),
8544         BPF_PROG_SEC("perf_event",              BPF_PROG_TYPE_PERF_EVENT),
8545         BPF_PROG_SEC("lwt_in",                  BPF_PROG_TYPE_LWT_IN),
8546         BPF_PROG_SEC("lwt_out",                 BPF_PROG_TYPE_LWT_OUT),
8547         BPF_PROG_SEC("lwt_xmit",                BPF_PROG_TYPE_LWT_XMIT),
8548         BPF_PROG_SEC("lwt_seg6local",           BPF_PROG_TYPE_LWT_SEG6LOCAL),
8549         BPF_APROG_SEC("cgroup_skb/ingress",     BPF_PROG_TYPE_CGROUP_SKB,
8550                                                 BPF_CGROUP_INET_INGRESS),
8551         BPF_APROG_SEC("cgroup_skb/egress",      BPF_PROG_TYPE_CGROUP_SKB,
8552                                                 BPF_CGROUP_INET_EGRESS),
8553         BPF_APROG_COMPAT("cgroup/skb",          BPF_PROG_TYPE_CGROUP_SKB),
8554         BPF_EAPROG_SEC("cgroup/sock_create",    BPF_PROG_TYPE_CGROUP_SOCK,
8555                                                 BPF_CGROUP_INET_SOCK_CREATE),
8556         BPF_EAPROG_SEC("cgroup/sock_release",   BPF_PROG_TYPE_CGROUP_SOCK,
8557                                                 BPF_CGROUP_INET_SOCK_RELEASE),
8558         BPF_APROG_SEC("cgroup/sock",            BPF_PROG_TYPE_CGROUP_SOCK,
8559                                                 BPF_CGROUP_INET_SOCK_CREATE),
8560         BPF_EAPROG_SEC("cgroup/post_bind4",     BPF_PROG_TYPE_CGROUP_SOCK,
8561                                                 BPF_CGROUP_INET4_POST_BIND),
8562         BPF_EAPROG_SEC("cgroup/post_bind6",     BPF_PROG_TYPE_CGROUP_SOCK,
8563                                                 BPF_CGROUP_INET6_POST_BIND),
8564         BPF_APROG_SEC("cgroup/dev",             BPF_PROG_TYPE_CGROUP_DEVICE,
8565                                                 BPF_CGROUP_DEVICE),
8566         BPF_APROG_SEC("sockops",                BPF_PROG_TYPE_SOCK_OPS,
8567                                                 BPF_CGROUP_SOCK_OPS),
8568         BPF_APROG_SEC("sk_skb/stream_parser",   BPF_PROG_TYPE_SK_SKB,
8569                                                 BPF_SK_SKB_STREAM_PARSER),
8570         BPF_APROG_SEC("sk_skb/stream_verdict",  BPF_PROG_TYPE_SK_SKB,
8571                                                 BPF_SK_SKB_STREAM_VERDICT),
8572         BPF_APROG_COMPAT("sk_skb",              BPF_PROG_TYPE_SK_SKB),
8573         BPF_APROG_SEC("sk_msg",                 BPF_PROG_TYPE_SK_MSG,
8574                                                 BPF_SK_MSG_VERDICT),
8575         BPF_APROG_SEC("lirc_mode2",             BPF_PROG_TYPE_LIRC_MODE2,
8576                                                 BPF_LIRC_MODE2),
8577         BPF_APROG_SEC("flow_dissector",         BPF_PROG_TYPE_FLOW_DISSECTOR,
8578                                                 BPF_FLOW_DISSECTOR),
8579         BPF_EAPROG_SEC("cgroup/bind4",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8580                                                 BPF_CGROUP_INET4_BIND),
8581         BPF_EAPROG_SEC("cgroup/bind6",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8582                                                 BPF_CGROUP_INET6_BIND),
8583         BPF_EAPROG_SEC("cgroup/connect4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8584                                                 BPF_CGROUP_INET4_CONNECT),
8585         BPF_EAPROG_SEC("cgroup/connect6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8586                                                 BPF_CGROUP_INET6_CONNECT),
8587         BPF_EAPROG_SEC("cgroup/sendmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8588                                                 BPF_CGROUP_UDP4_SENDMSG),
8589         BPF_EAPROG_SEC("cgroup/sendmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8590                                                 BPF_CGROUP_UDP6_SENDMSG),
8591         BPF_EAPROG_SEC("cgroup/recvmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8592                                                 BPF_CGROUP_UDP4_RECVMSG),
8593         BPF_EAPROG_SEC("cgroup/recvmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8594                                                 BPF_CGROUP_UDP6_RECVMSG),
8595         BPF_EAPROG_SEC("cgroup/getpeername4",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8596                                                 BPF_CGROUP_INET4_GETPEERNAME),
8597         BPF_EAPROG_SEC("cgroup/getpeername6",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8598                                                 BPF_CGROUP_INET6_GETPEERNAME),
8599         BPF_EAPROG_SEC("cgroup/getsockname4",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8600                                                 BPF_CGROUP_INET4_GETSOCKNAME),
8601         BPF_EAPROG_SEC("cgroup/getsockname6",   BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8602                                                 BPF_CGROUP_INET6_GETSOCKNAME),
8603         BPF_EAPROG_SEC("cgroup/sysctl",         BPF_PROG_TYPE_CGROUP_SYSCTL,
8604                                                 BPF_CGROUP_SYSCTL),
8605         BPF_EAPROG_SEC("cgroup/getsockopt",     BPF_PROG_TYPE_CGROUP_SOCKOPT,
8606                                                 BPF_CGROUP_GETSOCKOPT),
8607         BPF_EAPROG_SEC("cgroup/setsockopt",     BPF_PROG_TYPE_CGROUP_SOCKOPT,
8608                                                 BPF_CGROUP_SETSOCKOPT),
8609         BPF_PROG_SEC("struct_ops",              BPF_PROG_TYPE_STRUCT_OPS),
8610         BPF_EAPROG_SEC("sk_lookup/",            BPF_PROG_TYPE_SK_LOOKUP,
8611                                                 BPF_SK_LOOKUP),
8612 };
8613
8614 #undef BPF_PROG_SEC_IMPL
8615 #undef BPF_PROG_SEC
8616 #undef BPF_APROG_SEC
8617 #undef BPF_EAPROG_SEC
8618 #undef BPF_APROG_COMPAT
8619 #undef SEC_DEF
8620
8621 #define MAX_TYPE_NAME_SIZE 32
8622
8623 static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8624 {
8625         int i, n = ARRAY_SIZE(section_defs);
8626
8627         for (i = 0; i < n; i++) {
8628                 if (strncmp(sec_name,
8629                             section_defs[i].sec, section_defs[i].len))
8630                         continue;
8631                 return &section_defs[i];
8632         }
8633         return NULL;
8634 }
8635
8636 static char *libbpf_get_type_names(bool attach_type)
8637 {
8638         int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8639         char *buf;
8640
8641         buf = malloc(len);
8642         if (!buf)
8643                 return NULL;
8644
8645         buf[0] = '\0';
8646         /* Forge string buf with all available names */
8647         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8648                 if (attach_type && !section_defs[i].is_attachable)
8649                         continue;
8650
8651                 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8652                         free(buf);
8653                         return NULL;
8654                 }
8655                 strcat(buf, " ");
8656                 strcat(buf, section_defs[i].sec);
8657         }
8658
8659         return buf;
8660 }
8661
8662 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8663                              enum bpf_attach_type *expected_attach_type)
8664 {
8665         const struct bpf_sec_def *sec_def;
8666         char *type_names;
8667
8668         if (!name)
8669                 return -EINVAL;
8670
8671         sec_def = find_sec_def(name);
8672         if (sec_def) {
8673                 *prog_type = sec_def->prog_type;
8674                 *expected_attach_type = sec_def->expected_attach_type;
8675                 return 0;
8676         }
8677
8678         pr_debug("failed to guess program type from ELF section '%s'\n", name);
8679         type_names = libbpf_get_type_names(false);
8680         if (type_names != NULL) {
8681                 pr_debug("supported section(type) names are:%s\n", type_names);
8682                 free(type_names);
8683         }
8684
8685         return -ESRCH;
8686 }
8687
8688 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8689                                                      size_t offset)
8690 {
8691         struct bpf_map *map;
8692         size_t i;
8693
8694         for (i = 0; i < obj->nr_maps; i++) {
8695                 map = &obj->maps[i];
8696                 if (!bpf_map__is_struct_ops(map))
8697                         continue;
8698                 if (map->sec_offset <= offset &&
8699                     offset - map->sec_offset < map->def.value_size)
8700                         return map;
8701         }
8702
8703         return NULL;
8704 }
8705
8706 /* Collect the reloc from ELF and populate the st_ops->progs[] */
8707 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8708                                             GElf_Shdr *shdr, Elf_Data *data)
8709 {
8710         const struct btf_member *member;
8711         struct bpf_struct_ops *st_ops;
8712         struct bpf_program *prog;
8713         unsigned int shdr_idx;
8714         const struct btf *btf;
8715         struct bpf_map *map;
8716         Elf_Data *symbols;
8717         unsigned int moff, insn_idx;
8718         const char *name;
8719         __u32 member_idx;
8720         GElf_Sym sym;
8721         GElf_Rel rel;
8722         int i, nrels;
8723
8724         symbols = obj->efile.symbols;
8725         btf = obj->btf;
8726         nrels = shdr->sh_size / shdr->sh_entsize;
8727         for (i = 0; i < nrels; i++) {
8728                 if (!gelf_getrel(data, i, &rel)) {
8729                         pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8730                         return -LIBBPF_ERRNO__FORMAT;
8731                 }
8732
8733                 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
8734                         pr_warn("struct_ops reloc: symbol %zx not found\n",
8735                                 (size_t)GELF_R_SYM(rel.r_info));
8736                         return -LIBBPF_ERRNO__FORMAT;
8737                 }
8738
8739                 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
8740                 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
8741                 if (!map) {
8742                         pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
8743                                 (size_t)rel.r_offset);
8744                         return -EINVAL;
8745                 }
8746
8747                 moff = rel.r_offset - map->sec_offset;
8748                 shdr_idx = sym.st_shndx;
8749                 st_ops = map->st_ops;
8750                 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8751                          map->name,
8752                          (long long)(rel.r_info >> 32),
8753                          (long long)sym.st_value,
8754                          shdr_idx, (size_t)rel.r_offset,
8755                          map->sec_offset, sym.st_name, name);
8756
8757                 if (shdr_idx >= SHN_LORESERVE) {
8758                         pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
8759                                 map->name, (size_t)rel.r_offset, shdr_idx);
8760                         return -LIBBPF_ERRNO__RELOC;
8761                 }
8762                 if (sym.st_value % BPF_INSN_SZ) {
8763                         pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8764                                 map->name, (unsigned long long)sym.st_value);
8765                         return -LIBBPF_ERRNO__FORMAT;
8766                 }
8767                 insn_idx = sym.st_value / BPF_INSN_SZ;
8768
8769                 member = find_member_by_offset(st_ops->type, moff * 8);
8770                 if (!member) {
8771                         pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8772                                 map->name, moff);
8773                         return -EINVAL;
8774                 }
8775                 member_idx = member - btf_members(st_ops->type);
8776                 name = btf__name_by_offset(btf, member->name_off);
8777
8778                 if (!resolve_func_ptr(btf, member->type, NULL)) {
8779                         pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8780                                 map->name, name);
8781                         return -EINVAL;
8782                 }
8783
8784                 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8785                 if (!prog) {
8786                         pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8787                                 map->name, shdr_idx, name);
8788                         return -EINVAL;
8789                 }
8790
8791                 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
8792                         const struct bpf_sec_def *sec_def;
8793
8794                         sec_def = find_sec_def(prog->sec_name);
8795                         if (sec_def &&
8796                             sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
8797                                 /* for pr_warn */
8798                                 prog->type = sec_def->prog_type;
8799                                 goto invalid_prog;
8800                         }
8801
8802                         prog->type = BPF_PROG_TYPE_STRUCT_OPS;
8803                         prog->attach_btf_id = st_ops->type_id;
8804                         prog->expected_attach_type = member_idx;
8805                 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
8806                            prog->attach_btf_id != st_ops->type_id ||
8807                            prog->expected_attach_type != member_idx) {
8808                         goto invalid_prog;
8809                 }
8810                 st_ops->progs[member_idx] = prog;
8811         }
8812
8813         return 0;
8814
8815 invalid_prog:
8816         pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8817                 map->name, prog->name, prog->sec_name, prog->type,
8818                 prog->attach_btf_id, prog->expected_attach_type, name);
8819         return -EINVAL;
8820 }
8821
8822 #define BTF_TRACE_PREFIX "btf_trace_"
8823 #define BTF_LSM_PREFIX "bpf_lsm_"
8824 #define BTF_ITER_PREFIX "bpf_iter_"
8825 #define BTF_MAX_NAME_SIZE 128
8826
8827 static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
8828                                    const char *name, __u32 kind)
8829 {
8830         char btf_type_name[BTF_MAX_NAME_SIZE];
8831         int ret;
8832
8833         ret = snprintf(btf_type_name, sizeof(btf_type_name),
8834                        "%s%s", prefix, name);
8835         /* snprintf returns the number of characters written excluding the
8836          * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it
8837          * indicates truncation.
8838          */
8839         if (ret < 0 || ret >= sizeof(btf_type_name))
8840                 return -ENAMETOOLONG;
8841         return btf__find_by_name_kind(btf, btf_type_name, kind);
8842 }
8843
8844 static inline int find_attach_btf_id(struct btf *btf, const char *name,
8845                                      enum bpf_attach_type attach_type)
8846 {
8847         int err;
8848
8849         if (attach_type == BPF_TRACE_RAW_TP)
8850                 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name,
8851                                               BTF_KIND_TYPEDEF);
8852         else if (attach_type == BPF_LSM_MAC)
8853                 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name,
8854                                               BTF_KIND_FUNC);
8855         else if (attach_type == BPF_TRACE_ITER)
8856                 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name,
8857                                               BTF_KIND_FUNC);
8858         else
8859                 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8860
8861         return err;
8862 }
8863
8864 int libbpf_find_vmlinux_btf_id(const char *name,
8865                                enum bpf_attach_type attach_type)
8866 {
8867         struct btf *btf;
8868         int err;
8869
8870         btf = libbpf_find_kernel_btf();
8871         if (IS_ERR(btf)) {
8872                 pr_warn("vmlinux BTF is not found\n");
8873                 return -EINVAL;
8874         }
8875
8876         err = find_attach_btf_id(btf, name, attach_type);
8877         if (err <= 0)
8878                 pr_warn("%s is not found in vmlinux BTF\n", name);
8879
8880         btf__free(btf);
8881         return err;
8882 }
8883
8884 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
8885 {
8886         struct bpf_prog_info_linear *info_linear;
8887         struct bpf_prog_info *info;
8888         struct btf *btf = NULL;
8889         int err = -EINVAL;
8890
8891         info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
8892         if (IS_ERR_OR_NULL(info_linear)) {
8893                 pr_warn("failed get_prog_info_linear for FD %d\n",
8894                         attach_prog_fd);
8895                 return -EINVAL;
8896         }
8897         info = &info_linear->info;
8898         if (!info->btf_id) {
8899                 pr_warn("The target program doesn't have BTF\n");
8900                 goto out;
8901         }
8902         if (btf__get_from_id(info->btf_id, &btf)) {
8903                 pr_warn("Failed to get BTF of the program\n");
8904                 goto out;
8905         }
8906         err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8907         btf__free(btf);
8908         if (err <= 0) {
8909                 pr_warn("%s is not found in prog's BTF\n", name);
8910                 goto out;
8911         }
8912 out:
8913         free(info_linear);
8914         return err;
8915 }
8916
8917 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
8918                               enum bpf_attach_type attach_type,
8919                               int *btf_obj_fd, int *btf_type_id)
8920 {
8921         int ret, i;
8922
8923         ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
8924         if (ret > 0) {
8925                 *btf_obj_fd = 0; /* vmlinux BTF */
8926                 *btf_type_id = ret;
8927                 return 0;
8928         }
8929         if (ret != -ENOENT)
8930                 return ret;
8931
8932         ret = load_module_btfs(obj);
8933         if (ret)
8934                 return ret;
8935
8936         for (i = 0; i < obj->btf_module_cnt; i++) {
8937                 const struct module_btf *mod = &obj->btf_modules[i];
8938
8939                 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
8940                 if (ret > 0) {
8941                         *btf_obj_fd = mod->fd;
8942                         *btf_type_id = ret;
8943                         return 0;
8944                 }
8945                 if (ret == -ENOENT)
8946                         continue;
8947
8948                 return ret;
8949         }
8950
8951         return -ESRCH;
8952 }
8953
8954 static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id)
8955 {
8956         enum bpf_attach_type attach_type = prog->expected_attach_type;
8957         __u32 attach_prog_fd = prog->attach_prog_fd;
8958         const char *name = prog->sec_name, *attach_name;
8959         const struct bpf_sec_def *sec = NULL;
8960         int i, err;
8961
8962         if (!name)
8963                 return -EINVAL;
8964
8965         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8966                 if (!section_defs[i].is_attach_btf)
8967                         continue;
8968                 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
8969                         continue;
8970
8971                 sec = &section_defs[i];
8972                 break;
8973         }
8974
8975         if (!sec) {
8976                 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);
8977                 return -ESRCH;
8978         }
8979         attach_name = name + sec->len;
8980
8981         /* BPF program's BTF ID */
8982         if (attach_prog_fd) {
8983                 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
8984                 if (err < 0) {
8985                         pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
8986                                  attach_prog_fd, attach_name, err);
8987                         return err;
8988                 }
8989                 *btf_obj_fd = 0;
8990                 *btf_type_id = err;
8991                 return 0;
8992         }
8993
8994         /* kernel/module BTF ID */
8995         err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
8996         if (err) {
8997                 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
8998                 return err;
8999         }
9000         return 0;
9001 }
9002
9003 int libbpf_attach_type_by_name(const char *name,
9004                                enum bpf_attach_type *attach_type)
9005 {
9006         char *type_names;
9007         int i;
9008
9009         if (!name)
9010                 return -EINVAL;
9011
9012         for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
9013                 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
9014                         continue;
9015                 if (!section_defs[i].is_attachable)
9016                         return -EINVAL;
9017                 *attach_type = section_defs[i].expected_attach_type;
9018                 return 0;
9019         }
9020         pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9021         type_names = libbpf_get_type_names(true);
9022         if (type_names != NULL) {
9023                 pr_debug("attachable section(type) names are:%s\n", type_names);
9024                 free(type_names);
9025         }
9026
9027         return -EINVAL;
9028 }
9029
9030 int bpf_map__fd(const struct bpf_map *map)
9031 {
9032         return map ? map->fd : -EINVAL;
9033 }
9034
9035 const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
9036 {
9037         return map ? &map->def : ERR_PTR(-EINVAL);
9038 }
9039
9040 const char *bpf_map__name(const struct bpf_map *map)
9041 {
9042         return map ? map->name : NULL;
9043 }
9044
9045 enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9046 {
9047         return map->def.type;
9048 }
9049
9050 int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9051 {
9052         if (map->fd >= 0)
9053                 return -EBUSY;
9054         map->def.type = type;
9055         return 0;
9056 }
9057
9058 __u32 bpf_map__map_flags(const struct bpf_map *map)
9059 {
9060         return map->def.map_flags;
9061 }
9062
9063 int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9064 {
9065         if (map->fd >= 0)
9066                 return -EBUSY;
9067         map->def.map_flags = flags;
9068         return 0;
9069 }
9070
9071 __u32 bpf_map__numa_node(const struct bpf_map *map)
9072 {
9073         return map->numa_node;
9074 }
9075
9076 int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9077 {
9078         if (map->fd >= 0)
9079                 return -EBUSY;
9080         map->numa_node = numa_node;
9081         return 0;
9082 }
9083
9084 __u32 bpf_map__key_size(const struct bpf_map *map)
9085 {
9086         return map->def.key_size;
9087 }
9088
9089 int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9090 {
9091         if (map->fd >= 0)
9092                 return -EBUSY;
9093         map->def.key_size = size;
9094         return 0;
9095 }
9096
9097 __u32 bpf_map__value_size(const struct bpf_map *map)
9098 {
9099         return map->def.value_size;
9100 }
9101
9102 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9103 {
9104         if (map->fd >= 0)
9105                 return -EBUSY;
9106         map->def.value_size = size;
9107         return 0;
9108 }
9109
9110 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9111 {
9112         return map ? map->btf_key_type_id : 0;
9113 }
9114
9115 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9116 {
9117         return map ? map->btf_value_type_id : 0;
9118 }
9119
9120 int bpf_map__set_priv(struct bpf_map *map, void *priv,
9121                      bpf_map_clear_priv_t clear_priv)
9122 {
9123         if (!map)
9124                 return -EINVAL;
9125
9126         if (map->priv) {
9127                 if (map->clear_priv)
9128                         map->clear_priv(map, map->priv);
9129         }
9130
9131         map->priv = priv;
9132         map->clear_priv = clear_priv;
9133         return 0;
9134 }
9135
9136 void *bpf_map__priv(const struct bpf_map *map)
9137 {
9138         return map ? map->priv : ERR_PTR(-EINVAL);
9139 }
9140
9141 int bpf_map__set_initial_value(struct bpf_map *map,
9142                                const void *data, size_t size)
9143 {
9144         if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9145             size != map->def.value_size || map->fd >= 0)
9146                 return -EINVAL;
9147
9148         memcpy(map->mmaped, data, size);
9149         return 0;
9150 }
9151
9152 bool bpf_map__is_offload_neutral(const struct bpf_map *map)
9153 {
9154         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
9155 }
9156
9157 bool bpf_map__is_internal(const struct bpf_map *map)
9158 {
9159         return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9160 }
9161
9162 __u32 bpf_map__ifindex(const struct bpf_map *map)
9163 {
9164         return map->map_ifindex;
9165 }
9166
9167 int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9168 {
9169         if (map->fd >= 0)
9170                 return -EBUSY;
9171         map->map_ifindex = ifindex;
9172         return 0;
9173 }
9174
9175 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9176 {
9177         if (!bpf_map_type__is_map_in_map(map->def.type)) {
9178                 pr_warn("error: unsupported map type\n");
9179                 return -EINVAL;
9180         }
9181         if (map->inner_map_fd != -1) {
9182                 pr_warn("error: inner_map_fd already specified\n");
9183                 return -EINVAL;
9184         }
9185         map->inner_map_fd = fd;
9186         return 0;
9187 }
9188
9189 static struct bpf_map *
9190 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9191 {
9192         ssize_t idx;
9193         struct bpf_map *s, *e;
9194
9195         if (!obj || !obj->maps)
9196                 return NULL;
9197
9198         s = obj->maps;
9199         e = obj->maps + obj->nr_maps;
9200
9201         if ((m < s) || (m >= e)) {
9202                 pr_warn("error in %s: map handler doesn't belong to object\n",
9203                          __func__);
9204                 return NULL;
9205         }
9206
9207         idx = (m - obj->maps) + i;
9208         if (idx >= obj->nr_maps || idx < 0)
9209                 return NULL;
9210         return &obj->maps[idx];
9211 }
9212
9213 struct bpf_map *
9214 bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
9215 {
9216         if (prev == NULL)
9217                 return obj->maps;
9218
9219         return __bpf_map__iter(prev, obj, 1);
9220 }
9221
9222 struct bpf_map *
9223 bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
9224 {
9225         if (next == NULL) {
9226                 if (!obj->nr_maps)
9227                         return NULL;
9228                 return obj->maps + obj->nr_maps - 1;
9229         }
9230
9231         return __bpf_map__iter(next, obj, -1);
9232 }
9233
9234 struct bpf_map *
9235 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9236 {
9237         struct bpf_map *pos;
9238
9239         bpf_object__for_each_map(pos, obj) {
9240                 if (pos->name && !strcmp(pos->name, name))
9241                         return pos;
9242         }
9243         return NULL;
9244 }
9245
9246 int
9247 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9248 {
9249         return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9250 }
9251
9252 struct bpf_map *
9253 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
9254 {
9255         return ERR_PTR(-ENOTSUP);
9256 }
9257
9258 long libbpf_get_error(const void *ptr)
9259 {
9260         return PTR_ERR_OR_ZERO(ptr);
9261 }
9262
9263 int bpf_prog_load(const char *file, enum bpf_prog_type type,
9264                   struct bpf_object **pobj, int *prog_fd)
9265 {
9266         struct bpf_prog_load_attr attr;
9267
9268         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
9269         attr.file = file;
9270         attr.prog_type = type;
9271         attr.expected_attach_type = 0;
9272
9273         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
9274 }
9275
9276 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
9277                         struct bpf_object **pobj, int *prog_fd)
9278 {
9279         struct bpf_object_open_attr open_attr = {};
9280         struct bpf_program *prog, *first_prog = NULL;
9281         struct bpf_object *obj;
9282         struct bpf_map *map;
9283         int err;
9284
9285         if (!attr)
9286                 return -EINVAL;
9287         if (!attr->file)
9288                 return -EINVAL;
9289
9290         open_attr.file = attr->file;
9291         open_attr.prog_type = attr->prog_type;
9292
9293         obj = bpf_object__open_xattr(&open_attr);
9294         if (IS_ERR_OR_NULL(obj))
9295                 return -ENOENT;
9296
9297         bpf_object__for_each_program(prog, obj) {
9298                 enum bpf_attach_type attach_type = attr->expected_attach_type;
9299                 /*
9300                  * to preserve backwards compatibility, bpf_prog_load treats
9301                  * attr->prog_type, if specified, as an override to whatever
9302                  * bpf_object__open guessed
9303                  */
9304                 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
9305                         bpf_program__set_type(prog, attr->prog_type);
9306                         bpf_program__set_expected_attach_type(prog,
9307                                                               attach_type);
9308                 }
9309                 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
9310                         /*
9311                          * we haven't guessed from section name and user
9312                          * didn't provide a fallback type, too bad...
9313                          */
9314                         bpf_object__close(obj);
9315                         return -EINVAL;
9316                 }
9317
9318                 prog->prog_ifindex = attr->ifindex;
9319                 prog->log_level = attr->log_level;
9320                 prog->prog_flags |= attr->prog_flags;
9321                 if (!first_prog)
9322                         first_prog = prog;
9323         }
9324
9325         bpf_object__for_each_map(map, obj) {
9326                 if (!bpf_map__is_offload_neutral(map))
9327                         map->map_ifindex = attr->ifindex;
9328         }
9329
9330         if (!first_prog) {
9331                 pr_warn("object file doesn't contain bpf program\n");
9332                 bpf_object__close(obj);
9333                 return -ENOENT;
9334         }
9335
9336         err = bpf_object__load(obj);
9337         if (err) {
9338                 bpf_object__close(obj);
9339                 return err;
9340         }
9341
9342         *pobj = obj;
9343         *prog_fd = bpf_program__fd(first_prog);
9344         return 0;
9345 }
9346
9347 struct bpf_link {
9348         int (*detach)(struct bpf_link *link);
9349         int (*destroy)(struct bpf_link *link);
9350         char *pin_path;         /* NULL, if not pinned */
9351         int fd;                 /* hook FD, -1 if not applicable */
9352         bool disconnected;
9353 };
9354
9355 /* Replace link's underlying BPF program with the new one */
9356 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9357 {
9358         return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9359 }
9360
9361 /* Release "ownership" of underlying BPF resource (typically, BPF program
9362  * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected
9363  * link, when destructed through bpf_link__destroy() call won't attempt to
9364  * detach/unregisted that BPF resource. This is useful in situations where,
9365  * say, attached BPF program has to outlive userspace program that attached it
9366  * in the system. Depending on type of BPF program, though, there might be
9367  * additional steps (like pinning BPF program in BPF FS) necessary to ensure
9368  * exit of userspace program doesn't trigger automatic detachment and clean up
9369  * inside the kernel.
9370  */
9371 void bpf_link__disconnect(struct bpf_link *link)
9372 {
9373         link->disconnected = true;
9374 }
9375
9376 int bpf_link__destroy(struct bpf_link *link)
9377 {
9378         int err = 0;
9379
9380         if (IS_ERR_OR_NULL(link))
9381                 return 0;
9382
9383         if (!link->disconnected && link->detach)
9384                 err = link->detach(link);
9385         if (link->destroy)
9386                 link->destroy(link);
9387         if (link->pin_path)
9388                 free(link->pin_path);
9389         free(link);
9390
9391         return err;
9392 }
9393
9394 int bpf_link__fd(const struct bpf_link *link)
9395 {
9396         return link->fd;
9397 }
9398
9399 const char *bpf_link__pin_path(const struct bpf_link *link)
9400 {
9401         return link->pin_path;
9402 }
9403
9404 static int bpf_link__detach_fd(struct bpf_link *link)
9405 {
9406         return close(link->fd);
9407 }
9408
9409 struct bpf_link *bpf_link__open(const char *path)
9410 {
9411         struct bpf_link *link;
9412         int fd;
9413
9414         fd = bpf_obj_get(path);
9415         if (fd < 0) {
9416                 fd = -errno;
9417                 pr_warn("failed to open link at %s: %d\n", path, fd);
9418                 return ERR_PTR(fd);
9419         }
9420
9421         link = calloc(1, sizeof(*link));
9422         if (!link) {
9423                 close(fd);
9424                 return ERR_PTR(-ENOMEM);
9425         }
9426         link->detach = &bpf_link__detach_fd;
9427         link->fd = fd;
9428
9429         link->pin_path = strdup(path);
9430         if (!link->pin_path) {
9431                 bpf_link__destroy(link);
9432                 return ERR_PTR(-ENOMEM);
9433         }
9434
9435         return link;
9436 }
9437
9438 int bpf_link__detach(struct bpf_link *link)
9439 {
9440         return bpf_link_detach(link->fd) ? -errno : 0;
9441 }
9442
9443 int bpf_link__pin(struct bpf_link *link, const char *path)
9444 {
9445         int err;
9446
9447         if (link->pin_path)
9448                 return -EBUSY;
9449         err = make_parent_dir(path);
9450         if (err)
9451                 return err;
9452         err = check_path(path);
9453         if (err)
9454                 return err;
9455
9456         link->pin_path = strdup(path);
9457         if (!link->pin_path)
9458                 return -ENOMEM;
9459
9460         if (bpf_obj_pin(link->fd, link->pin_path)) {
9461                 err = -errno;
9462                 zfree(&link->pin_path);
9463                 return err;
9464         }
9465
9466         pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9467         return 0;
9468 }
9469
9470 int bpf_link__unpin(struct bpf_link *link)
9471 {
9472         int err;
9473
9474         if (!link->pin_path)
9475                 return -EINVAL;
9476
9477         err = unlink(link->pin_path);
9478         if (err != 0)
9479                 return -errno;
9480
9481         pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9482         zfree(&link->pin_path);
9483         return 0;
9484 }
9485
9486 static int bpf_link__detach_perf_event(struct bpf_link *link)
9487 {
9488         int err;
9489
9490         err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0);
9491         if (err)
9492                 err = -errno;
9493
9494         close(link->fd);
9495         return err;
9496 }
9497
9498 struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
9499                                                 int pfd)
9500 {
9501         char errmsg[STRERR_BUFSIZE];
9502         struct bpf_link *link;
9503         int prog_fd, err;
9504
9505         if (pfd < 0) {
9506                 pr_warn("prog '%s': invalid perf event FD %d\n",
9507                         prog->name, pfd);
9508                 return ERR_PTR(-EINVAL);
9509         }
9510         prog_fd = bpf_program__fd(prog);
9511         if (prog_fd < 0) {
9512                 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9513                         prog->name);
9514                 return ERR_PTR(-EINVAL);
9515         }
9516
9517         link = calloc(1, sizeof(*link));
9518         if (!link)
9519                 return ERR_PTR(-ENOMEM);
9520         link->detach = &bpf_link__detach_perf_event;
9521         link->fd = pfd;
9522
9523         if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9524                 err = -errno;
9525                 free(link);
9526                 pr_warn("prog '%s': failed to attach to pfd %d: %s\n",
9527                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9528                 if (err == -EPROTO)
9529                         pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9530                                 prog->name, pfd);
9531                 return ERR_PTR(err);
9532         }
9533         if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9534                 err = -errno;
9535                 free(link);
9536                 pr_warn("prog '%s': failed to enable pfd %d: %s\n",
9537                         prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9538                 return ERR_PTR(err);
9539         }
9540         return link;
9541 }
9542
9543 /*
9544  * this function is expected to parse integer in the range of [0, 2^31-1] from
9545  * given file using scanf format string fmt. If actual parsed value is
9546  * negative, the result might be indistinguishable from error
9547  */
9548 static int parse_uint_from_file(const char *file, const char *fmt)
9549 {
9550         char buf[STRERR_BUFSIZE];
9551         int err, ret;
9552         FILE *f;
9553
9554         f = fopen(file, "r");
9555         if (!f) {
9556                 err = -errno;
9557                 pr_debug("failed to open '%s': %s\n", file,
9558                          libbpf_strerror_r(err, buf, sizeof(buf)));
9559                 return err;
9560         }
9561         err = fscanf(f, fmt, &ret);
9562         if (err != 1) {
9563                 err = err == EOF ? -EIO : -errno;
9564                 pr_debug("failed to parse '%s': %s\n", file,
9565                         libbpf_strerror_r(err, buf, sizeof(buf)));
9566                 fclose(f);
9567                 return err;
9568         }
9569         fclose(f);
9570         return ret;
9571 }
9572
9573 static int determine_kprobe_perf_type(void)
9574 {
9575         const char *file = "/sys/bus/event_source/devices/kprobe/type";
9576
9577         return parse_uint_from_file(file, "%d\n");
9578 }
9579
9580 static int determine_uprobe_perf_type(void)
9581 {
9582         const char *file = "/sys/bus/event_source/devices/uprobe/type";
9583
9584         return parse_uint_from_file(file, "%d\n");
9585 }
9586
9587 static int determine_kprobe_retprobe_bit(void)
9588 {
9589         const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9590
9591         return parse_uint_from_file(file, "config:%d\n");
9592 }
9593
9594 static int determine_uprobe_retprobe_bit(void)
9595 {
9596         const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9597
9598         return parse_uint_from_file(file, "config:%d\n");
9599 }
9600
9601 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9602                                  uint64_t offset, int pid)
9603 {
9604         struct perf_event_attr attr = {};
9605         char errmsg[STRERR_BUFSIZE];
9606         int type, pfd, err;
9607
9608         type = uprobe ? determine_uprobe_perf_type()
9609                       : determine_kprobe_perf_type();
9610         if (type < 0) {
9611                 pr_warn("failed to determine %s perf type: %s\n",
9612                         uprobe ? "uprobe" : "kprobe",
9613                         libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9614                 return type;
9615         }
9616         if (retprobe) {
9617                 int bit = uprobe ? determine_uprobe_retprobe_bit()
9618                                  : determine_kprobe_retprobe_bit();
9619
9620                 if (bit < 0) {
9621                         pr_warn("failed to determine %s retprobe bit: %s\n",
9622                                 uprobe ? "uprobe" : "kprobe",
9623                                 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9624                         return bit;
9625                 }
9626                 attr.config |= 1 << bit;
9627         }
9628         attr.size = sizeof(attr);
9629         attr.type = type;
9630         attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
9631         attr.config2 = offset;           /* kprobe_addr or probe_offset */
9632
9633         /* pid filter is meaningful only for uprobes */
9634         pfd = syscall(__NR_perf_event_open, &attr,
9635                       pid < 0 ? -1 : pid /* pid */,
9636                       pid == -1 ? 0 : -1 /* cpu */,
9637                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9638         if (pfd < 0) {
9639                 err = -errno;
9640                 pr_warn("%s perf_event_open() failed: %s\n",
9641                         uprobe ? "uprobe" : "kprobe",
9642                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9643                 return err;
9644         }
9645         return pfd;
9646 }
9647
9648 struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
9649                                             bool retprobe,
9650                                             const char *func_name)
9651 {
9652         char errmsg[STRERR_BUFSIZE];
9653         struct bpf_link *link;
9654         int pfd, err;
9655
9656         pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,
9657                                     0 /* offset */, -1 /* pid */);
9658         if (pfd < 0) {
9659                 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n",
9660                         prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9661                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9662                 return ERR_PTR(pfd);
9663         }
9664         link = bpf_program__attach_perf_event(prog, pfd);
9665         if (IS_ERR(link)) {
9666                 close(pfd);
9667                 err = PTR_ERR(link);
9668                 pr_warn("prog '%s': failed to attach to %s '%s': %s\n",
9669                         prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9670                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9671                 return link;
9672         }
9673         return link;
9674 }
9675
9676 static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
9677                                       struct bpf_program *prog)
9678 {
9679         const char *func_name;
9680         bool retprobe;
9681
9682         func_name = prog->sec_name + sec->len;
9683         retprobe = strcmp(sec->sec, "kretprobe/") == 0;
9684
9685         return bpf_program__attach_kprobe(prog, retprobe, func_name);
9686 }
9687
9688 struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
9689                                             bool retprobe, pid_t pid,
9690                                             const char *binary_path,
9691                                             size_t func_offset)
9692 {
9693         char errmsg[STRERR_BUFSIZE];
9694         struct bpf_link *link;
9695         int pfd, err;
9696
9697         pfd = perf_event_open_probe(true /* uprobe */, retprobe,
9698                                     binary_path, func_offset, pid);
9699         if (pfd < 0) {
9700                 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
9701                         prog->name, retprobe ? "uretprobe" : "uprobe",
9702                         binary_path, func_offset,
9703                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9704                 return ERR_PTR(pfd);
9705         }
9706         link = bpf_program__attach_perf_event(prog, pfd);
9707         if (IS_ERR(link)) {
9708                 close(pfd);
9709                 err = PTR_ERR(link);
9710                 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
9711                         prog->name, retprobe ? "uretprobe" : "uprobe",
9712                         binary_path, func_offset,
9713                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9714                 return link;
9715         }
9716         return link;
9717 }
9718
9719 static int determine_tracepoint_id(const char *tp_category,
9720                                    const char *tp_name)
9721 {
9722         char file[PATH_MAX];
9723         int ret;
9724
9725         ret = snprintf(file, sizeof(file),
9726                        "/sys/kernel/debug/tracing/events/%s/%s/id",
9727                        tp_category, tp_name);
9728         if (ret < 0)
9729                 return -errno;
9730         if (ret >= sizeof(file)) {
9731                 pr_debug("tracepoint %s/%s path is too long\n",
9732                          tp_category, tp_name);
9733                 return -E2BIG;
9734         }
9735         return parse_uint_from_file(file, "%d\n");
9736 }
9737
9738 static int perf_event_open_tracepoint(const char *tp_category,
9739                                       const char *tp_name)
9740 {
9741         struct perf_event_attr attr = {};
9742         char errmsg[STRERR_BUFSIZE];
9743         int tp_id, pfd, err;
9744
9745         tp_id = determine_tracepoint_id(tp_category, tp_name);
9746         if (tp_id < 0) {
9747                 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
9748                         tp_category, tp_name,
9749                         libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
9750                 return tp_id;
9751         }
9752
9753         attr.type = PERF_TYPE_TRACEPOINT;
9754         attr.size = sizeof(attr);
9755         attr.config = tp_id;
9756
9757         pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
9758                       -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9759         if (pfd < 0) {
9760                 err = -errno;
9761                 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
9762                         tp_category, tp_name,
9763                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9764                 return err;
9765         }
9766         return pfd;
9767 }
9768
9769 struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
9770                                                 const char *tp_category,
9771                                                 const char *tp_name)
9772 {
9773         char errmsg[STRERR_BUFSIZE];
9774         struct bpf_link *link;
9775         int pfd, err;
9776
9777         pfd = perf_event_open_tracepoint(tp_category, tp_name);
9778         if (pfd < 0) {
9779                 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
9780                         prog->name, tp_category, tp_name,
9781                         libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9782                 return ERR_PTR(pfd);
9783         }
9784         link = bpf_program__attach_perf_event(prog, pfd);
9785         if (IS_ERR(link)) {
9786                 close(pfd);
9787                 err = PTR_ERR(link);
9788                 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
9789                         prog->name, tp_category, tp_name,
9790                         libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9791                 return link;
9792         }
9793         return link;
9794 }
9795
9796 static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
9797                                   struct bpf_program *prog)
9798 {
9799         char *sec_name, *tp_cat, *tp_name;
9800         struct bpf_link *link;
9801
9802         sec_name = strdup(prog->sec_name);
9803         if (!sec_name)
9804                 return ERR_PTR(-ENOMEM);
9805
9806         /* extract "tp/<category>/<name>" */
9807         tp_cat = sec_name + sec->len;
9808         tp_name = strchr(tp_cat, '/');
9809         if (!tp_name) {
9810                 link = ERR_PTR(-EINVAL);
9811                 goto out;
9812         }
9813         *tp_name = '\0';
9814         tp_name++;
9815
9816         link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
9817 out:
9818         free(sec_name);
9819         return link;
9820 }
9821
9822 struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
9823                                                     const char *tp_name)
9824 {
9825         char errmsg[STRERR_BUFSIZE];
9826         struct bpf_link *link;
9827         int prog_fd, pfd;
9828
9829         prog_fd = bpf_program__fd(prog);
9830         if (prog_fd < 0) {
9831                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9832                 return ERR_PTR(-EINVAL);
9833         }
9834
9835         link = calloc(1, sizeof(*link));
9836         if (!link)
9837                 return ERR_PTR(-ENOMEM);
9838         link->detach = &bpf_link__detach_fd;
9839
9840         pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
9841         if (pfd < 0) {
9842                 pfd = -errno;
9843                 free(link);
9844                 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
9845                         prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9846                 return ERR_PTR(pfd);
9847         }
9848         link->fd = pfd;
9849         return link;
9850 }
9851
9852 static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
9853                                       struct bpf_program *prog)
9854 {
9855         const char *tp_name = prog->sec_name + sec->len;
9856
9857         return bpf_program__attach_raw_tracepoint(prog, tp_name);
9858 }
9859
9860 /* Common logic for all BPF program types that attach to a btf_id */
9861 static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
9862 {
9863         char errmsg[STRERR_BUFSIZE];
9864         struct bpf_link *link;
9865         int prog_fd, pfd;
9866
9867         prog_fd = bpf_program__fd(prog);
9868         if (prog_fd < 0) {
9869                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9870                 return ERR_PTR(-EINVAL);
9871         }
9872
9873         link = calloc(1, sizeof(*link));
9874         if (!link)
9875                 return ERR_PTR(-ENOMEM);
9876         link->detach = &bpf_link__detach_fd;
9877
9878         pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
9879         if (pfd < 0) {
9880                 pfd = -errno;
9881                 free(link);
9882                 pr_warn("prog '%s': failed to attach: %s\n",
9883                         prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9884                 return ERR_PTR(pfd);
9885         }
9886         link->fd = pfd;
9887         return (struct bpf_link *)link;
9888 }
9889
9890 struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
9891 {
9892         return bpf_program__attach_btf_id(prog);
9893 }
9894
9895 struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
9896 {
9897         return bpf_program__attach_btf_id(prog);
9898 }
9899
9900 static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
9901                                      struct bpf_program *prog)
9902 {
9903         return bpf_program__attach_trace(prog);
9904 }
9905
9906 static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
9907                                    struct bpf_program *prog)
9908 {
9909         return bpf_program__attach_lsm(prog);
9910 }
9911
9912 static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
9913                                     struct bpf_program *prog)
9914 {
9915         return bpf_program__attach_iter(prog, NULL);
9916 }
9917
9918 static struct bpf_link *
9919 bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id,
9920                        const char *target_name)
9921 {
9922         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
9923                             .target_btf_id = btf_id);
9924         enum bpf_attach_type attach_type;
9925         char errmsg[STRERR_BUFSIZE];
9926         struct bpf_link *link;
9927         int prog_fd, link_fd;
9928
9929         prog_fd = bpf_program__fd(prog);
9930         if (prog_fd < 0) {
9931                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9932                 return ERR_PTR(-EINVAL);
9933         }
9934
9935         link = calloc(1, sizeof(*link));
9936         if (!link)
9937                 return ERR_PTR(-ENOMEM);
9938         link->detach = &bpf_link__detach_fd;
9939
9940         attach_type = bpf_program__get_expected_attach_type(prog);
9941         link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
9942         if (link_fd < 0) {
9943                 link_fd = -errno;
9944                 free(link);
9945                 pr_warn("prog '%s': failed to attach to %s: %s\n",
9946                         prog->name, target_name,
9947                         libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
9948                 return ERR_PTR(link_fd);
9949         }
9950         link->fd = link_fd;
9951         return link;
9952 }
9953
9954 struct bpf_link *
9955 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
9956 {
9957         return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
9958 }
9959
9960 struct bpf_link *
9961 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)
9962 {
9963         return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
9964 }
9965
9966 struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)
9967 {
9968         /* target_fd/target_ifindex use the same field in LINK_CREATE */
9969         return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
9970 }
9971
9972 struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog,
9973                                               int target_fd,
9974                                               const char *attach_func_name)
9975 {
9976         int btf_id;
9977
9978         if (!!target_fd != !!attach_func_name) {
9979                 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
9980                         prog->name);
9981                 return ERR_PTR(-EINVAL);
9982         }
9983
9984         if (prog->type != BPF_PROG_TYPE_EXT) {
9985                 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
9986                         prog->name);
9987                 return ERR_PTR(-EINVAL);
9988         }
9989
9990         if (target_fd) {
9991                 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
9992                 if (btf_id < 0)
9993                         return ERR_PTR(btf_id);
9994
9995                 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
9996         } else {
9997                 /* no target, so use raw_tracepoint_open for compatibility
9998                  * with old kernels
9999                  */
10000                 return bpf_program__attach_trace(prog);
10001         }
10002 }
10003
10004 struct bpf_link *
10005 bpf_program__attach_iter(struct bpf_program *prog,
10006                          const struct bpf_iter_attach_opts *opts)
10007 {
10008         DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
10009         char errmsg[STRERR_BUFSIZE];
10010         struct bpf_link *link;
10011         int prog_fd, link_fd;
10012         __u32 target_fd = 0;
10013
10014         if (!OPTS_VALID(opts, bpf_iter_attach_opts))
10015                 return ERR_PTR(-EINVAL);
10016
10017         link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
10018         link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
10019
10020         prog_fd = bpf_program__fd(prog);
10021         if (prog_fd < 0) {
10022                 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
10023                 return ERR_PTR(-EINVAL);
10024         }
10025
10026         link = calloc(1, sizeof(*link));
10027         if (!link)
10028                 return ERR_PTR(-ENOMEM);
10029         link->detach = &bpf_link__detach_fd;
10030
10031         link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
10032                                   &link_create_opts);
10033         if (link_fd < 0) {
10034                 link_fd = -errno;
10035                 free(link);
10036                 pr_warn("prog '%s': failed to attach to iterator: %s\n",
10037                         prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
10038                 return ERR_PTR(link_fd);
10039         }
10040         link->fd = link_fd;
10041         return link;
10042 }
10043
10044 struct bpf_link *bpf_program__attach(struct bpf_program *prog)
10045 {
10046         const struct bpf_sec_def *sec_def;
10047
10048         sec_def = find_sec_def(prog->sec_name);
10049         if (!sec_def || !sec_def->attach_fn)
10050                 return ERR_PTR(-ESRCH);
10051
10052         return sec_def->attach_fn(sec_def, prog);
10053 }
10054
10055 static int bpf_link__detach_struct_ops(struct bpf_link *link)
10056 {
10057         __u32 zero = 0;
10058
10059         if (bpf_map_delete_elem(link->fd, &zero))
10060                 return -errno;
10061
10062         return 0;
10063 }
10064
10065 struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
10066 {
10067         struct bpf_struct_ops *st_ops;
10068         struct bpf_link *link;
10069         __u32 i, zero = 0;
10070         int err;
10071
10072         if (!bpf_map__is_struct_ops(map) || map->fd == -1)
10073                 return ERR_PTR(-EINVAL);
10074
10075         link = calloc(1, sizeof(*link));
10076         if (!link)
10077                 return ERR_PTR(-EINVAL);
10078
10079         st_ops = map->st_ops;
10080         for (i = 0; i < btf_vlen(st_ops->type); i++) {
10081                 struct bpf_program *prog = st_ops->progs[i];
10082                 void *kern_data;
10083                 int prog_fd;
10084
10085                 if (!prog)
10086                         continue;
10087
10088                 prog_fd = bpf_program__fd(prog);
10089                 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
10090                 *(unsigned long *)kern_data = prog_fd;
10091         }
10092
10093         err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
10094         if (err) {
10095                 err = -errno;
10096                 free(link);
10097                 return ERR_PTR(err);
10098         }
10099
10100         link->detach = bpf_link__detach_struct_ops;
10101         link->fd = map->fd;
10102
10103         return link;
10104 }
10105
10106 enum bpf_perf_event_ret
10107 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
10108                            void **copy_mem, size_t *copy_size,
10109                            bpf_perf_event_print_t fn, void *private_data)
10110 {
10111         struct perf_event_mmap_page *header = mmap_mem;
10112         __u64 data_head = ring_buffer_read_head(header);
10113         __u64 data_tail = header->data_tail;
10114         void *base = ((__u8 *)header) + page_size;
10115         int ret = LIBBPF_PERF_EVENT_CONT;
10116         struct perf_event_header *ehdr;
10117         size_t ehdr_size;
10118
10119         while (data_head != data_tail) {
10120                 ehdr = base + (data_tail & (mmap_size - 1));
10121                 ehdr_size = ehdr->size;
10122
10123                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
10124                         void *copy_start = ehdr;
10125                         size_t len_first = base + mmap_size - copy_start;
10126                         size_t len_secnd = ehdr_size - len_first;
10127
10128                         if (*copy_size < ehdr_size) {
10129                                 free(*copy_mem);
10130                                 *copy_mem = malloc(ehdr_size);
10131                                 if (!*copy_mem) {
10132                                         *copy_size = 0;
10133                                         ret = LIBBPF_PERF_EVENT_ERROR;
10134                                         break;
10135                                 }
10136                                 *copy_size = ehdr_size;
10137                         }
10138
10139                         memcpy(*copy_mem, copy_start, len_first);
10140                         memcpy(*copy_mem + len_first, base, len_secnd);
10141                         ehdr = *copy_mem;
10142                 }
10143
10144                 ret = fn(ehdr, private_data);
10145                 data_tail += ehdr_size;
10146                 if (ret != LIBBPF_PERF_EVENT_CONT)
10147                         break;
10148         }
10149
10150         ring_buffer_write_tail(header, data_tail);
10151         return ret;
10152 }
10153
10154 struct perf_buffer;
10155
10156 struct perf_buffer_params {
10157         struct perf_event_attr *attr;
10158         /* if event_cb is specified, it takes precendence */
10159         perf_buffer_event_fn event_cb;
10160         /* sample_cb and lost_cb are higher-level common-case callbacks */
10161         perf_buffer_sample_fn sample_cb;
10162         perf_buffer_lost_fn lost_cb;
10163         void *ctx;
10164         int cpu_cnt;
10165         int *cpus;
10166         int *map_keys;
10167 };
10168
10169 struct perf_cpu_buf {
10170         struct perf_buffer *pb;
10171         void *base; /* mmap()'ed memory */
10172         void *buf; /* for reconstructing segmented data */
10173         size_t buf_size;
10174         int fd;
10175         int cpu;
10176         int map_key;
10177 };
10178
10179 struct perf_buffer {
10180         perf_buffer_event_fn event_cb;
10181         perf_buffer_sample_fn sample_cb;
10182         perf_buffer_lost_fn lost_cb;
10183         void *ctx; /* passed into callbacks */
10184
10185         size_t page_size;
10186         size_t mmap_size;
10187         struct perf_cpu_buf **cpu_bufs;
10188         struct epoll_event *events;
10189         int cpu_cnt; /* number of allocated CPU buffers */
10190         int epoll_fd; /* perf event FD */
10191         int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */
10192 };
10193
10194 static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
10195                                       struct perf_cpu_buf *cpu_buf)
10196 {
10197         if (!cpu_buf)
10198                 return;
10199         if (cpu_buf->base &&
10200             munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
10201                 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
10202         if (cpu_buf->fd >= 0) {
10203                 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
10204                 close(cpu_buf->fd);
10205         }
10206         free(cpu_buf->buf);
10207         free(cpu_buf);
10208 }
10209
10210 void perf_buffer__free(struct perf_buffer *pb)
10211 {
10212         int i;
10213
10214         if (IS_ERR_OR_NULL(pb))
10215                 return;
10216         if (pb->cpu_bufs) {
10217                 for (i = 0; i < pb->cpu_cnt; i++) {
10218                         struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10219
10220                         if (!cpu_buf)
10221                                 continue;
10222
10223                         bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
10224                         perf_buffer__free_cpu_buf(pb, cpu_buf);
10225                 }
10226                 free(pb->cpu_bufs);
10227         }
10228         if (pb->epoll_fd >= 0)
10229                 close(pb->epoll_fd);
10230         free(pb->events);
10231         free(pb);
10232 }
10233
10234 static struct perf_cpu_buf *
10235 perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
10236                           int cpu, int map_key)
10237 {
10238         struct perf_cpu_buf *cpu_buf;
10239         char msg[STRERR_BUFSIZE];
10240         int err;
10241
10242         cpu_buf = calloc(1, sizeof(*cpu_buf));
10243         if (!cpu_buf)
10244                 return ERR_PTR(-ENOMEM);
10245
10246         cpu_buf->pb = pb;
10247         cpu_buf->cpu = cpu;
10248         cpu_buf->map_key = map_key;
10249
10250         cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,
10251                               -1, PERF_FLAG_FD_CLOEXEC);
10252         if (cpu_buf->fd < 0) {
10253                 err = -errno;
10254                 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
10255                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10256                 goto error;
10257         }
10258
10259         cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
10260                              PROT_READ | PROT_WRITE, MAP_SHARED,
10261                              cpu_buf->fd, 0);
10262         if (cpu_buf->base == MAP_FAILED) {
10263                 cpu_buf->base = NULL;
10264                 err = -errno;
10265                 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
10266                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10267                 goto error;
10268         }
10269
10270         if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
10271                 err = -errno;
10272                 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
10273                         cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10274                 goto error;
10275         }
10276
10277         return cpu_buf;
10278
10279 error:
10280         perf_buffer__free_cpu_buf(pb, cpu_buf);
10281         return (struct perf_cpu_buf *)ERR_PTR(err);
10282 }
10283
10284 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10285                                               struct perf_buffer_params *p);
10286
10287 struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
10288                                      const struct perf_buffer_opts *opts)
10289 {
10290         struct perf_buffer_params p = {};
10291         struct perf_event_attr attr = { 0, };
10292
10293         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
10294         attr.type = PERF_TYPE_SOFTWARE;
10295         attr.sample_type = PERF_SAMPLE_RAW;
10296         attr.sample_period = 1;
10297         attr.wakeup_events = 1;
10298
10299         p.attr = &attr;
10300         p.sample_cb = opts ? opts->sample_cb : NULL;
10301         p.lost_cb = opts ? opts->lost_cb : NULL;
10302         p.ctx = opts ? opts->ctx : NULL;
10303
10304         return __perf_buffer__new(map_fd, page_cnt, &p);
10305 }
10306
10307 struct perf_buffer *
10308 perf_buffer__new_raw(int map_fd, size_t page_cnt,
10309                      const struct perf_buffer_raw_opts *opts)
10310 {
10311         struct perf_buffer_params p = {};
10312
10313         p.attr = opts->attr;
10314         p.event_cb = opts->event_cb;
10315         p.ctx = opts->ctx;
10316         p.cpu_cnt = opts->cpu_cnt;
10317         p.cpus = opts->cpus;
10318         p.map_keys = opts->map_keys;
10319
10320         return __perf_buffer__new(map_fd, page_cnt, &p);
10321 }
10322
10323 static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10324                                               struct perf_buffer_params *p)
10325 {
10326         const char *online_cpus_file = "/sys/devices/system/cpu/online";
10327         struct bpf_map_info map;
10328         char msg[STRERR_BUFSIZE];
10329         struct perf_buffer *pb;
10330         bool *online = NULL;
10331         __u32 map_info_len;
10332         int err, i, j, n;
10333
10334         if (page_cnt & (page_cnt - 1)) {
10335                 pr_warn("page count should be power of two, but is %zu\n",
10336                         page_cnt);
10337                 return ERR_PTR(-EINVAL);
10338         }
10339
10340         /* best-effort sanity checks */
10341         memset(&map, 0, sizeof(map));
10342         map_info_len = sizeof(map);
10343         err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
10344         if (err) {
10345                 err = -errno;
10346                 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return
10347                  * -EBADFD, -EFAULT, or -E2BIG on real error
10348                  */
10349                 if (err != -EINVAL) {
10350                         pr_warn("failed to get map info for map FD %d: %s\n",
10351                                 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
10352                         return ERR_PTR(err);
10353                 }
10354                 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
10355                          map_fd);
10356         } else {
10357                 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
10358                         pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
10359                                 map.name);
10360                         return ERR_PTR(-EINVAL);
10361                 }
10362         }
10363
10364         pb = calloc(1, sizeof(*pb));
10365         if (!pb)
10366                 return ERR_PTR(-ENOMEM);
10367
10368         pb->event_cb = p->event_cb;
10369         pb->sample_cb = p->sample_cb;
10370         pb->lost_cb = p->lost_cb;
10371         pb->ctx = p->ctx;
10372
10373         pb->page_size = getpagesize();
10374         pb->mmap_size = pb->page_size * page_cnt;
10375         pb->map_fd = map_fd;
10376
10377         pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
10378         if (pb->epoll_fd < 0) {
10379                 err = -errno;
10380                 pr_warn("failed to create epoll instance: %s\n",
10381                         libbpf_strerror_r(err, msg, sizeof(msg)));
10382                 goto error;
10383         }
10384
10385         if (p->cpu_cnt > 0) {
10386                 pb->cpu_cnt = p->cpu_cnt;
10387         } else {
10388                 pb->cpu_cnt = libbpf_num_possible_cpus();
10389                 if (pb->cpu_cnt < 0) {
10390                         err = pb->cpu_cnt;
10391                         goto error;
10392                 }
10393                 if (map.max_entries && map.max_entries < pb->cpu_cnt)
10394                         pb->cpu_cnt = map.max_entries;
10395         }
10396
10397         pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
10398         if (!pb->events) {
10399                 err = -ENOMEM;
10400                 pr_warn("failed to allocate events: out of memory\n");
10401                 goto error;
10402         }
10403         pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
10404         if (!pb->cpu_bufs) {
10405                 err = -ENOMEM;
10406                 pr_warn("failed to allocate buffers: out of memory\n");
10407                 goto error;
10408         }
10409
10410         err = parse_cpu_mask_file(online_cpus_file, &online, &n);
10411         if (err) {
10412                 pr_warn("failed to get online CPU mask: %d\n", err);
10413                 goto error;
10414         }
10415
10416         for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
10417                 struct perf_cpu_buf *cpu_buf;
10418                 int cpu, map_key;
10419
10420                 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
10421                 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
10422
10423                 /* in case user didn't explicitly requested particular CPUs to
10424                  * be attached to, skip offline/not present CPUs
10425                  */
10426                 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
10427                         continue;
10428
10429                 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
10430                 if (IS_ERR(cpu_buf)) {
10431                         err = PTR_ERR(cpu_buf);
10432                         goto error;
10433                 }
10434
10435                 pb->cpu_bufs[j] = cpu_buf;
10436
10437                 err = bpf_map_update_elem(pb->map_fd, &map_key,
10438                                           &cpu_buf->fd, 0);
10439                 if (err) {
10440                         err = -errno;
10441                         pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
10442                                 cpu, map_key, cpu_buf->fd,
10443                                 libbpf_strerror_r(err, msg, sizeof(msg)));
10444                         goto error;
10445                 }
10446
10447                 pb->events[j].events = EPOLLIN;
10448                 pb->events[j].data.ptr = cpu_buf;
10449                 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
10450                               &pb->events[j]) < 0) {
10451                         err = -errno;
10452                         pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
10453                                 cpu, cpu_buf->fd,
10454                                 libbpf_strerror_r(err, msg, sizeof(msg)));
10455                         goto error;
10456                 }
10457                 j++;
10458         }
10459         pb->cpu_cnt = j;
10460         free(online);
10461
10462         return pb;
10463
10464 error:
10465         free(online);
10466         if (pb)
10467                 perf_buffer__free(pb);
10468         return ERR_PTR(err);
10469 }
10470
10471 struct perf_sample_raw {
10472         struct perf_event_header header;
10473         uint32_t size;
10474         char data[];
10475 };
10476
10477 struct perf_sample_lost {
10478         struct perf_event_header header;
10479         uint64_t id;
10480         uint64_t lost;
10481         uint64_t sample_id;
10482 };
10483
10484 static enum bpf_perf_event_ret
10485 perf_buffer__process_record(struct perf_event_header *e, void *ctx)
10486 {
10487         struct perf_cpu_buf *cpu_buf = ctx;
10488         struct perf_buffer *pb = cpu_buf->pb;
10489         void *data = e;
10490
10491         /* user wants full control over parsing perf event */
10492         if (pb->event_cb)
10493                 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
10494
10495         switch (e->type) {
10496         case PERF_RECORD_SAMPLE: {
10497                 struct perf_sample_raw *s = data;
10498
10499                 if (pb->sample_cb)
10500                         pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
10501                 break;
10502         }
10503         case PERF_RECORD_LOST: {
10504                 struct perf_sample_lost *s = data;
10505
10506                 if (pb->lost_cb)
10507                         pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
10508                 break;
10509         }
10510         default:
10511                 pr_warn("unknown perf sample type %d\n", e->type);
10512                 return LIBBPF_PERF_EVENT_ERROR;
10513         }
10514         return LIBBPF_PERF_EVENT_CONT;
10515 }
10516
10517 static int perf_buffer__process_records(struct perf_buffer *pb,
10518                                         struct perf_cpu_buf *cpu_buf)
10519 {
10520         enum bpf_perf_event_ret ret;
10521
10522         ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
10523                                          pb->page_size, &cpu_buf->buf,
10524                                          &cpu_buf->buf_size,
10525                                          perf_buffer__process_record, cpu_buf);
10526         if (ret != LIBBPF_PERF_EVENT_CONT)
10527                 return ret;
10528         return 0;
10529 }
10530
10531 int perf_buffer__epoll_fd(const struct perf_buffer *pb)
10532 {
10533         return pb->epoll_fd;
10534 }
10535
10536 int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
10537 {
10538         int i, cnt, err;
10539
10540         cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
10541         for (i = 0; i < cnt; i++) {
10542                 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
10543
10544                 err = perf_buffer__process_records(pb, cpu_buf);
10545                 if (err) {
10546                         pr_warn("error while processing records: %d\n", err);
10547                         return err;
10548                 }
10549         }
10550         return cnt < 0 ? -errno : cnt;
10551 }
10552
10553 /* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer
10554  * manager.
10555  */
10556 size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
10557 {
10558         return pb->cpu_cnt;
10559 }
10560
10561 /*
10562  * Return perf_event FD of a ring buffer in *buf_idx* slot of
10563  * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using
10564  * select()/poll()/epoll() Linux syscalls.
10565  */
10566 int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
10567 {
10568         struct perf_cpu_buf *cpu_buf;
10569
10570         if (buf_idx >= pb->cpu_cnt)
10571                 return -EINVAL;
10572
10573         cpu_buf = pb->cpu_bufs[buf_idx];
10574         if (!cpu_buf)
10575                 return -ENOENT;
10576
10577         return cpu_buf->fd;
10578 }
10579
10580 /*
10581  * Consume data from perf ring buffer corresponding to slot *buf_idx* in
10582  * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
10583  * consume, do nothing and return success.
10584  * Returns:
10585  *   - 0 on success;
10586  *   - <0 on failure.
10587  */
10588 int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
10589 {
10590         struct perf_cpu_buf *cpu_buf;
10591
10592         if (buf_idx >= pb->cpu_cnt)
10593                 return -EINVAL;
10594
10595         cpu_buf = pb->cpu_bufs[buf_idx];
10596         if (!cpu_buf)
10597                 return -ENOENT;
10598
10599         return perf_buffer__process_records(pb, cpu_buf);
10600 }
10601
10602 int perf_buffer__consume(struct perf_buffer *pb)
10603 {
10604         int i, err;
10605
10606         for (i = 0; i < pb->cpu_cnt; i++) {
10607                 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10608
10609                 if (!cpu_buf)
10610                         continue;
10611
10612                 err = perf_buffer__process_records(pb, cpu_buf);
10613                 if (err) {
10614                         pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
10615                         return err;
10616                 }
10617         }
10618         return 0;
10619 }
10620
10621 struct bpf_prog_info_array_desc {
10622         int     array_offset;   /* e.g. offset of jited_prog_insns */
10623         int     count_offset;   /* e.g. offset of jited_prog_len */
10624         int     size_offset;    /* > 0: offset of rec size,
10625                                  * < 0: fix size of -size_offset
10626                                  */
10627 };
10628
10629 static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
10630         [BPF_PROG_INFO_JITED_INSNS] = {
10631                 offsetof(struct bpf_prog_info, jited_prog_insns),
10632                 offsetof(struct bpf_prog_info, jited_prog_len),
10633                 -1,
10634         },
10635         [BPF_PROG_INFO_XLATED_INSNS] = {
10636                 offsetof(struct bpf_prog_info, xlated_prog_insns),
10637                 offsetof(struct bpf_prog_info, xlated_prog_len),
10638                 -1,
10639         },
10640         [BPF_PROG_INFO_MAP_IDS] = {
10641                 offsetof(struct bpf_prog_info, map_ids),
10642                 offsetof(struct bpf_prog_info, nr_map_ids),
10643                 -(int)sizeof(__u32),
10644         },
10645         [BPF_PROG_INFO_JITED_KSYMS] = {
10646                 offsetof(struct bpf_prog_info, jited_ksyms),
10647                 offsetof(struct bpf_prog_info, nr_jited_ksyms),
10648                 -(int)sizeof(__u64),
10649         },
10650         [BPF_PROG_INFO_JITED_FUNC_LENS] = {
10651                 offsetof(struct bpf_prog_info, jited_func_lens),
10652                 offsetof(struct bpf_prog_info, nr_jited_func_lens),
10653                 -(int)sizeof(__u32),
10654         },
10655         [BPF_PROG_INFO_FUNC_INFO] = {
10656                 offsetof(struct bpf_prog_info, func_info),
10657                 offsetof(struct bpf_prog_info, nr_func_info),
10658                 offsetof(struct bpf_prog_info, func_info_rec_size),
10659         },
10660         [BPF_PROG_INFO_LINE_INFO] = {
10661                 offsetof(struct bpf_prog_info, line_info),
10662                 offsetof(struct bpf_prog_info, nr_line_info),
10663                 offsetof(struct bpf_prog_info, line_info_rec_size),
10664         },
10665         [BPF_PROG_INFO_JITED_LINE_INFO] = {
10666                 offsetof(struct bpf_prog_info, jited_line_info),
10667                 offsetof(struct bpf_prog_info, nr_jited_line_info),
10668                 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
10669         },
10670         [BPF_PROG_INFO_PROG_TAGS] = {
10671                 offsetof(struct bpf_prog_info, prog_tags),
10672                 offsetof(struct bpf_prog_info, nr_prog_tags),
10673                 -(int)sizeof(__u8) * BPF_TAG_SIZE,
10674         },
10675
10676 };
10677
10678 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
10679                                            int offset)
10680 {
10681         __u32 *array = (__u32 *)info;
10682
10683         if (offset >= 0)
10684                 return array[offset / sizeof(__u32)];
10685         return -(int)offset;
10686 }
10687
10688 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
10689                                            int offset)
10690 {
10691         __u64 *array = (__u64 *)info;
10692
10693         if (offset >= 0)
10694                 return array[offset / sizeof(__u64)];
10695         return -(int)offset;
10696 }
10697
10698 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
10699                                          __u32 val)
10700 {
10701         __u32 *array = (__u32 *)info;
10702
10703         if (offset >= 0)
10704                 array[offset / sizeof(__u32)] = val;
10705 }
10706
10707 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
10708                                          __u64 val)
10709 {
10710         __u64 *array = (__u64 *)info;
10711
10712         if (offset >= 0)
10713                 array[offset / sizeof(__u64)] = val;
10714 }
10715
10716 struct bpf_prog_info_linear *
10717 bpf_program__get_prog_info_linear(int fd, __u64 arrays)
10718 {
10719         struct bpf_prog_info_linear *info_linear;
10720         struct bpf_prog_info info = {};
10721         __u32 info_len = sizeof(info);
10722         __u32 data_len = 0;
10723         int i, err;
10724         void *ptr;
10725
10726         if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
10727                 return ERR_PTR(-EINVAL);
10728
10729         /* step 1: get array dimensions */
10730         err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
10731         if (err) {
10732                 pr_debug("can't get prog info: %s", strerror(errno));
10733                 return ERR_PTR(-EFAULT);
10734         }
10735
10736         /* step 2: calculate total size of all arrays */
10737         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10738                 bool include_array = (arrays & (1UL << i)) > 0;
10739                 struct bpf_prog_info_array_desc *desc;
10740                 __u32 count, size;
10741
10742                 desc = bpf_prog_info_array_desc + i;
10743
10744                 /* kernel is too old to support this field */
10745                 if (info_len < desc->array_offset + sizeof(__u32) ||
10746                     info_len < desc->count_offset + sizeof(__u32) ||
10747                     (desc->size_offset > 0 && info_len < desc->size_offset))
10748                         include_array = false;
10749
10750                 if (!include_array) {
10751                         arrays &= ~(1UL << i);  /* clear the bit */
10752                         continue;
10753                 }
10754
10755                 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10756                 size  = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10757
10758                 data_len += count * size;
10759         }
10760
10761         /* step 3: allocate continuous memory */
10762         data_len = roundup(data_len, sizeof(__u64));
10763         info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
10764         if (!info_linear)
10765                 return ERR_PTR(-ENOMEM);
10766
10767         /* step 4: fill data to info_linear->info */
10768         info_linear->arrays = arrays;
10769         memset(&info_linear->info, 0, sizeof(info));
10770         ptr = info_linear->data;
10771
10772         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10773                 struct bpf_prog_info_array_desc *desc;
10774                 __u32 count, size;
10775
10776                 if ((arrays & (1UL << i)) == 0)
10777                         continue;
10778
10779                 desc  = bpf_prog_info_array_desc + i;
10780                 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10781                 size  = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10782                 bpf_prog_info_set_offset_u32(&info_linear->info,
10783                                              desc->count_offset, count);
10784                 bpf_prog_info_set_offset_u32(&info_linear->info,
10785                                              desc->size_offset, size);
10786                 bpf_prog_info_set_offset_u64(&info_linear->info,
10787                                              desc->array_offset,
10788                                              ptr_to_u64(ptr));
10789                 ptr += count * size;
10790         }
10791
10792         /* step 5: call syscall again to get required arrays */
10793         err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
10794         if (err) {
10795                 pr_debug("can't get prog info: %s", strerror(errno));
10796                 free(info_linear);
10797                 return ERR_PTR(-EFAULT);
10798         }
10799
10800         /* step 6: verify the data */
10801         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10802                 struct bpf_prog_info_array_desc *desc;
10803                 __u32 v1, v2;
10804
10805                 if ((arrays & (1UL << i)) == 0)
10806                         continue;
10807
10808                 desc = bpf_prog_info_array_desc + i;
10809                 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10810                 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10811                                                    desc->count_offset);
10812                 if (v1 != v2)
10813                         pr_warn("%s: mismatch in element count\n", __func__);
10814
10815                 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10816                 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10817                                                    desc->size_offset);
10818                 if (v1 != v2)
10819                         pr_warn("%s: mismatch in rec size\n", __func__);
10820         }
10821
10822         /* step 7: update info_len and data_len */
10823         info_linear->info_len = sizeof(struct bpf_prog_info);
10824         info_linear->data_len = data_len;
10825
10826         return info_linear;
10827 }
10828
10829 void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
10830 {
10831         int i;
10832
10833         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10834                 struct bpf_prog_info_array_desc *desc;
10835                 __u64 addr, offs;
10836
10837                 if ((info_linear->arrays & (1UL << i)) == 0)
10838                         continue;
10839
10840                 desc = bpf_prog_info_array_desc + i;
10841                 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
10842                                                      desc->array_offset);
10843                 offs = addr - ptr_to_u64(info_linear->data);
10844                 bpf_prog_info_set_offset_u64(&info_linear->info,
10845                                              desc->array_offset, offs);
10846         }
10847 }
10848
10849 void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
10850 {
10851         int i;
10852
10853         for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10854                 struct bpf_prog_info_array_desc *desc;
10855                 __u64 addr, offs;
10856
10857                 if ((info_linear->arrays & (1UL << i)) == 0)
10858                         continue;
10859
10860                 desc = bpf_prog_info_array_desc + i;
10861                 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
10862                                                      desc->array_offset);
10863                 addr = offs + ptr_to_u64(info_linear->data);
10864                 bpf_prog_info_set_offset_u64(&info_linear->info,
10865                                              desc->array_offset, addr);
10866         }
10867 }
10868
10869 int bpf_program__set_attach_target(struct bpf_program *prog,
10870                                    int attach_prog_fd,
10871                                    const char *attach_func_name)
10872 {
10873         int btf_id;
10874
10875         if (!prog || attach_prog_fd < 0 || !attach_func_name)
10876                 return -EINVAL;
10877
10878         if (attach_prog_fd)
10879                 btf_id = libbpf_find_prog_btf_id(attach_func_name,
10880                                                  attach_prog_fd);
10881         else
10882                 btf_id = libbpf_find_vmlinux_btf_id(attach_func_name,
10883                                                     prog->expected_attach_type);
10884
10885         if (btf_id < 0)
10886                 return btf_id;
10887
10888         prog->attach_btf_id = btf_id;
10889         prog->attach_btf_obj_fd = 0;
10890         prog->attach_prog_fd = attach_prog_fd;
10891         return 0;
10892 }
10893
10894 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
10895 {
10896         int err = 0, n, len, start, end = -1;
10897         bool *tmp;
10898
10899         *mask = NULL;
10900         *mask_sz = 0;
10901
10902         /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
10903         while (*s) {
10904                 if (*s == ',' || *s == '\n') {
10905                         s++;
10906                         continue;
10907                 }
10908                 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
10909                 if (n <= 0 || n > 2) {
10910                         pr_warn("Failed to get CPU range %s: %d\n", s, n);
10911                         err = -EINVAL;
10912                         goto cleanup;
10913                 } else if (n == 1) {
10914                         end = start;
10915                 }
10916                 if (start < 0 || start > end) {
10917                         pr_warn("Invalid CPU range [%d,%d] in %s\n",
10918                                 start, end, s);
10919                         err = -EINVAL;
10920                         goto cleanup;
10921                 }
10922                 tmp = realloc(*mask, end + 1);
10923                 if (!tmp) {
10924                         err = -ENOMEM;
10925                         goto cleanup;
10926                 }
10927                 *mask = tmp;
10928                 memset(tmp + *mask_sz, 0, start - *mask_sz);
10929                 memset(tmp + start, 1, end - start + 1);
10930                 *mask_sz = end + 1;
10931                 s += len;
10932         }
10933         if (!*mask_sz) {
10934                 pr_warn("Empty CPU range\n");
10935                 return -EINVAL;
10936         }
10937         return 0;
10938 cleanup:
10939         free(*mask);
10940         *mask = NULL;
10941         return err;
10942 }
10943
10944 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
10945 {
10946         int fd, err = 0, len;
10947         char buf[128];
10948
10949         fd = open(fcpu, O_RDONLY);
10950         if (fd < 0) {
10951                 err = -errno;
10952                 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
10953                 return err;
10954         }
10955         len = read(fd, buf, sizeof(buf));
10956         close(fd);
10957         if (len <= 0) {
10958                 err = len ? -errno : -EINVAL;
10959                 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
10960                 return err;
10961         }
10962         if (len >= sizeof(buf)) {
10963                 pr_warn("CPU mask is too big in file %s\n", fcpu);
10964                 return -E2BIG;
10965         }
10966         buf[len] = '\0';
10967
10968         return parse_cpu_mask_str(buf, mask, mask_sz);
10969 }
10970
10971 int libbpf_num_possible_cpus(void)
10972 {
10973         static const char *fcpu = "/sys/devices/system/cpu/possible";
10974         static int cpus;
10975         int err, n, i, tmp_cpus;
10976         bool *mask;
10977
10978         tmp_cpus = READ_ONCE(cpus);
10979         if (tmp_cpus > 0)
10980                 return tmp_cpus;
10981
10982         err = parse_cpu_mask_file(fcpu, &mask, &n);
10983         if (err)
10984                 return err;
10985
10986         tmp_cpus = 0;
10987         for (i = 0; i < n; i++) {
10988                 if (mask[i])
10989                         tmp_cpus++;
10990         }
10991         free(mask);
10992
10993         WRITE_ONCE(cpus, tmp_cpus);
10994         return tmp_cpus;
10995 }
10996
10997 int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
10998                               const struct bpf_object_open_opts *opts)
10999 {
11000         DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
11001                 .object_name = s->name,
11002         );
11003         struct bpf_object *obj;
11004         int i;
11005
11006         /* Attempt to preserve opts->object_name, unless overriden by user
11007          * explicitly. Overwriting object name for skeletons is discouraged,
11008          * as it breaks global data maps, because they contain object name
11009          * prefix as their own map name prefix. When skeleton is generated,
11010          * bpftool is making an assumption that this name will stay the same.
11011          */
11012         if (opts) {
11013                 memcpy(&skel_opts, opts, sizeof(*opts));
11014                 if (!opts->object_name)
11015                         skel_opts.object_name = s->name;
11016         }
11017
11018         obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
11019         if (IS_ERR(obj)) {
11020                 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n",
11021                         s->name, PTR_ERR(obj));
11022                 return PTR_ERR(obj);
11023         }
11024
11025         *s->obj = obj;
11026
11027         for (i = 0; i < s->map_cnt; i++) {
11028                 struct bpf_map **map = s->maps[i].map;
11029                 const char *name = s->maps[i].name;
11030                 void **mmaped = s->maps[i].mmaped;
11031
11032                 *map = bpf_object__find_map_by_name(obj, name);
11033                 if (!*map) {
11034                         pr_warn("failed to find skeleton map '%s'\n", name);
11035                         return -ESRCH;
11036                 }
11037
11038                 /* externs shouldn't be pre-setup from user code */
11039                 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
11040                         *mmaped = (*map)->mmaped;
11041         }
11042
11043         for (i = 0; i < s->prog_cnt; i++) {
11044                 struct bpf_program **prog = s->progs[i].prog;
11045                 const char *name = s->progs[i].name;
11046
11047                 *prog = bpf_object__find_program_by_name(obj, name);
11048                 if (!*prog) {
11049                         pr_warn("failed to find skeleton program '%s'\n", name);
11050                         return -ESRCH;
11051                 }
11052         }
11053
11054         return 0;
11055 }
11056
11057 int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
11058 {
11059         int i, err;
11060
11061         err = bpf_object__load(*s->obj);
11062         if (err) {
11063                 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
11064                 return err;
11065         }
11066
11067         for (i = 0; i < s->map_cnt; i++) {
11068                 struct bpf_map *map = *s->maps[i].map;
11069                 size_t mmap_sz = bpf_map_mmap_sz(map);
11070                 int prot, map_fd = bpf_map__fd(map);
11071                 void **mmaped = s->maps[i].mmaped;
11072
11073                 if (!mmaped)
11074                         continue;
11075
11076                 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
11077                         *mmaped = NULL;
11078                         continue;
11079                 }
11080
11081                 if (map->def.map_flags & BPF_F_RDONLY_PROG)
11082                         prot = PROT_READ;
11083                 else
11084                         prot = PROT_READ | PROT_WRITE;
11085
11086                 /* Remap anonymous mmap()-ed "map initialization image" as
11087                  * a BPF map-backed mmap()-ed memory, but preserving the same
11088                  * memory address. This will cause kernel to change process'
11089                  * page table to point to a different piece of kernel memory,
11090                  * but from userspace point of view memory address (and its
11091                  * contents, being identical at this point) will stay the
11092                  * same. This mapping will be released by bpf_object__close()
11093                  * as per normal clean up procedure, so we don't need to worry
11094                  * about it from skeleton's clean up perspective.
11095                  */
11096                 *mmaped = mmap(map->mmaped, mmap_sz, prot,
11097                                 MAP_SHARED | MAP_FIXED, map_fd, 0);
11098                 if (*mmaped == MAP_FAILED) {
11099                         err = -errno;
11100                         *mmaped = NULL;
11101                         pr_warn("failed to re-mmap() map '%s': %d\n",
11102                                  bpf_map__name(map), err);
11103                         return err;
11104                 }
11105         }
11106
11107         return 0;
11108 }
11109
11110 int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
11111 {
11112         int i;
11113
11114         for (i = 0; i < s->prog_cnt; i++) {
11115                 struct bpf_program *prog = *s->progs[i].prog;
11116                 struct bpf_link **link = s->progs[i].link;
11117                 const struct bpf_sec_def *sec_def;
11118
11119                 if (!prog->load)
11120                         continue;
11121
11122                 sec_def = find_sec_def(prog->sec_name);
11123                 if (!sec_def || !sec_def->attach_fn)
11124                         continue;
11125
11126                 *link = sec_def->attach_fn(sec_def, prog);
11127                 if (IS_ERR(*link)) {
11128                         pr_warn("failed to auto-attach program '%s': %ld\n",
11129                                 bpf_program__name(prog), PTR_ERR(*link));
11130                         return PTR_ERR(*link);
11131                 }
11132         }
11133
11134         return 0;
11135 }
11136
11137 void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
11138 {
11139         int i;
11140
11141         for (i = 0; i < s->prog_cnt; i++) {
11142                 struct bpf_link **link = s->progs[i].link;
11143
11144                 bpf_link__destroy(*link);
11145                 *link = NULL;
11146         }
11147 }
11148
11149 void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
11150 {
11151         if (s->progs)
11152                 bpf_object__detach_skeleton(s);
11153         if (s->obj)
11154                 bpf_object__close(*s->obj);
11155         free(s->maps);
11156         free(s->progs);
11157         free(s);
11158 }