tools/lib/bpf/linker.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2 /*
   3  * BPF static linker
   4  *
   5  * Copyright (c) 2021 Facebook
   6  */
   7 #include <stdbool.h>
   8 #include <stddef.h>
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <unistd.h>
  13 #include <errno.h>
  14 #include <linux/err.h>
  15 #include <linux/btf.h>
  16 #include <elf.h>
  17 #include <libelf.h>
  18 #include <gelf.h>
  19 #include <fcntl.h>
  20 #include "libbpf.h"
  21 #include "btf.h"
  22 #include "libbpf_internal.h"
  23 #include "strset.h"
  24
  25 struct src_sec {
  26         const char *sec_name;
  27         /* positional (not necessarily ELF) index in an array of sections */
  28         int id;
  29         /* positional (not necessarily ELF) index of a matching section in a final object file */
  30         int dst_id;
  31         /* section data offset in a matching output section */
  32         int dst_off;
  33         /* whether section is omitted from the final ELF file */
  34         bool skipped;
  35         /* whether section is an ephemeral section, not mapped to an ELF section */
  36         bool ephemeral;
  37
  38         /* ELF info */
  39         size_t sec_idx;
  40         Elf_Scn *scn;
  41         Elf64_Shdr *shdr;
  42         Elf_Data *data;
  43
  44         /* corresponding BTF DATASEC type ID */
  45         int sec_type_id;
  46 };
  47
  48 struct src_obj {
  49         const char *filename;
  50         int fd;
  51         Elf *elf;
  52         /* Section header strings section index */
  53         size_t shstrs_sec_idx;
  54         /* SYMTAB section index */
  55         size_t symtab_sec_idx;
  56
  57         struct btf *btf;
  58         struct btf_ext *btf_ext;
  59
  60         /* List of sections (including ephemeral). Slot zero is unused. */
  61         struct src_sec *secs;
  62         int sec_cnt;
  63
  64         /* mapping of symbol indices from src to dst ELF */
  65         int *sym_map;
  66         /* mapping from the src BTF type IDs to dst ones */
  67         int *btf_type_map;
  68 };
  69
  70 /* single .BTF.ext data section */
  71 struct btf_ext_sec_data {
  72         size_t rec_cnt;
  73         __u32 rec_sz;
  74         void *recs;
  75 };
  76
  77 struct dst_sec {
  78         char *sec_name;
  79         /* positional (not necessarily ELF) index in an array of sections */
  80         int id;
  81
  82         /* ELF info */
  83         size_t sec_idx;
  84         Elf_Scn *scn;
  85         Elf64_Shdr *shdr;
  86         Elf_Data *data;
  87
  88         /* final output section size */
  89         int sec_sz;
  90         /* final output contents of the section */
  91         void *raw_data;
  92
  93         /* corresponding STT_SECTION symbol index in SYMTAB */
  94         int sec_sym_idx;
  95
  96         /* section's DATASEC variable info, emitted on BTF finalization */
  97         bool has_btf;
  98         int sec_var_cnt;
  99         struct btf_var_secinfo *sec_vars;
 100
 101         /* section's .BTF.ext data */
 102         struct btf_ext_sec_data func_info;
 103         struct btf_ext_sec_data line_info;
 104         struct btf_ext_sec_data core_relo_info;
 105 };
 106
 107 struct bpf_linker {
 108         char *filename;
 109         int fd;
 110         Elf *elf;
 111         Elf64_Ehdr *elf_hdr;
 112
 113         /* Output sections metadata */
 114         struct dst_sec *secs;
 115         int sec_cnt;
 116
 117         struct strset *strtab_strs; /* STRTAB unique strings */
 118         size_t strtab_sec_idx; /* STRTAB section index */
 119         size_t symtab_sec_idx; /* SYMTAB section index */
 120
 121         struct btf *btf;
 122         struct btf_ext *btf_ext;
 123 };
 124
 125 #define pr_warn_elf(fmt, ...)                                                                   \
 126         libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
 127
 128 static int init_output_elf(struct bpf_linker *linker, const char *file);
 129
 130 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj);
 131 static int linker_sanity_check_elf(struct src_obj *obj);
 132 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
 133 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
 134 static int linker_sanity_check_btf(struct src_obj *obj);
 135 static int linker_sanity_check_btf_ext(struct src_obj *obj);
 136 static int linker_fixup_btf(struct src_obj *obj);
 137 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
 138 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
 139 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
 140 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
 141 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
 142
 143 static int finalize_btf(struct bpf_linker *linker);
 144 static int finalize_btf_ext(struct bpf_linker *linker);
 145
 146 void bpf_linker__free(struct bpf_linker *linker)
 147 {
 148         int i;
 149
 150         if (!linker)
 151                 return;
 152
 153         free(linker->filename);
 154
 155         if (linker->elf)
 156                 elf_end(linker->elf);
 157
 158         if (linker->fd >= 0)
 159                 close(linker->fd);
 160
 161         strset__free(linker->strtab_strs);
 162
 163         btf__free(linker->btf);
 164         btf_ext__free(linker->btf_ext);
 165
 166         for (i = 1; i < linker->sec_cnt; i++) {
 167                 struct dst_sec *sec = &linker->secs[i];
 168
 169                 free(sec->sec_name);
 170                 free(sec->raw_data);
 171                 free(sec->sec_vars);
 172
 173                 free(sec->func_info.recs);
 174                 free(sec->line_info.recs);
 175                 free(sec->core_relo_info.recs);
 176         }
 177         free(linker->secs);
 178
 179         free(linker);
 180 }
 181
 182 struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
 183 {
 184         struct bpf_linker *linker;
 185         int err;
 186
 187         if (!OPTS_VALID(opts, bpf_linker_opts))
 188                 return NULL;
 189
 190         if (elf_version(EV_CURRENT) == EV_NONE) {
 191                 pr_warn_elf("libelf initialization failed");
 192                 return NULL;
 193         }
 194
 195         linker = calloc(1, sizeof(*linker));
 196         if (!linker)
 197                 return NULL;
 198
 199         linker->fd = -1;
 200
 201         err = init_output_elf(linker, filename);
 202         if (err)
 203                 goto err_out;
 204
 205         return linker;
 206
 207 err_out:
 208         bpf_linker__free(linker);
 209         return NULL;
 210 }
 211
 212 static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
 213 {
 214         struct dst_sec *secs = linker->secs, *sec;
 215         size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
 216
 217         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 218         if (!secs)
 219                 return NULL;
 220
 221         /* zero out newly allocated memory */
 222         memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
 223
 224         linker->secs = secs;
 225         linker->sec_cnt = new_cnt;
 226
 227         sec = &linker->secs[new_cnt - 1];
 228         sec->id = new_cnt - 1;
 229         sec->sec_name = strdup(sec_name);
 230         if (!sec->sec_name)
 231                 return NULL;
 232
 233         return sec;
 234 }
 235
 236 static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
 237 {
 238         struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
 239         Elf64_Sym *syms, *sym;
 240         size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
 241
 242         syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
 243         if (!syms)
 244                 return NULL;
 245
 246         sym = &syms[sym_cnt];
 247         memset(sym, 0, sizeof(*sym));
 248
 249         symtab->raw_data = syms;
 250         symtab->sec_sz += sizeof(*sym);
 251         symtab->shdr->sh_size += sizeof(*sym);
 252         symtab->data->d_size += sizeof(*sym);
 253
 254         if (sym_idx)
 255                 *sym_idx = sym_cnt;
 256
 257         return sym;
 258 }
 259
 260 static int init_output_elf(struct bpf_linker *linker, const char *file)
 261 {
 262         int err, str_off;
 263         Elf64_Sym *init_sym;
 264         struct dst_sec *sec;
 265
 266         linker->filename = strdup(file);
 267         if (!linker->filename)
 268                 return -ENOMEM;
 269
 270         linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
 271         if (linker->fd < 0) {
 272                 err = -errno;
 273                 pr_warn("failed to create '%s': %d\n", file, err);
 274                 return err;
 275         }
 276
 277         linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
 278         if (!linker->elf) {
 279                 pr_warn_elf("failed to create ELF object");
 280                 return -EINVAL;
 281         }
 282
 283         /* ELF header */
 284         linker->elf_hdr = elf64_newehdr(linker->elf);
 285         if (!linker->elf_hdr) {
 286                 pr_warn_elf("failed to create ELF header");
 287                 return -EINVAL;
 288         }
 289
 290         linker->elf_hdr->e_machine = EM_BPF;
 291         linker->elf_hdr->e_type = ET_REL;
 292 #if __BYTE_ORDER == __LITTLE_ENDIAN
 293         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
 294 #elif __BYTE_ORDER == __BIG_ENDIAN
 295         linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
 296 #else
 297 #error "Unknown __BYTE_ORDER"
 298 #endif
 299
 300         /* STRTAB */
 301         /* initialize strset with an empty string to conform to ELF */
 302         linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
 303         if (libbpf_get_error(linker->strtab_strs))
 304                 return libbpf_get_error(linker->strtab_strs);
 305
 306         sec = add_dst_sec(linker, ".strtab");
 307         if (!sec)
 308                 return -ENOMEM;
 309
 310         sec->scn = elf_newscn(linker->elf);
 311         if (!sec->scn) {
 312                 pr_warn_elf("failed to create STRTAB section");
 313                 return -EINVAL;
 314         }
 315
 316         sec->shdr = elf64_getshdr(sec->scn);
 317         if (!sec->shdr)
 318                 return -EINVAL;
 319
 320         sec->data = elf_newdata(sec->scn);
 321         if (!sec->data) {
 322                 pr_warn_elf("failed to create STRTAB data");
 323                 return -EINVAL;
 324         }
 325
 326         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 327         if (str_off < 0)
 328                 return str_off;
 329
 330         sec->sec_idx = elf_ndxscn(sec->scn);
 331         linker->elf_hdr->e_shstrndx = sec->sec_idx;
 332         linker->strtab_sec_idx = sec->sec_idx;
 333
 334         sec->shdr->sh_name = str_off;
 335         sec->shdr->sh_type = SHT_STRTAB;
 336         sec->shdr->sh_flags = SHF_STRINGS;
 337         sec->shdr->sh_offset = 0;
 338         sec->shdr->sh_link = 0;
 339         sec->shdr->sh_info = 0;
 340         sec->shdr->sh_addralign = 1;
 341         sec->shdr->sh_size = sec->sec_sz = 0;
 342         sec->shdr->sh_entsize = 0;
 343
 344         /* SYMTAB */
 345         sec = add_dst_sec(linker, ".symtab");
 346         if (!sec)
 347                 return -ENOMEM;
 348
 349         sec->scn = elf_newscn(linker->elf);
 350         if (!sec->scn) {
 351                 pr_warn_elf("failed to create SYMTAB section");
 352                 return -EINVAL;
 353         }
 354
 355         sec->shdr = elf64_getshdr(sec->scn);
 356         if (!sec->shdr)
 357                 return -EINVAL;
 358
 359         sec->data = elf_newdata(sec->scn);
 360         if (!sec->data) {
 361                 pr_warn_elf("failed to create SYMTAB data");
 362                 return -EINVAL;
 363         }
 364
 365         str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
 366         if (str_off < 0)
 367                 return str_off;
 368
 369         sec->sec_idx = elf_ndxscn(sec->scn);
 370         linker->symtab_sec_idx = sec->sec_idx;
 371
 372         sec->shdr->sh_name = str_off;
 373         sec->shdr->sh_type = SHT_SYMTAB;
 374         sec->shdr->sh_flags = 0;
 375         sec->shdr->sh_offset = 0;
 376         sec->shdr->sh_link = linker->strtab_sec_idx;
 377         /* sh_info should be one greater than the index of the last local
 378          * symbol (i.e., binding is STB_LOCAL). But why and who cares?
 379          */
 380         sec->shdr->sh_info = 0;
 381         sec->shdr->sh_addralign = 8;
 382         sec->shdr->sh_entsize = sizeof(Elf64_Sym);
 383
 384         /* .BTF */
 385         linker->btf = btf__new_empty();
 386         err = libbpf_get_error(linker->btf);
 387         if (err)
 388                 return err;
 389
 390         /* add the special all-zero symbol */
 391         init_sym = add_new_sym(linker, NULL);
 392         if (!init_sym)
 393                 return -EINVAL;
 394
 395         init_sym->st_name = 0;
 396         init_sym->st_info = 0;
 397         init_sym->st_other = 0;
 398         init_sym->st_shndx = SHN_UNDEF;
 399         init_sym->st_value = 0;
 400         init_sym->st_size = 0;
 401
 402         return 0;
 403 }
 404
 405 int bpf_linker__add_file(struct bpf_linker *linker, const char *filename)
 406 {
 407         struct src_obj obj = {};
 408         int err = 0;
 409
 410         if (!linker->elf)
 411                 return -EINVAL;
 412
 413         err = err ?: linker_load_obj_file(linker, filename, &obj);
 414         err = err ?: linker_append_sec_data(linker, &obj);
 415         err = err ?: linker_append_elf_syms(linker, &obj);
 416         err = err ?: linker_append_elf_relos(linker, &obj);
 417         err = err ?: linker_append_btf(linker, &obj);
 418         err = err ?: linker_append_btf_ext(linker, &obj);
 419
 420         /* free up src_obj resources */
 421         free(obj.btf_type_map);
 422         btf__free(obj.btf);
 423         btf_ext__free(obj.btf_ext);
 424         free(obj.secs);
 425         free(obj.sym_map);
 426         if (obj.elf)
 427                 elf_end(obj.elf);
 428         if (obj.fd >= 0)
 429                 close(obj.fd);
 430
 431         return err;
 432 }
 433
 434 static bool is_dwarf_sec_name(const char *name)
 435 {
 436         /* approximation, but the actual list is too long */
 437         return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
 438 }
 439
 440 static bool is_ignored_sec(struct src_sec *sec)
 441 {
 442         Elf64_Shdr *shdr = sec->shdr;
 443         const char *name = sec->sec_name;
 444
 445         /* no special handling of .strtab */
 446         if (shdr->sh_type == SHT_STRTAB)
 447                 return true;
 448
 449         /* ignore .llvm_addrsig section as well */
 450         if (shdr->sh_type == SHT_LLVM_ADDRSIG)
 451                 return true;
 452
 453         /* no subprograms will lead to an empty .text section, ignore it */
 454         if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
 455             strcmp(sec->sec_name, ".text") == 0)
 456                 return true;
 457
 458         /* DWARF sections */
 459         if (is_dwarf_sec_name(sec->sec_name))
 460                 return true;
 461
 462         if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
 463                 name += sizeof(".rel") - 1;
 464                 /* DWARF section relocations */
 465                 if (is_dwarf_sec_name(name))
 466                         return true;
 467
 468                 /* .BTF and .BTF.ext don't need relocations */
 469                 if (strcmp(name, BTF_ELF_SEC) == 0 ||
 470                     strcmp(name, BTF_EXT_ELF_SEC) == 0)
 471                         return true;
 472         }
 473
 474         return false;
 475 }
 476
 477 static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
 478 {
 479         struct src_sec *secs = obj->secs, *sec;
 480         size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
 481
 482         secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
 483         if (!secs)
 484                 return NULL;
 485
 486         /* zero out newly allocated memory */
 487         memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
 488
 489         obj->secs = secs;
 490         obj->sec_cnt = new_cnt;
 491
 492         sec = &obj->secs[new_cnt - 1];
 493         sec->id = new_cnt - 1;
 494         sec->sec_name = sec_name;
 495
 496         return sec;
 497 }
 498
 499 static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, struct src_obj *obj)
 500 {
 501 #if __BYTE_ORDER == __LITTLE_ENDIAN
 502         const int host_endianness = ELFDATA2LSB;
 503 #elif __BYTE_ORDER == __BIG_ENDIAN
 504         const int host_endianness = ELFDATA2MSB;
 505 #else
 506 #error "Unknown __BYTE_ORDER"
 507 #endif
 508         int err = 0;
 509         Elf_Scn *scn;
 510         Elf_Data *data;
 511         Elf64_Ehdr *ehdr;
 512         Elf64_Shdr *shdr;
 513         struct src_sec *sec;
 514
 515         pr_debug("linker: adding object file '%s'...\n", filename);
 516
 517         obj->filename = filename;
 518
 519         obj->fd = open(filename, O_RDONLY);
 520         if (obj->fd < 0) {
 521                 err = -errno;
 522                 pr_warn("failed to open file '%s': %d\n", filename, err);
 523                 return err;
 524         }
 525         obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
 526         if (!obj->elf) {
 527                 err = -errno;
 528                 pr_warn_elf("failed to parse ELF file '%s'", filename);
 529                 return err;
 530         }
 531
 532         /* Sanity check ELF file high-level properties */
 533         ehdr = elf64_getehdr(obj->elf);
 534         if (!ehdr) {
 535                 err = -errno;
 536                 pr_warn_elf("failed to get ELF header for %s", filename);
 537                 return err;
 538         }
 539         if (ehdr->e_ident[EI_DATA] != host_endianness) {
 540                 err = -EOPNOTSUPP;
 541                 pr_warn_elf("unsupported byte order of ELF file %s", filename);
 542                 return err;
 543         }
 544         if (ehdr->e_type != ET_REL
 545             || ehdr->e_machine != EM_BPF
 546             || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
 547                 err = -EOPNOTSUPP;
 548                 pr_warn_elf("unsupported kind of ELF file %s", filename);
 549                 return err;
 550         }
 551
 552         if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
 553                 err = -errno;
 554                 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
 555                 return err;
 556         }
 557
 558         scn = NULL;
 559         while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
 560                 size_t sec_idx = elf_ndxscn(scn);
 561                 const char *sec_name;
 562
 563                 shdr = elf64_getshdr(scn);
 564                 if (!shdr) {
 565                         err = -errno;
 566                         pr_warn_elf("failed to get section #%zu header for %s",
 567                                     sec_idx, filename);
 568                         return err;
 569                 }
 570
 571                 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
 572                 if (!sec_name) {
 573                         err = -errno;
 574                         pr_warn_elf("failed to get section #%zu name for %s",
 575                                     sec_idx, filename);
 576                         return err;
 577                 }
 578
 579                 data = elf_getdata(scn, 0);
 580                 if (!data) {
 581                         err = -errno;
 582                         pr_warn_elf("failed to get section #%zu (%s) data from %s",
 583                                     sec_idx, sec_name, filename);
 584                         return err;
 585                 }
 586
 587                 sec = add_src_sec(obj, sec_name);
 588                 if (!sec)
 589                         return -ENOMEM;
 590
 591                 sec->scn = scn;
 592                 sec->shdr = shdr;
 593                 sec->data = data;
 594                 sec->sec_idx = elf_ndxscn(scn);
 595
 596                 if (is_ignored_sec(sec)) {
 597                         sec->skipped = true;
 598                         continue;
 599                 }
 600
 601                 switch (shdr->sh_type) {
 602                 case SHT_SYMTAB:
 603                         if (obj->symtab_sec_idx) {
 604                                 err = -EOPNOTSUPP;
 605                                 pr_warn("multiple SYMTAB sections found, not supported\n");
 606                                 return err;
 607                         }
 608                         obj->symtab_sec_idx = sec_idx;
 609                         break;
 610                 case SHT_STRTAB:
 611                         /* we'll construct our own string table */
 612                         break;
 613                 case SHT_PROGBITS:
 614                         if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
 615                                 obj->btf = btf__new(data->d_buf, shdr->sh_size);
 616                                 err = libbpf_get_error(obj->btf);
 617                                 if (err) {
 618                                         pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
 619                                         return err;
 620                                 }
 621                                 sec->skipped = true;
 622                                 continue;
 623                         }
 624                         if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
 625                                 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
 626                                 err = libbpf_get_error(obj->btf_ext);
 627                                 if (err) {
 628                                         pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
 629                                         return err;
 630                                 }
 631                                 sec->skipped = true;
 632                                 continue;
 633                         }
 634
 635                         /* data & code */
 636                         break;
 637                 case SHT_NOBITS:
 638                         /* BSS */
 639                         break;
 640                 case SHT_REL:
 641                         /* relocations */
 642                         break;
 643                 default:
 644                         pr_warn("unrecognized section #%zu (%s) in %s\n",
 645                                 sec_idx, sec_name, filename);
 646                         err = -EINVAL;
 647                         return err;
 648                 }
 649         }
 650
 651         err = err ?: linker_sanity_check_elf(obj);
 652         err = err ?: linker_sanity_check_btf(obj);
 653         err = err ?: linker_sanity_check_btf_ext(obj);
 654         err = err ?: linker_fixup_btf(obj);
 655
 656         return err;
 657 }
 658
 659 static bool is_pow_of_2(size_t x)
 660 {
 661         return x && (x & (x - 1)) == 0;
 662 }
 663
 664 static int linker_sanity_check_elf(struct src_obj *obj)
 665 {
 666         struct src_sec *sec;
 667         int i, err;
 668
 669         if (!obj->symtab_sec_idx) {
 670                 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
 671                 return -EINVAL;
 672         }
 673         if (!obj->shstrs_sec_idx) {
 674                 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
 675                 return -EINVAL;
 676         }
 677
 678         for (i = 1; i < obj->sec_cnt; i++) {
 679                 sec = &obj->secs[i];
 680
 681                 if (sec->sec_name[0] == '\0') {
 682                         pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
 683                         return -EINVAL;
 684                 }
 685
 686                 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
 687                         return -EINVAL;
 688                 if (sec->shdr->sh_addralign != sec->data->d_align)
 689                         return -EINVAL;
 690
 691                 if (sec->shdr->sh_size != sec->data->d_size)
 692                         return -EINVAL;
 693
 694                 switch (sec->shdr->sh_type) {
 695                 case SHT_SYMTAB:
 696                         err = linker_sanity_check_elf_symtab(obj, sec);
 697                         if (err)
 698                                 return err;
 699                         break;
 700                 case SHT_STRTAB:
 701                         break;
 702                 case SHT_PROGBITS:
 703                         if (sec->shdr->sh_flags & SHF_EXECINSTR) {
 704                                 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
 705                                         return -EINVAL;
 706                         }
 707                         break;
 708                 case SHT_NOBITS:
 709                         break;
 710                 case SHT_REL:
 711                         err = linker_sanity_check_elf_relos(obj, sec);
 712                         if (err)
 713                                 return err;
 714                         break;
 715                 case SHT_LLVM_ADDRSIG:
 716                         break;
 717                 default:
 718                         pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
 719                                 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
 720                         return -EINVAL;
 721                 }
 722         }
 723
 724         return 0;
 725 }
 726
 727 static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
 728 {
 729         struct src_sec *link_sec;
 730         Elf64_Sym *sym;
 731         int i, n;
 732
 733         if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
 734                 return -EINVAL;
 735         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 736                 return -EINVAL;
 737
 738         if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
 739                 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
 740                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 741                 return -EINVAL;
 742         }
 743         link_sec = &obj->secs[sec->shdr->sh_link];
 744         if (link_sec->shdr->sh_type != SHT_STRTAB) {
 745                 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
 746                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 747                 return -EINVAL;
 748         }
 749
 750         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 751         sym = sec->data->d_buf;
 752         for (i = 0; i < n; i++, sym++) {
 753                 int sym_type = ELF64_ST_TYPE(sym->st_info);
 754                 int sym_bind = ELF64_ST_BIND(sym->st_info);
 755                 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
 756
 757                 if (i == 0) {
 758                         if (sym->st_name != 0 || sym->st_info != 0
 759                             || sym->st_other != 0 || sym->st_shndx != 0
 760                             || sym->st_value != 0 || sym->st_size != 0) {
 761                                 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
 762                                 return -EINVAL;
 763                         }
 764                         continue;
 765                 }
 766                 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
 767                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
 768                                 i, sec->sec_idx, sym_bind);
 769                         return -EINVAL;
 770                 }
 771                 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
 772                         pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
 773                                 i, sec->sec_idx, sym_vis);
 774                         return -EINVAL;
 775                 }
 776                 if (sym->st_shndx == 0) {
 777                         if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
 778                             || sym->st_value != 0 || sym->st_size != 0) {
 779                                 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
 780                                         i, obj->filename);
 781
 782                                 return -EINVAL;
 783                         }
 784                         continue;
 785                 }
 786                 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
 787                         pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
 788                                 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
 789                         return -EINVAL;
 790                 }
 791                 if (sym_type == STT_SECTION) {
 792                         if (sym->st_value != 0)
 793                                 return -EINVAL;
 794                         continue;
 795                 }
 796         }
 797
 798         return 0;
 799 }
 800
 801 static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
 802 {
 803         struct src_sec *link_sec, *sym_sec;
 804         Elf64_Rel *relo;
 805         int i, n;
 806
 807         if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
 808                 return -EINVAL;
 809         if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
 810                 return -EINVAL;
 811
 812         /* SHT_REL's sh_link should point to SYMTAB */
 813         if (sec->shdr->sh_link != obj->symtab_sec_idx) {
 814                 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
 815                         sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
 816                 return -EINVAL;
 817         }
 818
 819         /* SHT_REL's sh_info points to relocated section */
 820         if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
 821                 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
 822                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 823                 return -EINVAL;
 824         }
 825         link_sec = &obj->secs[sec->shdr->sh_info];
 826
 827         /* .rel<secname> -> <secname> pattern is followed */
 828         if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
 829             || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
 830                 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
 831                         sec->sec_idx, obj->filename);
 832                 return -EINVAL;
 833         }
 834
 835         /* don't further validate relocations for ignored sections */
 836         if (link_sec->skipped)
 837                 return 0;
 838
 839         /* relocatable section is data or instructions */
 840         if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
 841                 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
 842                         sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
 843                 return -EINVAL;
 844         }
 845
 846         /* check sanity of each relocation */
 847         n = sec->shdr->sh_size / sec->shdr->sh_entsize;
 848         relo = sec->data->d_buf;
 849         sym_sec = &obj->secs[obj->symtab_sec_idx];
 850         for (i = 0; i < n; i++, relo++) {
 851                 size_t sym_idx = ELF64_R_SYM(relo->r_info);
 852                 size_t sym_type = ELF64_R_TYPE(relo->r_info);
 853
 854                 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32) {
 855                         pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
 856                                 i, sec->sec_idx, sym_type, obj->filename);
 857                         return -EINVAL;
 858                 }
 859
 860                 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
 861                         pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
 862                                 i, sec->sec_idx, sym_idx, obj->filename);
 863                         return -EINVAL;
 864                 }
 865
 866                 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
 867                         if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
 868                                 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
 869                                         i, sec->sec_idx, sym_idx, obj->filename);
 870                                 return -EINVAL;
 871                         }
 872                 }
 873         }
 874
 875         return 0;
 876 }
 877
 878 static int check_btf_type_id(__u32 *type_id, void *ctx)
 879 {
 880         struct btf *btf = ctx;
 881
 882         if (*type_id > btf__get_nr_types(btf))
 883                 return -EINVAL;
 884
 885         return 0;
 886 }
 887
 888 static int check_btf_str_off(__u32 *str_off, void *ctx)
 889 {
 890         struct btf *btf = ctx;
 891         const char *s;
 892
 893         s = btf__str_by_offset(btf, *str_off);
 894
 895         if (!s)
 896                 return -EINVAL;
 897
 898         return 0;
 899 }
 900
 901 static int linker_sanity_check_btf(struct src_obj *obj)
 902 {
 903         struct btf_type *t;
 904         int i, n, err = 0;
 905
 906         if (!obj->btf)
 907                 return 0;
 908
 909         n = btf__get_nr_types(obj->btf);
 910         for (i = 1; i <= n; i++) {
 911                 t = btf_type_by_id(obj->btf, i);
 912
 913                 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
 914                 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
 915                 if (err)
 916                         return err;
 917         }
 918
 919         return 0;
 920 }
 921
 922 static int linker_sanity_check_btf_ext(struct src_obj *obj)
 923 {
 924         int err = 0;
 925
 926         if (!obj->btf_ext)
 927                 return 0;
 928
 929         /* can't use .BTF.ext without .BTF */
 930         if (!obj->btf)
 931                 return -EINVAL;
 932
 933         err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
 934         err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
 935         if (err)
 936                 return err;
 937
 938         return 0;
 939 }
 940
 941 static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
 942 {
 943         Elf_Scn *scn;
 944         Elf_Data *data;
 945         Elf64_Shdr *shdr;
 946         int name_off;
 947
 948         dst_sec->sec_sz = 0;
 949         dst_sec->sec_idx = 0;
 950
 951         /* ephemeral sections are just thin section shells lacking most parts */
 952         if (src_sec->ephemeral)
 953                 return 0;
 954
 955         scn = elf_newscn(linker->elf);
 956         if (!scn)
 957                 return -ENOMEM;
 958         data = elf_newdata(scn);
 959         if (!data)
 960                 return -ENOMEM;
 961         shdr = elf64_getshdr(scn);
 962         if (!shdr)
 963                 return -ENOMEM;
 964
 965         dst_sec->scn = scn;
 966         dst_sec->shdr = shdr;
 967         dst_sec->data = data;
 968         dst_sec->sec_idx = elf_ndxscn(scn);
 969
 970         name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
 971         if (name_off < 0)
 972                 return name_off;
 973
 974         shdr->sh_name = name_off;
 975         shdr->sh_type = src_sec->shdr->sh_type;
 976         shdr->sh_flags = src_sec->shdr->sh_flags;
 977         shdr->sh_size = 0;
 978         /* sh_link and sh_info have different meaning for different types of
 979          * sections, so we leave it up to the caller code to fill them in, if
 980          * necessary
 981          */
 982         shdr->sh_link = 0;
 983         shdr->sh_info = 0;
 984         shdr->sh_addralign = src_sec->shdr->sh_addralign;
 985         shdr->sh_entsize = src_sec->shdr->sh_entsize;
 986
 987         data->d_type = src_sec->data->d_type;
 988         data->d_size = 0;
 989         data->d_buf = NULL;
 990         data->d_align = src_sec->data->d_align;
 991         data->d_off = 0;
 992
 993         return 0;
 994 }
 995
 996 static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
 997 {
 998         struct dst_sec *sec;
 999         int i;
1000
1001         for (i = 1; i < linker->sec_cnt; i++) {
1002                 sec = &linker->secs[i];
1003
1004                 if (strcmp(sec->sec_name, sec_name) == 0)
1005                         return sec;
1006         }
1007
1008         return NULL;
1009 }
1010
1011 static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1012 {
1013         if (dst->shdr->sh_type != src->shdr->sh_type) {
1014                 pr_warn("sec %s types mismatch\n", dst->sec_name);
1015                 return false;
1016         }
1017         if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1018                 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1019                 return false;
1020         }
1021         if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1022                 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1023                 return false;
1024         }
1025
1026         return true;
1027 }
1028
1029 static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1030 {
1031         if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1032                 return false;
1033         if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1034                 return false;
1035         return true;
1036 }
1037
1038 static int extend_sec(struct dst_sec *dst, struct src_sec *src)
1039 {
1040         void *tmp;
1041         size_t dst_align = dst->shdr->sh_addralign;
1042         size_t src_align = src->shdr->sh_addralign;
1043         size_t dst_align_sz, dst_final_sz;
1044
1045         if (dst_align == 0)
1046                 dst_align = 1;
1047         if (dst_align < src_align)
1048                 dst_align = src_align;
1049
1050         dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1051
1052         /* no need to re-align final size */
1053         dst_final_sz = dst_align_sz + src->shdr->sh_size;
1054
1055         if (src->shdr->sh_type != SHT_NOBITS) {
1056                 tmp = realloc(dst->raw_data, dst_final_sz);
1057                 if (!tmp)
1058                         return -ENOMEM;
1059                 dst->raw_data = tmp;
1060
1061                 /* pad dst section, if it's alignment forced size increase */
1062                 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1063                 /* now copy src data at a properly aligned offset */
1064                 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1065         }
1066
1067         dst->sec_sz = dst_final_sz;
1068         dst->shdr->sh_size = dst_final_sz;
1069         dst->data->d_size = dst_final_sz;
1070
1071         dst->shdr->sh_addralign = dst_align;
1072         dst->data->d_align = dst_align;
1073
1074         src->dst_off = dst_align_sz;
1075
1076         return 0;
1077 }
1078
1079 static bool is_data_sec(struct src_sec *sec)
1080 {
1081         if (!sec || sec->skipped)
1082                 return false;
1083         /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */
1084         if (sec->ephemeral)
1085                 return true;
1086         return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1087 }
1088
1089 static bool is_relo_sec(struct src_sec *sec)
1090 {
1091         if (!sec || sec->skipped || sec->ephemeral)
1092                 return false;
1093         return sec->shdr->sh_type == SHT_REL;
1094 }
1095
1096 static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1097 {
1098         int i, err;
1099
1100         for (i = 1; i < obj->sec_cnt; i++) {
1101                 struct src_sec *src_sec;
1102                 struct dst_sec *dst_sec;
1103
1104                 src_sec = &obj->secs[i];
1105                 if (!is_data_sec(src_sec))
1106                         continue;
1107
1108                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1109                 if (!dst_sec) {
1110                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1111                         if (!dst_sec)
1112                                 return -ENOMEM;
1113                         err = init_sec(linker, dst_sec, src_sec);
1114                         if (err) {
1115                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1116                                 return err;
1117                         }
1118                 } else {
1119                         if (!secs_match(dst_sec, src_sec)) {
1120                                 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1121                                 return -1;
1122                         }
1123
1124                         /* "license" and "version" sections are deduped */
1125                         if (strcmp(src_sec->sec_name, "license") == 0
1126                             || strcmp(src_sec->sec_name, "version") == 0) {
1127                                 if (!sec_content_is_same(dst_sec, src_sec)) {
1128                                         pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1129                                         return -EINVAL;
1130                                 }
1131                                 src_sec->skipped = true;
1132                                 src_sec->dst_id = dst_sec->id;
1133                                 continue;
1134                         }
1135                 }
1136
1137                 /* record mapped section index */
1138                 src_sec->dst_id = dst_sec->id;
1139
1140                 if (src_sec->ephemeral)
1141                         continue;
1142
1143                 err = extend_sec(dst_sec, src_sec);
1144                 if (err)
1145                         return err;
1146         }
1147
1148         return 0;
1149 }
1150
1151 static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1152 {
1153         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1154         Elf64_Sym *sym = symtab->data->d_buf, *dst_sym;
1155         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
1156         int str_sec_idx = symtab->shdr->sh_link;
1157
1158         obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1159         if (!obj->sym_map)
1160                 return -ENOMEM;
1161
1162         for (i = 0; i < n; i++, sym++) {
1163                 struct src_sec *src_sec = NULL;
1164                 struct dst_sec *dst_sec = NULL;
1165                 const char *sym_name;
1166                 size_t dst_sym_idx;
1167                 int name_off;
1168
1169                 /* We already validated all-zero symbol #0 and we already
1170                  * appended it preventively to the final SYMTAB, so skip it.
1171                  */
1172                 if (i == 0)
1173                         continue;
1174
1175                 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1176                 if (!sym_name) {
1177                         pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1178                         return -EINVAL;
1179                 }
1180
1181                 if (sym->st_shndx && sym->st_shndx < SHN_LORESERVE) {
1182                         src_sec = &obj->secs[sym->st_shndx];
1183                         if (src_sec->skipped)
1184                                 continue;
1185                         dst_sec = &linker->secs[src_sec->dst_id];
1186
1187                         /* allow only one STT_SECTION symbol per section */
1188                         if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sec->sec_sym_idx) {
1189                                 obj->sym_map[i] = dst_sec->sec_sym_idx;
1190                                 continue;
1191                         }
1192                 }
1193
1194                 name_off = strset__add_str(linker->strtab_strs, sym_name);
1195                 if (name_off < 0)
1196                         return name_off;
1197
1198                 dst_sym = add_new_sym(linker, &dst_sym_idx);
1199                 if (!dst_sym)
1200                         return -ENOMEM;
1201
1202                 dst_sym->st_name = name_off;
1203                 dst_sym->st_info = sym->st_info;
1204                 dst_sym->st_other = sym->st_other;
1205                 dst_sym->st_shndx = src_sec ? dst_sec->sec_idx : sym->st_shndx;
1206                 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1207                 dst_sym->st_size = sym->st_size;
1208
1209                 obj->sym_map[i] = dst_sym_idx;
1210
1211                 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && dst_sym) {
1212                         dst_sec->sec_sym_idx = dst_sym_idx;
1213                         dst_sym->st_value = 0;
1214                 }
1215
1216         }
1217
1218         return 0;
1219 }
1220
1221 static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
1222 {
1223         struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
1224         struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx];
1225         int i, err;
1226
1227         for (i = 1; i < obj->sec_cnt; i++) {
1228                 struct src_sec *src_sec, *src_linked_sec;
1229                 struct dst_sec *dst_sec, *dst_linked_sec;
1230                 Elf64_Rel *src_rel, *dst_rel;
1231                 int j, n;
1232
1233                 src_sec = &obj->secs[i];
1234                 if (!is_relo_sec(src_sec))
1235                         continue;
1236
1237                 /* shdr->sh_info points to relocatable section */
1238                 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
1239                 if (src_linked_sec->skipped)
1240                         continue;
1241
1242                 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1243                 if (!dst_sec) {
1244                         dst_sec = add_dst_sec(linker, src_sec->sec_name);
1245                         if (!dst_sec)
1246                                 return -ENOMEM;
1247                         err = init_sec(linker, dst_sec, src_sec);
1248                         if (err) {
1249                                 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1250                                 return err;
1251                         }
1252                 } else if (!secs_match(dst_sec, src_sec)) {
1253                         pr_warn("sections %s are not compatible\n", src_sec->sec_name);
1254                         return -1;
1255                 }
1256
1257                 /* shdr->sh_link points to SYMTAB */
1258                 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
1259
1260                 /* shdr->sh_info points to relocated section */
1261                 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
1262                 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
1263
1264                 src_sec->dst_id = dst_sec->id;
1265                 err = extend_sec(dst_sec, src_sec);
1266                 if (err)
1267                         return err;
1268
1269                 src_rel = src_sec->data->d_buf;
1270                 dst_rel = dst_sec->raw_data + src_sec->dst_off;
1271                 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
1272                 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
1273                         size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info);
1274                         size_t sym_type = ELF64_R_TYPE(src_rel->r_info);
1275                         Elf64_Sym *src_sym, *dst_sym;
1276                         size_t dst_sym_idx;
1277
1278                         src_sym_idx = ELF64_R_SYM(src_rel->r_info);
1279                         src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
1280
1281                         dst_sym_idx = obj->sym_map[src_sym_idx];
1282                         dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx;
1283                         dst_rel->r_offset += src_linked_sec->dst_off;
1284                         sym_type = ELF64_R_TYPE(src_rel->r_info);
1285                         dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
1286
1287                         if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
1288                                 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
1289                                 struct bpf_insn *insn;
1290
1291                                 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
1292                                         /* calls to the very first static function inside
1293                                          * .text section at offset 0 will
1294                                          * reference section symbol, not the
1295                                          * function symbol. Fix that up,
1296                                          * otherwise it won't be possible to
1297                                          * relocate calls to two different
1298                                          * static functions with the same name
1299                                          * (rom two different object files)
1300                                          */
1301                                         insn = dst_linked_sec->raw_data + dst_rel->r_offset;
1302                                         if (insn->code == (BPF_JMP | BPF_CALL))
1303                                                 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
1304                                         else
1305                                                 insn->imm += sec->dst_off;
1306                                 } else {
1307                                         pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
1308                                         return -EINVAL;
1309                                 }
1310                         }
1311
1312                 }
1313         }
1314
1315         return 0;
1316 }
1317
1318 static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1319 {
1320         struct src_sec *sec;
1321         int i;
1322
1323         for (i = 1; i < obj->sec_cnt; i++) {
1324                 sec = &obj->secs[i];
1325
1326                 if (strcmp(sec->sec_name, sec_name) == 0)
1327                         return sec;
1328         }
1329
1330         return NULL;
1331 }
1332
1333 static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
1334                                    int sym_type, const char *sym_name)
1335 {
1336         struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1337         Elf64_Sym *sym = symtab->data->d_buf;
1338         int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
1339         int str_sec_idx = symtab->shdr->sh_link;
1340         const char *name;
1341
1342         for (i = 0; i < n; i++, sym++) {
1343                 if (sym->st_shndx != sec_idx)
1344                         continue;
1345                 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
1346                         continue;
1347
1348                 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1349                 if (!name)
1350                         return NULL;
1351
1352                 if (strcmp(sym_name, name) != 0)
1353                         continue;
1354
1355                 return sym;
1356         }
1357
1358         return NULL;
1359 }
1360
1361 static int linker_fixup_btf(struct src_obj *obj)
1362 {
1363         const char *sec_name;
1364         struct src_sec *sec;
1365         int i, j, n, m;
1366
1367         if (!obj->btf)
1368                 return 0;
1369
1370         n = btf__get_nr_types(obj->btf);
1371         for (i = 1; i <= n; i++) {
1372                 struct btf_var_secinfo *vi;
1373                 struct btf_type *t;
1374
1375                 t = btf_type_by_id(obj->btf, i);
1376                 if (btf_kind(t) != BTF_KIND_DATASEC)
1377                         continue;
1378
1379                 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1380                 sec = find_src_sec_by_name(obj, sec_name);
1381                 if (sec) {
1382                         /* record actual section size, unless ephemeral */
1383                         if (sec->shdr)
1384                                 t->size = sec->shdr->sh_size;
1385                 } else {
1386                         /* BTF can have some sections that are not represented
1387                          * in ELF, e.g., .kconfig and .ksyms, which are used
1388                          * for special extern variables.  Here we'll
1389                          * pre-create "section shells" for them to be able to
1390                          * keep track of extra per-section metadata later
1391                          * (e.g., BTF variables).
1392                          */
1393                         sec = add_src_sec(obj, sec_name);
1394                         if (!sec)
1395                                 return -ENOMEM;
1396
1397                         sec->ephemeral = true;
1398                         sec->sec_idx = 0; /* will match UNDEF shndx in ELF */
1399                 }
1400
1401                 /* remember ELF section and its BTF type ID match */
1402                 sec->sec_type_id = i;
1403
1404                 /* fix up variable offsets */
1405                 vi = btf_var_secinfos(t);
1406                 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1407                         const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
1408                         const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
1409                         int var_linkage = btf_var(vt)->linkage;
1410                         Elf64_Sym *sym;
1411
1412                         /* no need to patch up static or extern vars */
1413                         if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
1414                                 continue;
1415
1416                         sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
1417                         if (!sym) {
1418                                 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
1419                                 return -ENOENT;
1420                         }
1421
1422                         vi->offset = sym->st_value;
1423                 }
1424         }
1425
1426         return 0;
1427 }
1428
1429 static int remap_type_id(__u32 *type_id, void *ctx)
1430 {
1431         int *id_map = ctx;
1432
1433         *type_id = id_map[*type_id];
1434
1435         return 0;
1436 }
1437
1438 static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
1439 {
1440         const struct btf_type *t;
1441         int i, j, n, start_id, id;
1442
1443         if (!obj->btf)
1444                 return 0;
1445
1446         start_id = btf__get_nr_types(linker->btf) + 1;
1447         n = btf__get_nr_types(obj->btf);
1448
1449         obj->btf_type_map = calloc(n + 1, sizeof(int));
1450         if (!obj->btf_type_map)
1451                 return -ENOMEM;
1452
1453         for (i = 1; i <= n; i++) {
1454                 t = btf__type_by_id(obj->btf, i);
1455
1456                 /* DATASECs are handled specially below */
1457                 if (btf_kind(t) == BTF_KIND_DATASEC)
1458                         continue;
1459
1460                 id = btf__add_type(linker->btf, obj->btf, t);
1461                 if (id < 0) {
1462                         pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
1463                         return id;
1464                 }
1465
1466                 obj->btf_type_map[i] = id;
1467         }
1468
1469         /* remap all the types except DATASECs */
1470         n = btf__get_nr_types(linker->btf);
1471         for (i = start_id; i <= n; i++) {
1472                 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
1473
1474                 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
1475                         return -EINVAL;
1476         }
1477
1478         /* append DATASEC info */
1479         for (i = 1; i < obj->sec_cnt; i++) {
1480                 struct src_sec *src_sec;
1481                 struct dst_sec *dst_sec;
1482                 const struct btf_var_secinfo *src_var;
1483                 struct btf_var_secinfo *dst_var;
1484
1485                 src_sec = &obj->secs[i];
1486                 if (!src_sec->sec_type_id || src_sec->skipped)
1487                         continue;
1488                 dst_sec = &linker->secs[src_sec->dst_id];
1489
1490                 /* Mark section as having BTF regardless of the presence of
1491                  * variables. In some cases compiler might generate empty BTF
1492                  * with no variables information. E.g., when promoting local
1493                  * array/structure variable initial values and BPF object
1494                  * file otherwise has no read-only static variables in
1495                  * .rodata. We need to preserve such empty BTF and just set
1496                  * correct section size.
1497                  */
1498                 dst_sec->has_btf = true;
1499
1500                 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
1501                 src_var = btf_var_secinfos(t);
1502                 n = btf_vlen(t);
1503                 for (j = 0; j < n; j++, src_var++) {
1504                         void *sec_vars = dst_sec->sec_vars;
1505
1506                         sec_vars = libbpf_reallocarray(sec_vars,
1507                                                        dst_sec->sec_var_cnt + 1,
1508                                                        sizeof(*dst_sec->sec_vars));
1509                         if (!sec_vars)
1510                                 return -ENOMEM;
1511
1512                         dst_sec->sec_vars = sec_vars;
1513                         dst_sec->sec_var_cnt++;
1514
1515                         dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
1516                         dst_var->type = obj->btf_type_map[src_var->type];
1517                         dst_var->size = src_var->size;
1518                         dst_var->offset = src_sec->dst_off + src_var->offset;
1519                 }
1520         }
1521
1522         return 0;
1523 }
1524
1525 static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
1526 {
1527         void *tmp;
1528
1529         tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
1530         if (!tmp)
1531                 return NULL;
1532         ext_data->recs = tmp;
1533
1534         tmp += ext_data->rec_cnt * ext_data->rec_sz;
1535         memcpy(tmp, src_rec, ext_data->rec_sz);
1536
1537         ext_data->rec_cnt++;
1538
1539         return tmp;
1540 }
1541
1542 static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
1543 {
1544         const struct btf_ext_info_sec *ext_sec;
1545         const char *sec_name, *s;
1546         struct src_sec *src_sec;
1547         struct dst_sec *dst_sec;
1548         int rec_sz, str_off, i;
1549
1550         if (!obj->btf_ext)
1551                 return 0;
1552
1553         rec_sz = obj->btf_ext->func_info.rec_size;
1554         for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
1555                 struct bpf_func_info_min *src_rec, *dst_rec;
1556
1557                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1558                 src_sec = find_src_sec_by_name(obj, sec_name);
1559                 if (!src_sec) {
1560                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1561                         return -EINVAL;
1562                 }
1563                 dst_sec = &linker->secs[src_sec->dst_id];
1564
1565                 if (dst_sec->func_info.rec_sz == 0)
1566                         dst_sec->func_info.rec_sz = rec_sz;
1567                 if (dst_sec->func_info.rec_sz != rec_sz) {
1568                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1569                         return -EINVAL;
1570                 }
1571
1572                 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
1573                         dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
1574                         if (!dst_rec)
1575                                 return -ENOMEM;
1576
1577                         dst_rec->insn_off += src_sec->dst_off;
1578                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
1579                 }
1580         }
1581
1582         rec_sz = obj->btf_ext->line_info.rec_size;
1583         for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
1584                 struct bpf_line_info_min *src_rec, *dst_rec;
1585
1586                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1587                 src_sec = find_src_sec_by_name(obj, sec_name);
1588                 if (!src_sec) {
1589                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1590                         return -EINVAL;
1591                 }
1592                 dst_sec = &linker->secs[src_sec->dst_id];
1593
1594                 if (dst_sec->line_info.rec_sz == 0)
1595                         dst_sec->line_info.rec_sz = rec_sz;
1596                 if (dst_sec->line_info.rec_sz != rec_sz) {
1597                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1598                         return -EINVAL;
1599                 }
1600
1601                 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
1602                         dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
1603                         if (!dst_rec)
1604                                 return -ENOMEM;
1605
1606                         dst_rec->insn_off += src_sec->dst_off;
1607
1608                         s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
1609                         str_off = btf__add_str(linker->btf, s);
1610                         if (str_off < 0)
1611                                 return -ENOMEM;
1612                         dst_rec->file_name_off = str_off;
1613
1614                         s = btf__str_by_offset(obj->btf, src_rec->line_off);
1615                         str_off = btf__add_str(linker->btf, s);
1616                         if (str_off < 0)
1617                                 return -ENOMEM;
1618                         dst_rec->line_off = str_off;
1619
1620                         /* dst_rec->line_col is fine */
1621                 }
1622         }
1623
1624         rec_sz = obj->btf_ext->core_relo_info.rec_size;
1625         for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
1626                 struct bpf_core_relo *src_rec, *dst_rec;
1627
1628                 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
1629                 src_sec = find_src_sec_by_name(obj, sec_name);
1630                 if (!src_sec) {
1631                         pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
1632                         return -EINVAL;
1633                 }
1634                 dst_sec = &linker->secs[src_sec->dst_id];
1635
1636                 if (dst_sec->core_relo_info.rec_sz == 0)
1637                         dst_sec->core_relo_info.rec_sz = rec_sz;
1638                 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
1639                         pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
1640                         return -EINVAL;
1641                 }
1642
1643                 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
1644                         dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
1645                         if (!dst_rec)
1646                                 return -ENOMEM;
1647
1648                         dst_rec->insn_off += src_sec->dst_off;
1649                         dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
1650
1651                         s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
1652                         str_off = btf__add_str(linker->btf, s);
1653                         if (str_off < 0)
1654                                 return -ENOMEM;
1655                         dst_rec->access_str_off = str_off;
1656
1657                         /* dst_rec->kind is fine */
1658                 }
1659         }
1660
1661         return 0;
1662 }
1663
1664 int bpf_linker__finalize(struct bpf_linker *linker)
1665 {
1666         struct dst_sec *sec;
1667         size_t strs_sz;
1668         const void *strs;
1669         int err, i;
1670
1671         if (!linker->elf)
1672                 return -EINVAL;
1673
1674         err = finalize_btf(linker);
1675         if (err)
1676                 return err;
1677
1678         /* Finalize strings */
1679         strs_sz = strset__data_size(linker->strtab_strs);
1680         strs = strset__data(linker->strtab_strs);
1681
1682         sec = &linker->secs[linker->strtab_sec_idx];
1683         sec->data->d_align = 1;
1684         sec->data->d_off = 0LL;
1685         sec->data->d_buf = (void *)strs;
1686         sec->data->d_type = ELF_T_BYTE;
1687         sec->data->d_size = strs_sz;
1688         sec->shdr->sh_size = strs_sz;
1689
1690         for (i = 1; i < linker->sec_cnt; i++) {
1691                 sec = &linker->secs[i];
1692
1693                 /* STRTAB is handled specially above */
1694                 if (sec->sec_idx == linker->strtab_sec_idx)
1695                         continue;
1696
1697                 /* special ephemeral sections (.ksyms, .kconfig, etc) */
1698                 if (!sec->scn)
1699                         continue;
1700
1701                 sec->data->d_buf = sec->raw_data;
1702         }
1703
1704         /* Finalize ELF layout */
1705         if (elf_update(linker->elf, ELF_C_NULL) < 0) {
1706                 err = -errno;
1707                 pr_warn_elf("failed to finalize ELF layout");
1708                 return err;
1709         }
1710
1711         /* Write out final ELF contents */
1712         if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
1713                 err = -errno;
1714                 pr_warn_elf("failed to write ELF contents");
1715                 return err;
1716         }
1717
1718         elf_end(linker->elf);
1719         close(linker->fd);
1720
1721         linker->elf = NULL;
1722         linker->fd = -1;
1723
1724         return 0;
1725 }
1726
1727 static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
1728                              size_t align, const void *raw_data, size_t raw_sz)
1729 {
1730         Elf_Scn *scn;
1731         Elf_Data *data;
1732         Elf64_Shdr *shdr;
1733         int name_off;
1734
1735         name_off = strset__add_str(linker->strtab_strs, sec_name);
1736         if (name_off < 0)
1737                 return name_off;
1738
1739         scn = elf_newscn(linker->elf);
1740         if (!scn)
1741                 return -ENOMEM;
1742         data = elf_newdata(scn);
1743         if (!data)
1744                 return -ENOMEM;
1745         shdr = elf64_getshdr(scn);
1746         if (!shdr)
1747                 return -EINVAL;
1748
1749         shdr->sh_name = name_off;
1750         shdr->sh_type = SHT_PROGBITS;
1751         shdr->sh_flags = 0;
1752         shdr->sh_size = raw_sz;
1753         shdr->sh_link = 0;
1754         shdr->sh_info = 0;
1755         shdr->sh_addralign = align;
1756         shdr->sh_entsize = 0;
1757
1758         data->d_type = ELF_T_BYTE;
1759         data->d_size = raw_sz;
1760         data->d_buf = (void *)raw_data;
1761         data->d_align = align;
1762         data->d_off = 0;
1763
1764         return 0;
1765 }
1766
1767 static int finalize_btf(struct bpf_linker *linker)
1768 {
1769         struct btf *btf = linker->btf;
1770         const void *raw_data;
1771         int i, j, id, err;
1772         __u32 raw_sz;
1773
1774         /* bail out if no BTF data was produced */
1775         if (btf__get_nr_types(linker->btf) == 0)
1776                 return 0;
1777
1778         for (i = 1; i < linker->sec_cnt; i++) {
1779                 struct dst_sec *sec = &linker->secs[i];
1780
1781                 if (!sec->has_btf)
1782                         continue;
1783
1784                 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
1785                 if (id < 0) {
1786                         pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
1787                                 sec->sec_name, id);
1788                         return id;
1789                 }
1790
1791                 for (j = 0; j < sec->sec_var_cnt; j++) {
1792                         struct btf_var_secinfo *vi = &sec->sec_vars[j];
1793
1794                         if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
1795                                 return -EINVAL;
1796                 }
1797         }
1798
1799         err = finalize_btf_ext(linker);
1800         if (err) {
1801                 pr_warn(".BTF.ext generation failed: %d\n", err);
1802                 return err;
1803         }
1804
1805         err = btf__dedup(linker->btf, linker->btf_ext, NULL);
1806         if (err) {
1807                 pr_warn("BTF dedup failed: %d\n", err);
1808                 return err;
1809         }
1810
1811         /* Emit .BTF section */
1812         raw_data = btf__get_raw_data(linker->btf, &raw_sz);
1813         if (!raw_data)
1814                 return -ENOMEM;
1815
1816         err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
1817         if (err) {
1818                 pr_warn("failed to write out .BTF ELF section: %d\n", err);
1819                 return err;
1820         }
1821
1822         /* Emit .BTF.ext section */
1823         if (linker->btf_ext) {
1824                 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
1825                 if (!raw_data)
1826                         return -ENOMEM;
1827
1828                 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
1829                 if (err) {
1830                         pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
1831                         return err;
1832                 }
1833         }
1834
1835         return 0;
1836 }
1837
1838 static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
1839                              const char *sec_name, struct btf_ext_sec_data *sec_data)
1840 {
1841         struct btf_ext_info_sec *sec_info;
1842         void *cur = output;
1843         int str_off;
1844         size_t sz;
1845
1846         if (!sec_data->rec_cnt)
1847                 return 0;
1848
1849         str_off = btf__add_str(linker->btf, sec_name);
1850         if (str_off < 0)
1851                 return -ENOMEM;
1852
1853         sec_info = cur;
1854         sec_info->sec_name_off = str_off;
1855         sec_info->num_info = sec_data->rec_cnt;
1856         cur += sizeof(struct btf_ext_info_sec);
1857
1858         sz = sec_data->rec_cnt * sec_data->rec_sz;
1859         memcpy(cur, sec_data->recs, sz);
1860         cur += sz;
1861
1862         return cur - output;
1863 }
1864
1865 static int finalize_btf_ext(struct bpf_linker *linker)
1866 {
1867         size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
1868         size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
1869         struct btf_ext_header *hdr;
1870         void *data, *cur;
1871         int i, err, sz;
1872
1873         /* validate that all sections have the same .BTF.ext record sizes
1874          * and calculate total data size for each type of data (func info,
1875          * line info, core relos)
1876          */
1877         for (i = 1; i < linker->sec_cnt; i++) {
1878                 struct dst_sec *sec = &linker->secs[i];
1879
1880                 if (sec->func_info.rec_cnt) {
1881                         if (func_rec_sz == 0)
1882                                 func_rec_sz = sec->func_info.rec_sz;
1883                         if (func_rec_sz != sec->func_info.rec_sz) {
1884                                 pr_warn("mismatch in func_info record size %zu != %u\n",
1885                                         func_rec_sz, sec->func_info.rec_sz);
1886                                 return -EINVAL;
1887                         }
1888
1889                         funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
1890                 }
1891                 if (sec->line_info.rec_cnt) {
1892                         if (line_rec_sz == 0)
1893                                 line_rec_sz = sec->line_info.rec_sz;
1894                         if (line_rec_sz != sec->line_info.rec_sz) {
1895                                 pr_warn("mismatch in line_info record size %zu != %u\n",
1896                                         line_rec_sz, sec->line_info.rec_sz);
1897                                 return -EINVAL;
1898                         }
1899
1900                         lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
1901                 }
1902                 if (sec->core_relo_info.rec_cnt) {
1903                         if (core_relo_rec_sz == 0)
1904                                 core_relo_rec_sz = sec->core_relo_info.rec_sz;
1905                         if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
1906                                 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
1907                                         core_relo_rec_sz, sec->core_relo_info.rec_sz);
1908                                 return -EINVAL;
1909                         }
1910
1911                         core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
1912                 }
1913         }
1914
1915         if (!funcs_sz && !lines_sz && !core_relos_sz)
1916                 return 0;
1917
1918         total_sz += sizeof(struct btf_ext_header);
1919         if (funcs_sz) {
1920                 funcs_sz += sizeof(__u32); /* record size prefix */
1921                 total_sz += funcs_sz;
1922         }
1923         if (lines_sz) {
1924                 lines_sz += sizeof(__u32); /* record size prefix */
1925                 total_sz += lines_sz;
1926         }
1927         if (core_relos_sz) {
1928                 core_relos_sz += sizeof(__u32); /* record size prefix */
1929                 total_sz += core_relos_sz;
1930         }
1931
1932         cur = data = calloc(1, total_sz);
1933         if (!data)
1934                 return -ENOMEM;
1935
1936         hdr = cur;
1937         hdr->magic = BTF_MAGIC;
1938         hdr->version = BTF_VERSION;
1939         hdr->flags = 0;
1940         hdr->hdr_len = sizeof(struct btf_ext_header);
1941         cur += sizeof(struct btf_ext_header);
1942
1943         /* All offsets are in bytes relative to the end of this header */
1944         hdr->func_info_off = 0;
1945         hdr->func_info_len = funcs_sz;
1946         hdr->line_info_off = funcs_sz;
1947         hdr->line_info_len = lines_sz;
1948         hdr->core_relo_off = funcs_sz + lines_sz;
1949         hdr->core_relo_len = core_relos_sz;
1950
1951         if (funcs_sz) {
1952                 *(__u32 *)cur = func_rec_sz;
1953                 cur += sizeof(__u32);
1954
1955                 for (i = 1; i < linker->sec_cnt; i++) {
1956                         struct dst_sec *sec = &linker->secs[i];
1957
1958                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
1959                         if (sz < 0) {
1960                                 err = sz;
1961                                 goto out;
1962                         }
1963
1964                         cur += sz;
1965                 }
1966         }
1967
1968         if (lines_sz) {
1969                 *(__u32 *)cur = line_rec_sz;
1970                 cur += sizeof(__u32);
1971
1972                 for (i = 1; i < linker->sec_cnt; i++) {
1973                         struct dst_sec *sec = &linker->secs[i];
1974
1975                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
1976                         if (sz < 0) {
1977                                 err = sz;
1978                                 goto out;
1979                         }
1980
1981                         cur += sz;
1982                 }
1983         }
1984
1985         if (core_relos_sz) {
1986                 *(__u32 *)cur = core_relo_rec_sz;
1987                 cur += sizeof(__u32);
1988
1989                 for (i = 1; i < linker->sec_cnt; i++) {
1990                         struct dst_sec *sec = &linker->secs[i];
1991
1992                         sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
1993                         if (sz < 0) {
1994                                 err = sz;
1995                                 goto out;
1996                         }
1997
1998                         cur += sz;
1999                 }
2000         }
2001
2002         linker->btf_ext = btf_ext__new(data, total_sz);
2003         err = libbpf_get_error(linker->btf_ext);
2004         if (err) {
2005                 linker->btf_ext = NULL;
2006                 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2007                 goto out;
2008         }
2009
2010 out:
2011         free(data);
2012         return err;
2013 }